From 6a7579fee109503e99ca7df8aa380749c2e738d4 Mon Sep 17 00:00:00 2001 From: tomyrd Date: Mon, 23 Dec 2024 12:22:24 -0300 Subject: [PATCH 01/34] feat: give interior mutability to `NodeRpcClient` --- bin/miden-cli/src/lib.rs | 12 ++- bin/miden-cli/src/tests.rs | 6 +- crates/rust-client/Cargo.toml | 2 +- crates/rust-client/src/mock.rs | 19 ++-- crates/rust-client/src/rpc/mod.rs | 37 ++++---- .../rust-client/src/rpc/tonic_client/mod.rs | 86 +++++++------------ .../src/rpc/web_tonic_client/mod.rs | 19 ++-- crates/rust-client/src/tests.rs | 2 +- tests/integration/common.rs | 2 +- tests/integration/main.rs | 2 +- 10 files changed, 81 insertions(+), 106 deletions(-) diff --git a/bin/miden-cli/src/lib.rs b/bin/miden-cli/src/lib.rs index 42107c36b..7e02a57ad 100644 --- a/bin/miden-cli/src/lib.rs +++ b/bin/miden-cli/src/lib.rs @@ -111,10 +111,14 @@ impl Cli { let authenticator = StoreAuthenticator::new_with_rng(store.clone() as Arc, rng); let client = Client::new( - Box::new(TonicRpcClient::new( - cli_config.rpc.endpoint.clone().into(), - cli_config.rpc.timeout_ms, - )), + Box::new( + TonicRpcClient::new( + cli_config.rpc.endpoint.clone().into(), + cli_config.rpc.timeout_ms, + ) + .await + .map_err(ClientError::RpcError)?, + ), rng, store as Arc, Arc::new(authenticator), diff --git a/bin/miden-cli/src/tests.rs b/bin/miden-cli/src/tests.rs index 88c7fc620..dbd6669ea 100644 --- a/bin/miden-cli/src/tests.rs +++ b/bin/miden-cli/src/tests.rs @@ -634,7 +634,11 @@ async fn create_test_client_with_store_path(store_path: &Path) -> TestClient { let authenticator = StoreAuthenticator::new_with_rng(store.clone(), rng); TestClient::new( - Box::new(TonicRpcClient::new(rpc_config.endpoint.into(), rpc_config.timeout_ms)), + Box::new( + TonicRpcClient::new(rpc_config.endpoint.into(), rpc_config.timeout_ms) + .await + .unwrap(), + ), rng, store, std::sync::Arc::new(authenticator), diff --git a/crates/rust-client/Cargo.toml b/crates/rust-client/Cargo.toml index 69038a635..b927c8c4e 100644 --- a/crates/rust-client/Cargo.toml +++ b/crates/rust-client/Cargo.toml @@ -22,7 +22,7 @@ idxdb = ["dep:base64", "dep:serde-wasm-bindgen", "dep:wasm-bindgen", "dep:wasm-b sqlite = ["dep:rusqlite", "dep:deadpool-sqlite", "std"] std = ["miden-objects/std"] testing = ["miden-objects/testing", "miden-lib/testing", "miden-tx/testing"] -tonic = ["dep:hex", "dep:prost", "dep:tonic", "std", "tonic/transport"] +tonic = ["dep:hex", "dep:prost", "dep:tonic", "std", "tonic/transport", "tokio"] web-tonic = ["dep:hex", "dep:prost", "dep:tonic", "dep:tonic-web-wasm-client"] [dependencies] diff --git a/crates/rust-client/src/mock.rs b/crates/rust-client/src/mock.rs index 040413cd4..c6b98f74c 100644 --- a/crates/rust-client/src/mock.rs +++ b/crates/rust-client/src/mock.rs @@ -204,7 +204,7 @@ use alloc::boxed::Box; #[async_trait(?Send)] impl NodeRpcClient for MockRpcApi { async fn sync_notes( - &mut self, + &self, _block_num: u32, _note_tags: &[NoteTag], ) -> Result { @@ -220,7 +220,7 @@ impl NodeRpcClient for MockRpcApi { /// Executes the specified sync state request and returns the response. async fn sync_state( - &mut self, + &self, block_num: u32, _account_ids: &[AccountId], _note_tags: &[NoteTag], @@ -235,7 +235,7 @@ impl NodeRpcClient for MockRpcApi { /// Creates and executes a [GetBlockHeaderByNumberRequest]. /// Only used for retrieving genesis block right now so that's the only case we need to cover. async fn get_block_header_by_number( - &mut self, + &self, block_num: Option, include_mmr_proof: bool, ) -> Result<(BlockHeader, Option), RpcError> { @@ -257,7 +257,7 @@ impl NodeRpcClient for MockRpcApi { Ok((block.header(), mmr_proof)) } - async fn get_notes_by_id(&mut self, note_ids: &[NoteId]) -> Result, RpcError> { + async fn get_notes_by_id(&self, note_ids: &[NoteId]) -> Result, RpcError> { // assume all off-chain notes for now let hit_notes = note_ids.iter().filter_map(|id| self.notes.get(id)); let mut return_notes = vec![]; @@ -283,22 +283,19 @@ impl NodeRpcClient for MockRpcApi { } async fn submit_proven_transaction( - &mut self, + &self, _proven_transaction: ProvenTransaction, ) -> std::result::Result<(), RpcError> { // TODO: add some basic validations to test error cases Ok(()) } - async fn get_account_update( - &mut self, - _account_id: AccountId, - ) -> Result { + async fn get_account_update(&self, _account_id: AccountId) -> Result { panic!("shouldn't be used for now") } async fn get_account_proofs( - &mut self, + &self, _account_ids: &BTreeSet, _code_commitments: Vec, _include_headers: bool, @@ -308,7 +305,7 @@ impl NodeRpcClient for MockRpcApi { } async fn check_nullifiers_by_prefix( - &mut self, + &self, _prefix: &[u16], ) -> Result, RpcError> { // Always return an empty list for now since it's only used when importing diff --git a/crates/rust-client/src/rpc/mod.rs b/crates/rust-client/src/rpc/mod.rs index fa79ddb15..754c70907 100644 --- a/crates/rust-client/src/rpc/mod.rs +++ b/crates/rust-client/src/rpc/mod.rs @@ -57,43 +57,43 @@ pub trait NodeRpcClient { /// Given a Proven Transaction, send it to the node for it to be included in a future block /// using the `/SubmitProvenTransaction` RPC endpoint. async fn submit_proven_transaction( - &mut self, + &self, proven_transaction: ProvenTransaction, ) -> Result<(), RpcError>; /// Given a block number, fetches the block header corresponding to that height from the node /// using the `/GetBlockHeaderByNumber` endpoint. /// If `include_mmr_proof` is set to true and the function returns an `Ok`, the second value - /// of the return tuple should always be Some(MmrProof). + /// of the return tuple should always be Some(MmrProof) /// /// When `None` is provided, returns info regarding the latest block. async fn get_block_header_by_number( - &mut self, + &self, block_num: Option, include_mmr_proof: bool, ) -> Result<(BlockHeader, Option), RpcError>; - /// Fetches note-related data for a list of [NoteId] using the `/GetNotesById` rpc endpoint. + /// Fetches note-related data for a list of [NoteId] using the `/GetNotesById` rpc endpoint /// /// For any NoteType::Private note, the return data is only the /// [miden_objects::notes::NoteMetadata], whereas for NoteType::Onchain notes, the return /// data includes all details. - async fn get_notes_by_id(&mut self, note_ids: &[NoteId]) -> Result, RpcError>; + async fn get_notes_by_id(&self, note_ids: &[NoteId]) -> Result, RpcError>; /// Fetches info from the node necessary to perform a state sync using the - /// `/SyncState` RPC endpoint. + /// `/SyncState` RPC endpoint /// /// - `block_num` is the last block number known by the client. The returned [StateSyncInfo] /// should contain data starting from the next block, until the first block which contains a /// note of matching the requested tag, or the chain tip if there are no notes. - /// - `account_ids` is a list of account IDs and determines the accounts the client is + /// - `account_ids` is a list of account ids and determines the accounts the client is /// interested in and should receive account updates of. /// - `note_tags` is a list of tags used to filter the notes the client is interested in, which - /// serves as a "note group" filter. Notice that you can't filter by a specific note ID. + /// serves as a "note group" filter. Notice that you can't filter by a specific note id /// - `nullifiers_tags` similar to `note_tags`, is a list of tags used to filter the nullifiers /// corresponding to some notes the client is interested in. async fn sync_state( - &mut self, + &self, block_num: u32, account_ids: &[AccountId], note_tags: &[NoteTag], @@ -101,16 +101,13 @@ pub trait NodeRpcClient { ) -> Result; /// Fetches the current state of an account from the node using the `/GetAccountDetails` RPC - /// endpoint. + /// endpoint /// - /// - `account_id` is the ID of the wanted account. - async fn get_account_update( - &mut self, - account_id: AccountId, - ) -> Result; + /// - `account_id` is the id of the wanted account. + async fn get_account_update(&self, account_id: AccountId) -> Result; async fn sync_notes( - &mut self, + &self, block_num: u32, note_tags: &[NoteTag], ) -> Result; @@ -118,24 +115,24 @@ pub trait NodeRpcClient { /// Fetches the nullifiers corresponding to a list of prefixes using the /// `/CheckNullifiersByPrefix` RPC endpoint. async fn check_nullifiers_by_prefix( - &mut self, + &self, prefix: &[u16], ) -> Result, RpcError>; /// Fetches the current account state, using th `/GetAccountProofs` RPC endpoint. async fn get_account_proofs( - &mut self, + &self, account_ids: &BTreeSet, known_account_codes: Vec, include_headers: bool, ) -> Result; - /// Fetches the commit height where the nullifier was consumed. If the nullifier isn't found, + /// Fetches the commit height where the nullifier was consumed. If the nullifier is not found, /// then `None` is returned. /// /// The default implementation of this method uses [NodeRpcClient::check_nullifiers_by_prefix]. async fn get_nullifier_commit_height( - &mut self, + &self, nullifier: &Nullifier, ) -> Result, RpcError> { let nullifiers = diff --git a/crates/rust-client/src/rpc/tonic_client/mod.rs b/crates/rust-client/src/rpc/tonic_client/mod.rs index be54f111a..1c2138f1e 100644 --- a/crates/rust-client/src/rpc/tonic_client/mod.rs +++ b/crates/rust-client/src/rpc/tonic_client/mod.rs @@ -1,9 +1,4 @@ -use alloc::{ - boxed::Box, - collections::BTreeSet, - string::{String, ToString}, - vec::Vec, -}; +use alloc::{boxed::Box, collections::BTreeSet, string::ToString, vec::Vec}; use std::{collections::BTreeMap, time::Duration}; use async_trait::async_trait; @@ -16,6 +11,7 @@ use miden_objects::{ BlockHeader, Digest, }; use miden_tx::utils::Serializable; +use tokio::sync::RwLock; use tonic::transport::Channel; use tracing::info; @@ -40,52 +36,35 @@ use crate::rpc::generated::requests::GetBlockHeaderByNumberRequest; // ================================================================================================ /// Client for the Node RPC API using tonic. -/// -/// Wraps the ApiClient which defers establishing a connection with a node until necessary. pub struct TonicRpcClient { - rpc_api: Option>, - endpoint: String, - timeout_ms: u64, + rpc_api: RwLock>, } impl TonicRpcClient { /// Returns a new instance of [TonicRpcClient] that'll do calls to the provided [Endpoint] with - /// the given timeout in milliseconds. - pub fn new(endpoint: Endpoint, timeout_ms: u64) -> TonicRpcClient { - TonicRpcClient { - rpc_api: None, - endpoint: endpoint.to_string(), - timeout_ms, - } - } + /// the given timeout in milliseconds + pub async fn new(endpoint: Endpoint, timeout_ms: u64) -> Result { + let endpoint = tonic::transport::Endpoint::try_from(endpoint.to_string()) + .map_err(|err| RpcError::ConnectionError(err.to_string()))? + .timeout(Duration::from_millis(timeout_ms)); + let rpc_api = ApiClient::connect(endpoint) + .await + .map_err(|err| RpcError::ConnectionError(err.to_string()))?; - /// Takes care of establishing the RPC connection if not connected yet and returns a reference - /// to the inner ApiClient. - async fn rpc_api(&mut self) -> Result<&mut ApiClient, RpcError> { - if self.rpc_api.is_some() { - Ok(self.rpc_api.as_mut().unwrap()) - } else { - let endpoint = tonic::transport::Endpoint::try_from(self.endpoint.clone()) - .map_err(|err| RpcError::ConnectionError(err.to_string()))? - .timeout(Duration::from_millis(self.timeout_ms)); - let rpc_api = ApiClient::connect(endpoint) - .await - .map_err(|err| RpcError::ConnectionError(err.to_string()))?; - Ok(self.rpc_api.insert(rpc_api)) - } + Ok(TonicRpcClient { rpc_api: RwLock::new(rpc_api) }) } } #[async_trait(?Send)] impl NodeRpcClient for TonicRpcClient { async fn submit_proven_transaction( - &mut self, + &self, proven_transaction: ProvenTransaction, ) -> Result<(), RpcError> { let request = SubmitProvenTransactionRequest { transaction: proven_transaction.to_bytes(), }; - let rpc_api = self.rpc_api().await?; + let mut rpc_api = self.rpc_api.write().await; rpc_api.submit_proven_transaction(request).await.map_err(|err| { RpcError::RequestError( NodeRpcClientEndpoint::SubmitProvenTx.to_string(), @@ -97,7 +76,7 @@ impl NodeRpcClient for TonicRpcClient { } async fn get_block_header_by_number( - &mut self, + &self, block_num: Option, include_mmr_proof: bool, ) -> Result<(BlockHeader, Option), RpcError> { @@ -108,7 +87,7 @@ impl NodeRpcClient for TonicRpcClient { info!("Calling GetBlockHeaderByNumber: {:?}", request); - let rpc_api = self.rpc_api().await?; + let mut rpc_api = self.rpc_api.write().await; let api_response = rpc_api.get_block_header_by_number(request).await.map_err(|err| { RpcError::RequestError( NodeRpcClientEndpoint::GetBlockHeaderByNumber.to_string(), @@ -144,11 +123,11 @@ impl NodeRpcClient for TonicRpcClient { Ok((block_header, mmr_proof)) } - async fn get_notes_by_id(&mut self, note_ids: &[NoteId]) -> Result, RpcError> { + async fn get_notes_by_id(&self, note_ids: &[NoteId]) -> Result, RpcError> { let request = GetNotesByIdRequest { note_ids: note_ids.iter().map(|id| id.inner().into()).collect(), }; - let rpc_api = self.rpc_api().await?; + let mut rpc_api = self.rpc_api.write().await; let api_response = rpc_api.get_notes_by_id(request).await.map_err(|err| { RpcError::RequestError( NodeRpcClientEndpoint::GetBlockHeaderByNumber.to_string(), @@ -198,7 +177,7 @@ impl NodeRpcClient for TonicRpcClient { /// Sends a sync state request to the Miden node, validates and converts the response /// into a [StateSyncInfo] struct. async fn sync_state( - &mut self, + &self, block_num: u32, account_ids: &[AccountId], note_tags: &[NoteTag], @@ -217,7 +196,7 @@ impl NodeRpcClient for TonicRpcClient { nullifiers, }; - let rpc_api = self.rpc_api().await?; + let mut rpc_api = self.rpc_api.write().await; let response = rpc_api.sync_state(request).await.map_err(|err| { RpcError::RequestError(NodeRpcClientEndpoint::SyncState.to_string(), err.to_string()) })?; @@ -231,17 +210,14 @@ impl NodeRpcClient for TonicRpcClient { /// /// This function will return an error if: /// - /// - There was an error sending the request to the node. + /// - There was an error sending the request to the node /// - The answer had a `None` for one of the expected fields (account, summary, account_hash, /// details). - /// - There is an error during [Account] deserialization. - async fn get_account_update( - &mut self, - account_id: AccountId, - ) -> Result { + /// - There is an error during [Account] deserialization + async fn get_account_update(&self, account_id: AccountId) -> Result { let request = GetAccountDetailsRequest { account_id: Some(account_id.into()) }; - let rpc_api = self.rpc_api().await?; + let mut rpc_api = self.rpc_api.write().await; let response = rpc_api.get_account_details(request).await.map_err(|err| { RpcError::RequestError( @@ -285,12 +261,12 @@ impl NodeRpcClient for TonicRpcClient { /// /// This function will return an error if: /// - /// - One of the requested Accounts isn't public, or isn't returned by the node. + /// - One of the requested Accounts is not public, or is not returned by the node. /// - There was an error sending the request to the node. /// - The answer had a `None` for one of the expected fields. /// - There is an error during storage deserialization. async fn get_account_proofs( - &mut self, + &self, account_ids: &BTreeSet, known_account_codes: Vec, include_headers: bool, @@ -306,7 +282,7 @@ impl NodeRpcClient for TonicRpcClient { code_commitments: known_account_codes.keys().map(|c| c.into()).collect(), }; - let rpc_api = self.rpc_api().await?; + let mut rpc_api = self.rpc_api.write().await; let response = rpc_api .get_account_proofs(request) .await @@ -362,7 +338,7 @@ impl NodeRpcClient for TonicRpcClient { } async fn sync_notes( - &mut self, + &self, block_num: u32, note_tags: &[NoteTag], ) -> Result { @@ -370,7 +346,7 @@ impl NodeRpcClient for TonicRpcClient { let request = SyncNoteRequest { block_num, note_tags }; - let rpc_api = self.rpc_api().await?; + let mut rpc_api = self.rpc_api.write().await; let response = rpc_api.sync_notes(request).await.map_err(|err| { RpcError::RequestError(NodeRpcClientEndpoint::SyncNotes.to_string(), err.to_string()) @@ -380,14 +356,14 @@ impl NodeRpcClient for TonicRpcClient { } async fn check_nullifiers_by_prefix( - &mut self, + &self, prefixes: &[u16], ) -> Result, RpcError> { let request = CheckNullifiersByPrefixRequest { nullifiers: prefixes.iter().map(|&x| x as u32).collect(), prefix_len: 16, }; - let rpc_api = self.rpc_api().await?; + let mut rpc_api = self.rpc_api.write().await; let response = rpc_api.check_nullifiers_by_prefix(request).await.map_err(|err| { RpcError::RequestError( NodeRpcClientEndpoint::CheckNullifiersByPrefix.to_string(), diff --git a/crates/rust-client/src/rpc/web_tonic_client/mod.rs b/crates/rust-client/src/rpc/web_tonic_client/mod.rs index a16825964..632a8d594 100644 --- a/crates/rust-client/src/rpc/web_tonic_client/mod.rs +++ b/crates/rust-client/src/rpc/web_tonic_client/mod.rs @@ -52,7 +52,7 @@ impl WebTonicRpcClient { #[async_trait(?Send)] impl NodeRpcClient for WebTonicRpcClient { async fn submit_proven_transaction( - &mut self, + &self, proven_transaction: ProvenTransaction, ) -> Result<(), RpcError> { let mut query_client = self.build_api_client(); @@ -72,7 +72,7 @@ impl NodeRpcClient for WebTonicRpcClient { } async fn get_block_header_by_number( - &mut self, + &self, block_num: Option, include_mmr_proof: bool, ) -> Result<(BlockHeader, Option), RpcError> { @@ -121,7 +121,7 @@ impl NodeRpcClient for WebTonicRpcClient { Ok((block_header, mmr_proof)) } - async fn get_notes_by_id(&mut self, note_ids: &[NoteId]) -> Result, RpcError> { + async fn get_notes_by_id(&self, note_ids: &[NoteId]) -> Result, RpcError> { let mut query_client = self.build_api_client(); let request = GetNotesByIdRequest { @@ -176,7 +176,7 @@ impl NodeRpcClient for WebTonicRpcClient { /// Sends a sync state request to the Miden node, validates and converts the response /// into a [StateSyncInfo] struct. async fn sync_state( - &mut self, + &self, block_num: u32, account_ids: &[AccountId], note_tags: &[NoteTag], @@ -202,7 +202,7 @@ impl NodeRpcClient for WebTonicRpcClient { } async fn sync_notes( - &mut self, + &self, block_num: u32, note_tags: &[NoteTag], ) -> Result { @@ -230,7 +230,7 @@ impl NodeRpcClient for WebTonicRpcClient { /// - The answer had a `None` for one of the expected fields. /// - There is an error during storage deserialization. async fn get_account_proofs( - &mut self, + &self, account_ids: &BTreeSet, known_account_codes: Vec, include_headers: bool, @@ -315,10 +315,7 @@ impl NodeRpcClient for WebTonicRpcClient { /// - The answer had a `None` for its account, or the account had a `None` at the `details` /// field. /// - There is an error during [Account] deserialization. - async fn get_account_update( - &mut self, - account_id: AccountId, - ) -> Result { + async fn get_account_update(&self, account_id: AccountId) -> Result { let mut query_client = self.build_api_client(); let request = GetAccountDetailsRequest { account_id: Some(account_id.into()) }; @@ -360,7 +357,7 @@ impl NodeRpcClient for WebTonicRpcClient { } async fn check_nullifiers_by_prefix( - &mut self, + &self, prefixes: &[u16], ) -> Result, RpcError> { let mut query_client = self.build_api_client(); diff --git a/crates/rust-client/src/tests.rs b/crates/rust-client/src/tests.rs index 6f28fb22d..172b5713e 100644 --- a/crates/rust-client/src/tests.rs +++ b/crates/rust-client/src/tests.rs @@ -338,7 +338,7 @@ async fn test_sync_state() { #[tokio::test] async fn test_sync_state_mmr() { // generate test client with a random store name - let (mut client, mut rpc_api) = create_test_client().await; + let (mut client, rpc_api) = create_test_client().await; // Import note and create wallet so that synced notes do not get discarded (due to being // irrelevant) insert_new_wallet(&mut client, AccountStorageMode::Private).await.unwrap(); diff --git a/tests/integration/common.rs b/tests/integration/common.rs index da08abaad..595e1aca4 100644 --- a/tests/integration/common.rs +++ b/tests/integration/common.rs @@ -58,7 +58,7 @@ pub async fn create_test_client() -> TestClient { let authenticator = StoreAuthenticator::new_with_rng(store.clone(), rng); TestClient::new( - Box::new(TonicRpcClient::new(rpc_endpoint, rpc_timeout)), + Box::new(TonicRpcClient::new(rpc_endpoint, rpc_timeout).await.unwrap()), rng, store, Arc::new(authenticator), diff --git a/tests/integration/main.rs b/tests/integration/main.rs index 091f738f8..e8ad8e814 100644 --- a/tests/integration/main.rs +++ b/tests/integration/main.rs @@ -797,7 +797,7 @@ async fn test_get_account_update() { // [AccountDetails] should be received. // TODO: should we expose the `get_account_update` endpoint from the Client? let (endpoint, timeout, _) = get_client_config(); - let mut rpc_api = TonicRpcClient::new(endpoint, timeout); + let rpc_api = TonicRpcClient::new(endpoint, timeout).await.unwrap(); let details1 = rpc_api.get_account_update(basic_wallet_1.id()).await.unwrap(); let details2 = rpc_api.get_account_update(basic_wallet_2.id()).await.unwrap(); From 991779ecd2fc4815261a209a9d4476be0afd3d7d Mon Sep 17 00:00:00 2001 From: tomyrd Date: Mon, 23 Dec 2024 13:43:49 -0300 Subject: [PATCH 02/34] refactor: move client's rpc_api to Arc --- bin/miden-cli/src/lib.rs | 2 +- bin/miden-cli/src/tests.rs | 7 ++++--- crates/rust-client/src/lib.rs | 6 +++--- crates/rust-client/src/mock.rs | 4 ++-- crates/web-client/src/lib.rs | 2 +- tests/integration/common.rs | 2 +- 6 files changed, 12 insertions(+), 11 deletions(-) diff --git a/bin/miden-cli/src/lib.rs b/bin/miden-cli/src/lib.rs index 7e02a57ad..916b6474a 100644 --- a/bin/miden-cli/src/lib.rs +++ b/bin/miden-cli/src/lib.rs @@ -111,7 +111,7 @@ impl Cli { let authenticator = StoreAuthenticator::new_with_rng(store.clone() as Arc, rng); let client = Client::new( - Box::new( + Arc::new( TonicRpcClient::new( cli_config.rpc.endpoint.clone().into(), cli_config.rpc.timeout_ms, diff --git a/bin/miden-cli/src/tests.rs b/bin/miden-cli/src/tests.rs index dbd6669ea..7bf0bfbee 100644 --- a/bin/miden-cli/src/tests.rs +++ b/bin/miden-cli/src/tests.rs @@ -3,6 +3,7 @@ use std::{ fs::File, io::Read, path::{Path, PathBuf}, + sync::Arc, }; use assert_cmd::Command; @@ -624,7 +625,7 @@ async fn create_test_client_with_store_path(store_path: &Path) -> TestClient { let store = { let sqlite_store = SqliteStore::new(PathBuf::from(store_path)).await.unwrap(); - std::sync::Arc::new(sqlite_store) + Arc::new(sqlite_store) }; let mut rng = rand::thread_rng(); @@ -634,14 +635,14 @@ async fn create_test_client_with_store_path(store_path: &Path) -> TestClient { let authenticator = StoreAuthenticator::new_with_rng(store.clone(), rng); TestClient::new( - Box::new( + Arc::new( TonicRpcClient::new(rpc_config.endpoint.into(), rpc_config.timeout_ms) .await .unwrap(), ), rng, store, - std::sync::Arc::new(authenticator), + Arc::new(authenticator), true, ) } diff --git a/crates/rust-client/src/lib.rs b/crates/rust-client/src/lib.rs index 2fb414067..8f44b2e38 100644 --- a/crates/rust-client/src/lib.rs +++ b/crates/rust-client/src/lib.rs @@ -115,7 +115,7 @@ pub struct Client { rng: R, /// An instance of [NodeRpcClient] which provides a way for the client to connect to the /// Miden node. - rpc_api: Box, + rpc_api: Arc, /// An instance of a [LocalTransactionProver] which will be the default prover for the client. tx_prover: Arc, tx_executor: TransactionExecutor, @@ -147,7 +147,7 @@ impl Client { /// /// Returns an error if the client couldn't be instantiated. pub fn new( - rpc_api: Box, + rpc_api: Arc, rng: R, store: Arc, authenticator: Arc, @@ -182,7 +182,7 @@ impl Client { // -------------------------------------------------------------------------------------------- #[cfg(any(test, feature = "testing"))] - pub fn test_rpc_api(&mut self) -> &mut Box { + pub fn test_rpc_api(&mut self) -> &mut Arc { &mut self.rpc_api } diff --git a/crates/rust-client/src/mock.rs b/crates/rust-client/src/mock.rs index c6b98f74c..8d0baed95 100644 --- a/crates/rust-client/src/mock.rs +++ b/crates/rust-client/src/mock.rs @@ -327,9 +327,9 @@ pub async fn create_test_client() -> (MockClient, MockRpcApi) { let authenticator = StoreAuthenticator::new_with_rng(store.clone(), rng); let rpc_api = MockRpcApi::new(); - let boxed_rpc_api = Box::new(rpc_api.clone()); + let arc_rpc_api = Arc::new(rpc_api.clone()); - let client = MockClient::new(boxed_rpc_api, rng, store, Arc::new(authenticator), true); + let client = MockClient::new(arc_rpc_api, rng, store, Arc::new(authenticator), true); (client, rpc_api) } diff --git a/crates/web-client/src/lib.rs b/crates/web-client/src/lib.rs index 7acf4dd91..f16051a5e 100644 --- a/crates/web-client/src/lib.rs +++ b/crates/web-client/src/lib.rs @@ -66,7 +66,7 @@ impl WebClient { .map_err(|_| JsValue::from_str("Failed to initialize WebStore"))?; let web_store = Arc::new(web_store); let authenticator = Arc::new(StoreAuthenticator::new_with_rng(web_store.clone(), rng)); - let web_rpc_client = Box::new(WebTonicRpcClient::new( + let web_rpc_client = Arc::new(WebTonicRpcClient::new( &node_url.unwrap_or_else(|| "http://18.203.155.106:57291".to_string()), )); diff --git a/tests/integration/common.rs b/tests/integration/common.rs index 595e1aca4..222972e52 100644 --- a/tests/integration/common.rs +++ b/tests/integration/common.rs @@ -58,7 +58,7 @@ pub async fn create_test_client() -> TestClient { let authenticator = StoreAuthenticator::new_with_rng(store.clone(), rng); TestClient::new( - Box::new(TonicRpcClient::new(rpc_endpoint, rpc_timeout).await.unwrap()), + Arc::new(TonicRpcClient::new(rpc_endpoint, rpc_timeout).await.unwrap()), rng, store, Arc::new(authenticator), From d7c7fb693d1f181cffd6d6ae090a564c79668a8b Mon Sep 17 00:00:00 2001 From: tomyrd Date: Thu, 2 Jan 2025 12:30:16 -0300 Subject: [PATCH 03/34] feat: add `StateSync` component (wip) --- crates/rust-client/src/accounts.rs | 27 +- .../src/rpc/domain/transactions.rs | 1 + .../src/store/sqlite_store/sync.rs | 14 +- crates/rust-client/src/sync/block_headers.rs | 7 +- crates/rust-client/src/sync/mod.rs | 94 ++-- crates/rust-client/src/sync/state_sync.rs | 453 ++++++++++++++++++ crates/rust-client/src/transactions/mod.rs | 34 +- tests/integration/common.rs | 2 +- 8 files changed, 539 insertions(+), 93 deletions(-) create mode 100644 crates/rust-client/src/sync/state_sync.rs diff --git a/crates/rust-client/src/accounts.rs b/crates/rust-client/src/accounts.rs index 71998d1b2..b92d9ed30 100644 --- a/crates/rust-client/src/accounts.rs +++ b/crates/rust-client/src/accounts.rs @@ -161,34 +161,35 @@ impl Client { // ACCOUNT UPDATES // ================================================================================================ +#[derive(Debug, Clone)] /// Contains account changes to apply to the store. pub struct AccountUpdates { /// Updated public accounts. - updated_onchain_accounts: Vec, - /// Node account hashes that don't match the tracked information. - mismatched_offchain_accounts: Vec<(AccountId, Digest)>, + updated_public_accounts: Vec, + /// Updated network account hashes for private accounts. + private_account_hashes: Vec<(AccountId, Digest)>, } impl AccountUpdates { /// Creates a new instance of `AccountUpdates`. pub fn new( - updated_onchain_accounts: Vec, - mismatched_offchain_accounts: Vec<(AccountId, Digest)>, + updated_public_accounts: Vec, + private_account_hashes: Vec<(AccountId, Digest)>, ) -> Self { Self { - updated_onchain_accounts, - mismatched_offchain_accounts, + updated_public_accounts, + private_account_hashes, } } - /// Returns the updated public accounts. - pub fn updated_onchain_accounts(&self) -> &[Account] { - &self.updated_onchain_accounts + /// Returns updated public accounts. + pub fn updated_public_accounts(&self) -> &[Account] { + &self.updated_public_accounts } - /// Returns the mismatched offchain accounts. - pub fn mismatched_offchain_accounts(&self) -> &[(AccountId, Digest)] { - &self.mismatched_offchain_accounts + /// Returns updated network account hashes for private accounts. + pub fn private_account_hashes(&self) -> &[(AccountId, Digest)] { + &self.private_account_hashes } } diff --git a/crates/rust-client/src/rpc/domain/transactions.rs b/crates/rust-client/src/rpc/domain/transactions.rs index f31e17047..d72cbe9a6 100644 --- a/crates/rust-client/src/rpc/domain/transactions.rs +++ b/crates/rust-client/src/rpc/domain/transactions.rs @@ -36,6 +36,7 @@ impl TryFrom for TransactionId { // TRANSACTION UPDATE // ================================================================================================ +#[derive(Debug, Clone)] /// Represents a transaction that was included in the node at a certain block. pub struct TransactionUpdate { /// The transaction identifier. diff --git a/crates/rust-client/src/store/sqlite_store/sync.rs b/crates/rust-client/src/store/sqlite_store/sync.rs index dab152f13..0750834cc 100644 --- a/crates/rust-client/src/store/sqlite_store/sync.rs +++ b/crates/rust-client/src/store/sqlite_store/sync.rs @@ -7,10 +7,7 @@ use rusqlite::{params, Connection, Transaction}; use super::SqliteStore; use crate::{ store::{ - sqlite_store::{ - accounts::{lock_account, update_account}, - notes::apply_note_updates_tx, - }, + sqlite_store::{accounts::update_account, notes::apply_note_updates_tx}, StoreError, }, sync::{NoteTagRecord, NoteTagSource, StateSyncUpdate}, @@ -133,13 +130,14 @@ impl SqliteStore { Self::mark_transactions_as_discarded(&tx, &discarded_transactions)?; // Update onchain accounts on the db that have been updated onchain - for account in updated_accounts.updated_onchain_accounts() { + for account in updated_accounts.updated_public_accounts() { update_account(&tx, account)?; } - for (account_id, _) in updated_accounts.mismatched_offchain_accounts() { - lock_account(&tx, *account_id)?; - } + // TODO: Lock offchain accounts that have been updated onchain + // for (account_id, _) in updated_accounts.mismatched_offchain_accounts() { + // lock_account(&tx, *account_id)?; + // } // Commit the updates tx.commit()?; diff --git a/crates/rust-client/src/sync/block_headers.rs b/crates/rust-client/src/sync/block_headers.rs index 80051ff73..6a9e5cd89 100644 --- a/crates/rust-client/src/sync/block_headers.rs +++ b/crates/rust-client/src/sync/block_headers.rs @@ -75,7 +75,7 @@ impl Client { /// relevant to the client. If any of the notes are relevant, the function returns `true`. pub(crate) async fn check_block_relevance( &mut self, - committed_notes: &NoteUpdates, + note_updates: &NoteUpdates, ) -> Result { // We'll only do the check for either incoming public notes or expected input notes as // output notes are not really candidates to be consumed here. @@ -83,10 +83,11 @@ impl Client { let note_screener = NoteScreener::new(self.store.clone()); // Find all relevant Input Notes using the note checker - for input_note in committed_notes + for input_note in note_updates .updated_input_notes() .iter() - .chain(committed_notes.new_input_notes().iter()) + .chain(note_updates.new_input_notes().iter()) + .filter(|note| note.is_committed()) { if !note_screener .check_relevance(&input_note.try_into().map_err(ClientError::NoteRecordError)?) diff --git a/crates/rust-client/src/sync/mod.rs b/crates/rust-client/src/sync/mod.rs index d59d209f0..20485dbd1 100644 --- a/crates/rust-client/src/sync/mod.rs +++ b/crates/rust-client/src/sync/mod.rs @@ -39,6 +39,9 @@ use block_headers::apply_mmr_changes; mod tags; pub use tags::{NoteTagRecord, NoteTagSource}; +mod state_sync; +use state_sync::{StateSync, SyncStatus as OtherStateSync}; + /// Contains stats about the sync operation. pub struct SyncSummary { /// Block number up to which the client has been synced. @@ -201,6 +204,7 @@ impl Client { async fn sync_state_once(&mut self) -> Result { let current_block_num = self.store.get_sync_height().await?; + let current_block = self.store.get_block_header_by_num(current_block_num).await?.0; let accounts: Vec = self .store @@ -213,54 +217,22 @@ impl Client { let note_tags: Vec = self.store.get_unique_note_tags().await?.into_iter().collect(); - // To receive information about added nullifiers, we reduce them to the higher 16 bits - // Note that besides filtering by nullifier prefixes, the node also filters by block number - // (it only returns nullifiers from current_block_num until - // response.block_header.block_num()) - let nullifiers_tags: Vec = self - .store - .get_unspent_input_note_nullifiers() - .await? - .iter() - .map(get_nullifier_prefix) - .collect(); - - // Send request - let account_ids: Vec = accounts.iter().map(|acc| acc.id()).collect(); - let response = self - .rpc_api - .sync_state(current_block_num, &account_ids, ¬e_tags, &nullifiers_tags) - .await?; - - // We don't need to continue if the chain has not advanced, there are no new changes - if response.block_header.block_num() == current_block_num { - return Ok(SyncStatus::SyncedToLastBlock(SyncSummary::new_empty(current_block_num))); - } - - let (committed_note_updates, tags_to_remove) = self - .committed_note_updates(response.note_inclusions, &response.block_header) - .await?; + let unspent_notes = self.get_input_notes(NoteFilter::Unspent).await?; - let incoming_block_has_relevant_notes = - self.check_block_relevance(&committed_note_updates).await?; + let status = + StateSync::new(self.rpc_api.clone(), current_block, accounts, note_tags, unspent_notes) + .sync_state_step() + .await?; - let transactions_to_commit = self.get_transactions_to_commit(response.transactions).await?; - - let (consumed_note_updates, transactions_to_discard) = - self.consumed_note_updates(response.nullifiers, &transactions_to_commit).await?; - - let note_updates = committed_note_updates.combine_with(consumed_note_updates); - - let (onchain_accounts, offchain_accounts): (Vec<_>, Vec<_>) = - accounts.into_iter().partition(|account_header| account_header.id().is_public()); - - let updated_onchain_accounts = self - .get_updated_onchain_accounts(&response.account_hash_updates, &onchain_accounts) - .await?; + let (is_last_block, update) = match status { + None => { + return Ok(SyncStatus::SyncedToLastBlock(SyncSummary::new_empty(current_block_num))) + }, + Some(OtherStateSync::SyncedToLastBlock(update)) => (true, update), + Some(OtherStateSync::SyncedToBlock(update)) => (false, update), + }; - let mismatched_offchain_accounts = self - .validate_local_account_hashes(&response.account_hash_updates, &offchain_accounts) - .await?; + let sync_summary: SyncSummary = (&update).into(); // Build PartialMmr with current data and apply updates let (new_peaks, new_authentication_nodes) = { @@ -271,36 +243,24 @@ impl Client { apply_mmr_changes( current_partial_mmr, - response.mmr_delta, + update.mmr_delta, current_block, has_relevant_notes, )? }; - // Store summary to return later - let sync_summary = SyncSummary::new( - response.block_header.block_num(), - note_updates.new_input_notes().iter().map(|n| n.id()).collect(), - note_updates.committed_note_ids().into_iter().collect(), - note_updates.consumed_note_ids().into_iter().collect(), - updated_onchain_accounts.iter().map(|acc| acc.id()).collect(), - mismatched_offchain_accounts.iter().map(|(acc_id, _)| *acc_id).collect(), - transactions_to_commit.iter().map(|tx| tx.transaction_id).collect(), - ); + let block_has_relevant_notes = self.check_block_relevance(&update.note_updates).await?; let state_sync_update = StateSyncUpdate { - block_header: response.block_header, - note_updates, - transactions_to_commit, + block_header: update.block_header, + note_updates: update.note_updates, + transactions_to_commit: update.transaction_updates.committed_transactions().to_vec(), new_mmr_peaks: new_peaks, new_authentication_nodes, - updated_accounts: AccountUpdates::new( - updated_onchain_accounts, - mismatched_offchain_accounts, - ), - block_has_relevant_notes: incoming_block_has_relevant_notes, - transactions_to_discard, - tags_to_remove, + updated_accounts: update.account_updates, + block_has_relevant_notes, + transactions_to_discard: update.transaction_updates.discarded_transactions().to_vec(), + tags_to_remove: update.tags_to_remove, }; // Apply received and computed updates to the store @@ -309,7 +269,7 @@ impl Client { .await .map_err(ClientError::StoreError)?; - if response.chain_tip == response.block_header.block_num() { + if is_last_block { Ok(SyncStatus::SyncedToLastBlock(sync_summary)) } else { Ok(SyncStatus::SyncedToBlock(sync_summary)) diff --git a/crates/rust-client/src/sync/state_sync.rs b/crates/rust-client/src/sync/state_sync.rs new file mode 100644 index 000000000..2fb1b4aaf --- /dev/null +++ b/crates/rust-client/src/sync/state_sync.rs @@ -0,0 +1,453 @@ +use alloc::{ + collections::{BTreeMap, BTreeSet}, + sync::Arc, + vec::Vec, +}; + +use miden_objects::{ + accounts::{Account, AccountHeader, AccountId}, + crypto::merkle::MmrDelta, + notes::{NoteId, NoteInclusionProof, NoteTag, Nullifier}, + transaction::TransactionId, + BlockHeader, Digest, +}; +use tracing::info; + +use super::{get_nullifier_prefix, NoteTagRecord, SyncSummary}; +use crate::{ + accounts::AccountUpdates, + notes::NoteUpdates, + rpc::{ + domain::{ + accounts::AccountDetails, + notes::{CommittedNote, NoteDetails}, + nullifiers::NullifierUpdate, + transactions::TransactionUpdate, + }, + NodeRpcClient, RpcError, + }, + store::{input_note_states::CommittedNoteState, InputNoteRecord}, + transactions::TransactionUpdates, + ClientError, +}; + +/// Contains all information needed to apply the update in the store after syncing with the node. +pub struct StateSyncUpdate { + /// The new block header, returned as part of the + /// [StateSyncInfo](crate::rpc::domain::sync::StateSyncInfo) + pub block_header: BlockHeader, + /// Information about note changes after the sync. + pub note_updates: NoteUpdates, + /// Information about transaction changes after the sync. + pub transaction_updates: TransactionUpdates, + /// Information to update the local partial MMR. + pub mmr_delta: MmrDelta, + /// Information abount account changes after the sync. + pub account_updates: AccountUpdates, + /// Tag records that are no longer relevant. + pub tags_to_remove: Vec, +} + +impl StateSyncUpdate { + pub fn new_empty(block_header: BlockHeader) -> Self { + Self { + block_header, + note_updates: NoteUpdates::new(vec![], vec![], vec![], vec![]), + transaction_updates: TransactionUpdates::new(vec![], vec![]), + mmr_delta: MmrDelta { forest: 0, data: Vec::new() }, + account_updates: AccountUpdates::new(vec![], vec![]), + tags_to_remove: vec![], + } + } +} + +impl From<&StateSyncUpdate> for SyncSummary { + fn from(value: &StateSyncUpdate) -> Self { + SyncSummary::new( + value.block_header.block_num(), + value.note_updates.new_input_notes().iter().map(|n| n.id()).collect(), + value.note_updates.committed_note_ids().into_iter().collect(), + value.note_updates.consumed_note_ids().into_iter().collect(), + value + .account_updates + .updated_public_accounts() + .iter() + .map(|acc| acc.id()) + .collect(), + vec![], // TODO add these fields + vec![], + ) + } +} + +pub enum SyncStatus { + SyncedToLastBlock(StateSyncUpdate), + SyncedToBlock(StateSyncUpdate), +} + +impl SyncStatus { + pub fn into_state_sync_update(self) -> StateSyncUpdate { + match self { + SyncStatus::SyncedToLastBlock(update) => update, + SyncStatus::SyncedToBlock(update) => update, + } + } +} + +pub struct StateSync { + rpc_api: Arc, + current_block: BlockHeader, + accounts: Vec, + note_tags: Vec, + unspent_notes: BTreeMap, + changed_notes: BTreeSet, +} + +impl StateSync { + pub fn new( + rpc_api: Arc, + current_block: BlockHeader, + accounts: Vec, + note_tags: Vec, + unspent_notes: Vec, + ) -> Self { + let unspent_notes = unspent_notes.into_iter().map(|note| (note.id(), note)).collect(); + + Self { + rpc_api, + current_block, + accounts, + note_tags, + unspent_notes, + changed_notes: BTreeSet::new(), + } + } + + pub async fn sync_state_step(mut self) -> Result, ClientError> { + let current_block_num = self.current_block.block_num(); + let account_ids: Vec = self.accounts.iter().map(|acc| acc.id()).collect(); + + // To receive information about added nullifiers, we reduce them to the higher 16 bits + // Note that besides filtering by nullifier prefixes, the node also filters by block number + // (it only returns nullifiers from current_block_num until + // response.block_header.block_num()) + let nullifiers_tags: Vec = self + .unspent_notes + .values() + .map(|note| get_nullifier_prefix(¬e.nullifier())) + .collect(); + + let response = self + .rpc_api + .sync_state(current_block_num, &account_ids, &self.note_tags, &nullifiers_tags) + .await?; + + // We don't need to continue if the chain has not advanced, there are no new changes + if response.block_header.block_num() == current_block_num { + return Ok(None); + } + + let (note_updates, transaction_updates) = self + .note_state_sync( + &response.block_header, + response.note_inclusions, + response.nullifiers, + response.transactions, + ) + .await?; + + // We can remove tags from notes that got committed + let tags_to_remove = note_updates + .updated_input_notes() + .iter() + .filter_map(|note| { + note.is_committed().then(|| { + NoteTagRecord::with_note_source( + note.metadata().expect("Committed note should have metadata").tag(), + note.id(), + ) + }) + }) + .collect(); + + // ACCOUNTS + let account_updates = self.account_state_sync(&response.account_hash_updates).await?; + + let update = StateSyncUpdate { + block_header: response.block_header, + note_updates, + transaction_updates, + mmr_delta: response.mmr_delta, + account_updates, + tags_to_remove, /* TODO: I think this can be removed from the update and be inferred + * from the note updates */ + }; + + if response.chain_tip == response.block_header.block_num() { + Ok(Some(SyncStatus::SyncedToLastBlock(update))) + } else { + Ok(Some(SyncStatus::SyncedToBlock(update))) + } + } + + // HELPERS + // -------------------------------------------------------------------------------------------- + + async fn account_state_sync( + &self, + account_hash_updates: &[(AccountId, Digest)], + ) -> Result { + let (onchain_accounts, offchain_accounts): (Vec<_>, Vec<_>) = + self.accounts.iter().partition(|account_header| account_header.id().is_public()); + + let updated_onchain_accounts = self + .get_updated_onchain_accounts(account_hash_updates, &onchain_accounts) + .await?; + + let private_account_hashes = account_hash_updates + .iter() + .filter_map(|(account_id, _)| { + offchain_accounts + .iter() + .find(|account| account.id() == *account_id) + .map(|account| (account.id(), account.hash())) + }) + .collect::>(); + Ok(AccountUpdates::new(updated_onchain_accounts, private_account_hashes)) + } + + async fn note_state_sync( + &mut self, + block_header: &BlockHeader, + committed_notes: Vec, + nullifiers: Vec, + committed_transactions: Vec, + ) -> Result<(NoteUpdates, TransactionUpdates), ClientError> { + let new_notes = self.committed_note_updates(committed_notes, block_header).await?; + + let discarded_transactions = + self.consumed_note_updates(nullifiers, &committed_transactions).await?; + + let modified_notes: Vec = self + .changed_notes + .iter() + .filter_map(|note_id| self.unspent_notes.remove(note_id)) + .collect(); + + //TODO: Add output notes to update + let note_updates = NoteUpdates::new(new_notes, vec![], modified_notes, vec![]); + let transaction_updates = + TransactionUpdates::new(committed_transactions, discarded_transactions); + + Ok((note_updates, transaction_updates)) + } + + /// Updates the unspent notes with the notes that were committed in the block. Returns the IDs + /// of new public notes that matched the provided tags. + async fn committed_note_updates( + &mut self, + committed_notes: Vec, + block_header: &BlockHeader, + ) -> Result, ClientError> { + let mut new_public_notes = vec![]; + + // We'll only pick committed notes that we are tracking as input/output notes. Since the + // sync response contains notes matching either the provided accounts or the provided tag + // we might get many notes when we only care about a few of those. + for committed_note in committed_notes { + let inclusion_proof = NoteInclusionProof::new( + block_header.block_num(), + committed_note.note_index(), + committed_note.merkle_path().clone(), + )?; + + if let Some(note_record) = self.unspent_notes.get_mut(committed_note.note_id()) { + // The note belongs to our locally tracked set of input notes + + let inclusion_proof_received = note_record + .inclusion_proof_received(inclusion_proof.clone(), committed_note.metadata())?; + let block_header_received = note_record.block_header_received(*block_header)?; + + if inclusion_proof_received || block_header_received { + self.changed_notes.insert(*committed_note.note_id()); + } + } else { + // The note is public and we are tracking it, push to the list of IDs to query + new_public_notes.push(*committed_note.note_id()); + } + } + + // Query the node for input note data and build the entities + let new_public_notes = + self.fetch_public_note_details(&new_public_notes, block_header).await?; + + Ok(new_public_notes) + } + + /// Updates the unspent notes to nullify those that were consumed (either internally or + /// externally). Returns the IDs of the transactions that were discarded. + async fn consumed_note_updates( + &mut self, + nullifiers: Vec, + committed_transactions: &[TransactionUpdate], + ) -> Result, ClientError> { + let nullifier_filter: Vec = nullifiers.iter().map(|n| n.nullifier).collect(); + + let mut consumed_note_ids: BTreeMap = self + .unspent_notes + .values() + .filter_map(|n| { + nullifier_filter.contains(&n.nullifier()).then(|| (n.nullifier(), n.id())) + }) + .collect(); + + // Modify notes that were being processed by a transaciton that just got committed. These + // notes were consumed internally. + for transaction_update in committed_transactions { + // Get the notes that were being processed by the transaction + let transaction_consumed_notes: Vec = consumed_note_ids + .iter() + .filter_map(|(_, note_id)| { + let note_record = self.unspent_notes.get(note_id)?; + if note_record.is_processing() + && note_record.consumer_transaction_id() + == Some(&transaction_update.transaction_id) + { + Some(note_id) + } else { + None + } + }) + .cloned() + .collect(); + + for note_id in transaction_consumed_notes { + // SAFETY: The note IDs in `consumed_note_ids` were extracted from the + // `unspent_notes` map + let input_note_record = + self.unspent_notes.get_mut(¬e_id).expect("Note should exist"); + + if input_note_record.transaction_committed( + transaction_update.transaction_id, + transaction_update.block_num, + )? { + self.changed_notes.insert(note_id); + + // Remove the note from the list so it's not modified again in the next step + consumed_note_ids.remove(&input_note_record.nullifier()); + } + } + } + + let mut discarded_transactions = vec![]; + // Modify notes that were nullified and didn't have a committed transaction to consume them + // in the previous step. These notes were consumed externally. + for nullifier_update in nullifiers { + let nullifier = nullifier_update.nullifier; + let block_num = nullifier_update.block_num; + + if let Some(note_id) = consumed_note_ids.remove(&nullifier) { + // SAFETY: The note IDs in `consumed_note_ids` were extracted from the + // `unspent_notes` map + let input_note_record = + self.unspent_notes.get_mut(¬e_id).expect("Note should exist"); + + if input_note_record.is_processing() { + // The note was being processed by a local transaction but it was nullified + // externally so the transaction should be discarded + discarded_transactions.push( + *input_note_record + .consumer_transaction_id() + .expect("Processing note should have consumer transaction id"), + ); + } + + if input_note_record.consumed_externally(nullifier, block_num)? { + self.changed_notes.insert(note_id); + } + } + } + + Ok(discarded_transactions) + } + + /// Queries the node for all received notes that aren't being locally tracked in the client. + /// + /// The client can receive metadata for private notes that it's not tracking. In this case, + /// notes are ignored for now as they become useless until details are imported. + async fn fetch_public_note_details( + &self, + query_notes: &[NoteId], + block_header: &BlockHeader, + ) -> Result, ClientError> { + if query_notes.is_empty() { + return Ok(vec![]); + } + info!("Getting note details for notes that are not being tracked."); + + let notes_data = self.rpc_api.get_notes_by_id(query_notes).await?; + let mut return_notes = Vec::with_capacity(query_notes.len()); + for note_data in notes_data { + match note_data { + NoteDetails::Private(id, ..) => { + // TODO: Is there any benefit to not ignoring these? In any case we do not have + // the recipient which is mandatory right now. + info!("Note {} is private but the client is not tracking it, ignoring.", id); + }, + NoteDetails::Public(note, inclusion_proof) => { + info!("Retrieved details for Note ID {}.", note.id()); + let inclusion_proof = NoteInclusionProof::new( + block_header.block_num(), + inclusion_proof.note_index, + inclusion_proof.merkle_path, + ) + .map_err(ClientError::NoteError)?; + let metadata = *note.metadata(); + + return_notes.push(InputNoteRecord::new( + note.into(), + None, // TODO: Add timestamp + CommittedNoteState { + metadata, + inclusion_proof, + block_note_root: block_header.note_root(), + } + .into(), + )) + }, + } + } + Ok(return_notes) + } + + async fn get_updated_onchain_accounts( + &self, + account_updates: &[(AccountId, Digest)], + current_onchain_accounts: &[&AccountHeader], + ) -> Result, ClientError> { + let mut accounts_to_update: Vec = Vec::new(); + for (remote_account_id, remote_account_hash) in account_updates { + // check if this updated account is tracked by the client + let current_account = current_onchain_accounts + .iter() + .find(|acc| *remote_account_id == acc.id() && *remote_account_hash != acc.hash()); + + if let Some(tracked_account) = current_account { + info!("Public account hash difference detected for account with ID: {}. Fetching node for updates...", tracked_account.id()); + let account_details = self.rpc_api.get_account_update(tracked_account.id()).await?; + if let AccountDetails::Public(account, _) = account_details { + // We should only do the update if it's newer, otherwise we ignore it + if account.nonce().as_int() > tracked_account.nonce().as_int() { + accounts_to_update.push(account); + } + } else { + return Err(RpcError::AccountUpdateForPrivateAccountReceived( + account_details.account_id(), + ) + .into()); + } + } + } + Ok(accounts_to_update) + } +} diff --git a/crates/rust-client/src/transactions/mod.rs b/crates/rust-client/src/transactions/mod.rs index d182e5994..477f984f8 100644 --- a/crates/rust-client/src/transactions/mod.rs +++ b/crates/rust-client/src/transactions/mod.rs @@ -29,7 +29,7 @@ use tracing::info; use super::{Client, FeltRng}; use crate::{ notes::{NoteScreener, NoteUpdates}, - rpc::domain::accounts::AccountProof, + rpc::domain::{accounts::AccountProof, transactions::TransactionUpdate}, store::{ input_note_states::ExpectedNoteState, InputNoteRecord, InputNoteState, NoteFilter, OutputNoteRecord, StoreError, TransactionFilter, @@ -52,6 +52,38 @@ pub use miden_objects::transaction::{ pub use miden_tx::{DataStoreError, TransactionExecutorError}; pub use script_builder::TransactionScriptBuilderError; +// TRANSACTION UPDATES +// -------------------------------------------------------------------------------------------- + +/// Contains transaction changes to apply to the store. +pub struct TransactionUpdates { + /// Transaction updates for any transaction that was committed between the sync request's + /// block number and the response's block number. + committed_transactions: Vec, + /// Transaction IDs for any transactions that were discarded in the sync. + discarded_transactions: Vec, +} + +impl TransactionUpdates { + pub fn new( + committed_transactions: Vec, + discarded_transactions: Vec, + ) -> Self { + Self { + committed_transactions, + discarded_transactions, + } + } + + pub fn committed_transactions(&self) -> &[TransactionUpdate] { + &self.committed_transactions + } + + pub fn discarded_transactions(&self) -> &[TransactionId] { + &self.discarded_transactions + } +} + // TRANSACTION RESULT // -------------------------------------------------------------------------------------------- diff --git a/tests/integration/common.rs b/tests/integration/common.rs index 222972e52..e00d658a2 100644 --- a/tests/integration/common.rs +++ b/tests/integration/common.rs @@ -248,7 +248,7 @@ pub async fn wait_for_node(client: &mut TestClient) { std::thread::sleep(Duration::from_secs(NODE_TIME_BETWEEN_ATTEMPTS)); }, Err(other_error) => { - panic!("Unexpected error: {other_error}"); + panic!("Unexpected error: {other_error:?}"); }, _ => return, } From 08127002993ed02e29624257c54f6601b1627b01 Mon Sep 17 00:00:00 2001 From: tomyrd Date: Thu, 2 Jan 2025 15:11:54 -0300 Subject: [PATCH 04/34] feat: remove old sync structs --- crates/rust-client/src/notes/import.rs | 4 +- crates/rust-client/src/store/mod.rs | 48 +- .../rust-client/src/store/sqlite_store/mod.rs | 8 +- .../src/store/sqlite_store/sync.rs | 15 +- crates/rust-client/src/sync/block_headers.rs | 74 +-- crates/rust-client/src/sync/mod.rs | 516 ++---------------- crates/rust-client/src/sync/state_sync.rs | 77 ++- crates/rust-client/src/tests.rs | 2 +- crates/rust-client/src/transactions/mod.rs | 2 +- 9 files changed, 168 insertions(+), 578 deletions(-) diff --git a/crates/rust-client/src/notes/import.rs b/crates/rust-client/src/notes/import.rs index b1d2f35ea..58694d144 100644 --- a/crates/rust-client/src/notes/import.rs +++ b/crates/rust-client/src/notes/import.rs @@ -170,7 +170,7 @@ impl Client { note_record.inclusion_proof_received(inclusion_proof, metadata)?; if block_height < current_block_num { - let mut current_partial_mmr = self.build_current_partial_mmr(true).await?; + let mut current_partial_mmr = self.store.build_current_partial_mmr(true).await?; let block_header = self .get_and_store_authenticated_block(block_height, &mut current_partial_mmr) @@ -214,7 +214,7 @@ impl Client { match committed_note_data { Some((metadata, inclusion_proof)) => { - let mut current_partial_mmr = self.build_current_partial_mmr(true).await?; + let mut current_partial_mmr = self.store.build_current_partial_mmr(true).await?; let block_header = self .get_and_store_authenticated_block( inclusion_proof.location().block_num(), diff --git a/crates/rust-client/src/store/mod.rs b/crates/rust-client/src/store/mod.rs index 974e4480c..d2a1526eb 100644 --- a/crates/rust-client/src/store/mod.rs +++ b/crates/rust-client/src/store/mod.rs @@ -10,7 +10,7 @@ use core::fmt::Debug; use async_trait::async_trait; use miden_objects::{ accounts::{Account, AccountCode, AccountHeader, AccountId, AuthSecretKey}, - crypto::merkle::{InOrderIndex, MmrPeaks}, + crypto::merkle::{InOrderIndex, MmrPeaks, PartialMmr}, notes::{NoteId, NoteTag, Nullifier}, BlockHeader, Digest, Word, }; @@ -208,6 +208,44 @@ pub trait Store: Send + Sync { has_client_notes: bool, ) -> Result<(), StoreError>; + /// Builds the current store view of the chain's [PartialMmr]. Because we want to add all new + /// authentication nodes that could come from applying the MMR updates, we need to track all + /// known leaves thus far. + /// + /// As part of the syncing process, we add the current block number so we don't need to + /// track it here. + async fn build_current_partial_mmr( + &self, + include_current_block: bool, + ) -> Result { + let current_block_num = self.get_sync_height().await?; + + let tracked_nodes = self.get_chain_mmr_nodes(ChainMmrNodeFilter::All).await?; + let current_peaks = self.get_chain_mmr_peaks_by_block_num(current_block_num).await?; + + let track_latest = if current_block_num != 0 { + match self.get_block_header_by_num(current_block_num - 1).await { + Ok((_, previous_block_had_notes)) => Ok(previous_block_had_notes), + Err(StoreError::BlockHeaderNotFound(_)) => Ok(false), + Err(err) => Err(err), + }? + } else { + false + }; + + let mut current_partial_mmr = + PartialMmr::from_parts(current_peaks, tracked_nodes, track_latest); + + if include_current_block { + let (current_block, has_client_notes) = + self.get_block_header_by_num(current_block_num).await?; + + current_partial_mmr.add(current_block.hash(), has_client_notes); + } + + Ok(current_partial_mmr) + } + // ACCOUNT // -------------------------------------------------------------------------------------------- @@ -324,9 +362,13 @@ pub trait Store: Send + Sync { /// - Updating the corresponding tracked input/output notes. /// - Removing note tags that are no longer relevant. /// - Updating transactions in the store, marking as `committed` or `discarded`. - /// - Storing new MMR authentication nodes. + /// - Applies the MMR delta to the store. /// - Updating the tracked on-chain accounts. - async fn apply_state_sync(&self, state_sync_update: StateSyncUpdate) -> Result<(), StoreError>; + async fn apply_state_sync_step( + &self, + state_sync_update: StateSyncUpdate, + new_block_has_relevant_notes: bool, + ) -> Result<(), StoreError>; } // CHAIN MMR NODE FILTER diff --git a/crates/rust-client/src/store/sqlite_store/mod.rs b/crates/rust-client/src/store/sqlite_store/mod.rs index c70294b98..832ece5e0 100644 --- a/crates/rust-client/src/store/sqlite_store/mod.rs +++ b/crates/rust-client/src/store/sqlite_store/mod.rs @@ -137,9 +137,13 @@ impl Store for SqliteStore { self.interact_with_connection(SqliteStore::get_sync_height).await } - async fn apply_state_sync(&self, state_sync_update: StateSyncUpdate) -> Result<(), StoreError> { + async fn apply_state_sync_step( + &self, + state_sync_update: StateSyncUpdate, + block_has_relevant_notes: bool, + ) -> Result<(), StoreError> { self.interact_with_connection(move |conn| { - SqliteStore::apply_state_sync(conn, state_sync_update) + SqliteStore::apply_state_sync_step(conn, state_sync_update, block_has_relevant_notes) }) .await } diff --git a/crates/rust-client/src/store/sqlite_store/sync.rs b/crates/rust-client/src/store/sqlite_store/sync.rs index 0750834cc..541dff009 100644 --- a/crates/rust-client/src/store/sqlite_store/sync.rs +++ b/crates/rust-client/src/store/sqlite_store/sync.rs @@ -88,19 +88,18 @@ impl SqliteStore { .expect("state sync block number exists") } - pub(super) fn apply_state_sync( + pub(super) fn apply_state_sync_step( conn: &mut Connection, state_sync_update: StateSyncUpdate, + block_has_relevant_notes: bool, ) -> Result<(), StoreError> { let StateSyncUpdate { block_header, note_updates, - transactions_to_commit: committed_transactions, + transaction_updates, new_mmr_peaks, new_authentication_nodes, - updated_accounts, - block_has_relevant_notes, - transactions_to_discard: discarded_transactions, + account_updates, tags_to_remove, } = state_sync_update; @@ -124,13 +123,13 @@ impl SqliteStore { Self::insert_chain_mmr_nodes_tx(&tx, &new_authentication_nodes)?; // Mark transactions as committed - Self::mark_transactions_as_committed(&tx, &committed_transactions)?; + Self::mark_transactions_as_committed(&tx, transaction_updates.committed_transactions())?; // Marc transactions as discarded - Self::mark_transactions_as_discarded(&tx, &discarded_transactions)?; + Self::mark_transactions_as_discarded(&tx, transaction_updates.discarded_transactions())?; // Update onchain accounts on the db that have been updated onchain - for account in updated_accounts.updated_public_accounts() { + for account in account_updates.updated_public_accounts() { update_account(&tx, account)?; } diff --git a/crates/rust-client/src/sync/block_headers.rs b/crates/rust-client/src/sync/block_headers.rs index 6a9e5cd89..9ceb38ef6 100644 --- a/crates/rust-client/src/sync/block_headers.rs +++ b/crates/rust-client/src/sync/block_headers.rs @@ -1,6 +1,6 @@ use alloc::vec::Vec; -use crypto::merkle::{InOrderIndex, MmrDelta, MmrPeaks, PartialMmr}; +use crypto::merkle::{InOrderIndex, MmrPeaks, PartialMmr}; use miden_objects::{ block::{block_epoch_from_number, block_num_from_epoch}, crypto::{self, merkle::MerklePath, rand::FeltRng}, @@ -11,7 +11,7 @@ use tracing::warn; use super::NoteUpdates; use crate::{ notes::NoteScreener, - store::{ChainMmrNodeFilter, NoteFilter, StoreError}, + store::{NoteFilter, StoreError}, Client, ClientError, }; @@ -20,7 +20,7 @@ impl Client { /// Updates committed notes with no MMR data. These could be notes that were /// imported with an inclusion proof, but its block header isn't tracked. pub(crate) async fn update_mmr_data(&mut self) -> Result<(), ClientError> { - let mut current_partial_mmr = self.build_current_partial_mmr(true).await?; + let mut current_partial_mmr = self.store.build_current_partial_mmr(true).await?; let mut changed_notes = vec![]; for mut note in self.store.get_input_notes(NoteFilter::Unverified).await? { @@ -74,7 +74,7 @@ impl Client { /// Checks the relevance of the block by verifying if any of the input notes in the block are /// relevant to the client. If any of the notes are relevant, the function returns `true`. pub(crate) async fn check_block_relevance( - &mut self, + &self, note_updates: &NoteUpdates, ) -> Result { // We'll only do the check for either incoming public notes or expected input notes as @@ -101,44 +101,6 @@ impl Client { Ok(false) } - /// Builds the current view of the chain's [PartialMmr]. Because we want to add all new - /// authentication nodes that could come from applying the MMR updates, we need to track all - /// known leaves thus far. - /// - /// As part of the syncing process, we add the current block number so we don't need to - /// track it here. - pub(crate) async fn build_current_partial_mmr( - &self, - include_current_block: bool, - ) -> Result { - let current_block_num = self.store.get_sync_height().await?; - - let tracked_nodes = self.store.get_chain_mmr_nodes(ChainMmrNodeFilter::All).await?; - let current_peaks = self.store.get_chain_mmr_peaks_by_block_num(current_block_num).await?; - - let track_latest = if current_block_num != 0 { - match self.store.get_block_header_by_num(current_block_num - 1).await { - Ok((_, previous_block_had_notes)) => Ok(previous_block_had_notes), - Err(StoreError::BlockHeaderNotFound(_)) => Ok(false), - Err(err) => Err(ClientError::StoreError(err)), - }? - } else { - false - }; - - let mut current_partial_mmr = - PartialMmr::from_parts(current_peaks, tracked_nodes, track_latest); - - if include_current_block { - let (current_block, has_client_notes) = - self.store.get_block_header_by_num(current_block_num).await?; - - current_partial_mmr.add(current_block.hash(), has_client_notes); - } - - Ok(current_partial_mmr) - } - /// Retrieves and stores a [BlockHeader] by number, and stores its authentication data as well. /// /// If the store already contains MMR data for the requested block number, the request isn't @@ -196,7 +158,7 @@ impl Client { return self.ensure_genesis_in_place().await; } - let mut current_partial_mmr = self.build_current_partial_mmr(true).await?; + let mut current_partial_mmr = self.store.build_current_partial_mmr(true).await?; let anchor_block = self .get_and_store_authenticated_block(epoch_block_number, &mut current_partial_mmr) .await?; @@ -249,29 +211,3 @@ fn adjust_merkle_path_for_forest( path_nodes } - -/// Applies changes to the Mmr structure, storing authentication nodes for leaves we track -/// and returns the updated [PartialMmr]. -pub(crate) fn apply_mmr_changes( - current_partial_mmr: PartialMmr, - mmr_delta: MmrDelta, - current_block_header: BlockHeader, - current_block_has_relevant_notes: bool, -) -> Result<(MmrPeaks, Vec<(InOrderIndex, Digest)>), StoreError> { - let mut partial_mmr: PartialMmr = current_partial_mmr; - - // First, apply curent_block to the Mmr - let new_authentication_nodes = partial_mmr - .add(current_block_header.hash(), current_block_has_relevant_notes) - .into_iter(); - - // Apply the Mmr delta to bring Mmr to forest equal to chain tip - let new_authentication_nodes: Vec<(InOrderIndex, Digest)> = partial_mmr - .apply(mmr_delta) - .map_err(StoreError::MmrError)? - .into_iter() - .chain(new_authentication_nodes) - .collect(); - - Ok((partial_mmr.peaks(), new_authentication_nodes)) -} diff --git a/crates/rust-client/src/sync/mod.rs b/crates/rust-client/src/sync/mod.rs index 20485dbd1..acd8ffc7a 100644 --- a/crates/rust-client/src/sync/mod.rs +++ b/crates/rust-client/src/sync/mod.rs @@ -1,46 +1,25 @@ //! Provides the client APIs for synchronizing the client's local state with the Miden //! rollup network. It ensures that the client maintains a valid, up-to-date view of the chain. -use alloc::{collections::BTreeMap, vec::Vec}; +use alloc::vec::Vec; use core::cmp::max; -use crypto::merkle::{InOrderIndex, MmrPeaks}; use miden_objects::{ - accounts::{Account, AccountHeader, AccountId}, - crypto::{self, rand::FeltRng}, - notes::{NoteId, NoteInclusionProof, NoteTag, Nullifier}, + accounts::{AccountHeader, AccountId}, + crypto::rand::FeltRng, + notes::{NoteId, NoteTag, Nullifier}, transaction::TransactionId, - BlockHeader, Digest, }; -use tracing::info; -use crate::{ - accounts::AccountUpdates, - notes::NoteUpdates, - rpc::{ - domain::{ - accounts::AccountDetails, - notes::{CommittedNote, NoteDetails}, - nullifiers::NullifierUpdate, - transactions::TransactionUpdate, - }, - RpcError, - }, - store::{ - input_note_states::CommittedNoteState, InputNoteRecord, NoteFilter, OutputNoteRecord, - TransactionFilter, - }, - Client, ClientError, -}; +use crate::{notes::NoteUpdates, store::NoteFilter, Client, ClientError}; mod block_headers; -use block_headers::apply_mmr_changes; mod tags; pub use tags::{NoteTagRecord, NoteTagSource}; mod state_sync; -use state_sync::{StateSync, SyncStatus as OtherStateSync}; +pub use state_sync::{StateSync, StateSyncUpdate, SyncStatus}; /// Contains stats about the sync operation. pub struct SyncSummary { @@ -111,45 +90,6 @@ impl SyncSummary { } } -enum SyncStatus { - SyncedToLastBlock(SyncSummary), - SyncedToBlock(SyncSummary), -} - -impl SyncStatus { - pub fn into_sync_summary(self) -> SyncSummary { - match self { - SyncStatus::SyncedToLastBlock(summary) => summary, - SyncStatus::SyncedToBlock(summary) => summary, - } - } -} - -/// Contains all information needed to apply the update in the store after syncing with the node. -pub struct StateSyncUpdate { - /// The new block header, returned as part of the - /// [StateSyncInfo](crate::rpc::domain::sync::StateSyncInfo) - pub block_header: BlockHeader, - /// Information about note changes after the sync. - pub note_updates: NoteUpdates, - /// Transaction updates for any transaction that was committed between the sync request's - /// block number and the response's block number. - pub transactions_to_commit: Vec, - /// Transaction IDs for any transactions that were discarded in the sync. - pub transactions_to_discard: Vec, - /// New MMR peaks for the locally tracked MMR of the blockchain. - pub new_mmr_peaks: MmrPeaks, - /// New authentications nodes that are meant to be stored in order to authenticate block - /// headers. - pub new_authentication_nodes: Vec<(InOrderIndex, Digest)>, - /// Information abount account changes after the sync. - pub updated_accounts: AccountUpdates, - /// Whether the block header has notes relevant to the client. - pub block_has_relevant_notes: bool, - /// Tag records that are no longer relevant. - pub tags_to_remove: Vec, -} - // CONSTANTS // ================================================================================================ @@ -187,423 +127,71 @@ impl Client { /// 8. All updates are applied to the store to be persisted. pub async fn sync_state(&mut self) -> Result { _ = self.ensure_genesis_in_place().await?; - let mut total_sync_summary = SyncSummary::new_empty(0); - loop { - let response = self.sync_state_once().await?; - let is_last_block = matches!(response, SyncStatus::SyncedToLastBlock(_)); - total_sync_summary.combine_with(response.into_sync_summary()); - - if is_last_block { - break; - } - } - self.update_mmr_data().await?; - - Ok(total_sync_summary) - } - async fn sync_state_once(&mut self) -> Result { let current_block_num = self.store.get_sync_height().await?; - let current_block = self.store.get_block_header_by_num(current_block_num).await?.0; - - let accounts: Vec = self - .store - .get_account_headers() - .await? - .into_iter() - .map(|(acc_header, _)| acc_header) - .collect(); - - let note_tags: Vec = - self.store.get_unique_note_tags().await?.into_iter().collect(); - - let unspent_notes = self.get_input_notes(NoteFilter::Unspent).await?; - - let status = - StateSync::new(self.rpc_api.clone(), current_block, accounts, note_tags, unspent_notes) - .sync_state_step() - .await?; - - let (is_last_block, update) = match status { - None => { - return Ok(SyncStatus::SyncedToLastBlock(SyncSummary::new_empty(current_block_num))) - }, - Some(OtherStateSync::SyncedToLastBlock(update)) => (true, update), - Some(OtherStateSync::SyncedToBlock(update)) => (false, update), - }; - - let sync_summary: SyncSummary = (&update).into(); - - // Build PartialMmr with current data and apply updates - let (new_peaks, new_authentication_nodes) = { - let current_partial_mmr = self.build_current_partial_mmr(false).await?; + let mut total_sync_summary = SyncSummary::new_empty(current_block_num); + loop { + // Get current state of the client + let current_block_num = self.store.get_sync_height().await?; let (current_block, has_relevant_notes) = self.store.get_block_header_by_num(current_block_num).await?; - apply_mmr_changes( - current_partial_mmr, - update.mmr_delta, - current_block, - has_relevant_notes, - )? - }; - - let block_has_relevant_notes = self.check_block_relevance(&update.note_updates).await?; - - let state_sync_update = StateSyncUpdate { - block_header: update.block_header, - note_updates: update.note_updates, - transactions_to_commit: update.transaction_updates.committed_transactions().to_vec(), - new_mmr_peaks: new_peaks, - new_authentication_nodes, - updated_accounts: update.account_updates, - block_has_relevant_notes, - transactions_to_discard: update.transaction_updates.discarded_transactions().to_vec(), - tags_to_remove: update.tags_to_remove, - }; - - // Apply received and computed updates to the store - self.store - .apply_state_sync(state_sync_update) - .await - .map_err(ClientError::StoreError)?; - - if is_last_block { - Ok(SyncStatus::SyncedToLastBlock(sync_summary)) - } else { - Ok(SyncStatus::SyncedToBlock(sync_summary)) - } - } - - // HELPERS - // -------------------------------------------------------------------------------------------- - - /// Returns the [NoteUpdates] containing new public note and committed input/output notes and a - /// list or note tag records to be removed from the store. - async fn committed_note_updates( - &mut self, - committed_notes: Vec, - block_header: &BlockHeader, - ) -> Result<(NoteUpdates, Vec), ClientError> { - // We'll only pick committed notes that we are tracking as input/output notes. Since the - // sync response contains notes matching either the provided accounts or the provided tag - // we might get many notes when we only care about a few of those. - let relevant_note_filter = - NoteFilter::List(committed_notes.iter().map(|note| note.note_id()).cloned().collect()); - - let mut committed_input_notes: BTreeMap = self - .store - .get_input_notes(relevant_note_filter.clone()) - .await? - .into_iter() - .map(|n| (n.id(), n)) - .collect(); - - let mut committed_output_notes: BTreeMap = self - .store - .get_output_notes(relevant_note_filter) - .await? - .into_iter() - .map(|n| (n.id(), n)) - .collect(); - - let mut new_public_notes = vec![]; - let mut committed_tracked_input_notes = vec![]; - let mut committed_tracked_output_notes = vec![]; - let mut removed_tags = vec![]; - - for committed_note in committed_notes { - let inclusion_proof = NoteInclusionProof::new( - block_header.block_num(), - committed_note.note_index(), - committed_note.merkle_path().clone(), - )?; - - if let Some(mut note_record) = committed_input_notes.remove(committed_note.note_id()) { - // The note belongs to our locally tracked set of input notes - - let inclusion_proof_received = note_record - .inclusion_proof_received(inclusion_proof.clone(), committed_note.metadata())?; - let block_header_received = note_record.block_header_received(*block_header)?; - - removed_tags.push((¬e_record).try_into()?); - - if inclusion_proof_received || block_header_received { - committed_tracked_input_notes.push(note_record); - } - } - - if let Some(mut note_record) = committed_output_notes.remove(committed_note.note_id()) { - // The note belongs to our locally tracked set of output notes - - if note_record.inclusion_proof_received(inclusion_proof.clone())? { - committed_tracked_output_notes.push(note_record); - } - } - - if !committed_input_notes.contains_key(committed_note.note_id()) - && !committed_output_notes.contains_key(committed_note.note_id()) - { - // The note is public and we are not tracking it, push to the list of IDs to query - new_public_notes.push(*committed_note.note_id()); - } - } - - // Query the node for input note data and build the entities - let new_public_notes = - self.fetch_public_note_details(&new_public_notes, block_header).await?; - - Ok(( - NoteUpdates::new( - new_public_notes, - vec![], - committed_tracked_input_notes, - committed_tracked_output_notes, - ), - removed_tags, - )) - } + let accounts: Vec = self + .store + .get_account_headers() + .await? + .into_iter() + .map(|(acc_header, _)| acc_header) + .collect(); - /// Returns the [NoteUpdates] containing consumed input/output notes and a list of IDs of the - /// transactions that were discarded. - async fn consumed_note_updates( - &mut self, - nullifiers: Vec, - committed_transactions: &[TransactionUpdate], - ) -> Result<(NoteUpdates, Vec), ClientError> { - let nullifier_filter = NoteFilter::Nullifiers( - nullifiers.iter().map(|nullifier_update| nullifier_update.nullifier).collect(), - ); + let note_tags: Vec = + self.store.get_unique_note_tags().await?.into_iter().collect(); - let mut consumed_input_notes: BTreeMap = self - .store - .get_input_notes(nullifier_filter.clone()) - .await? - .into_iter() - .map(|n| (n.nullifier(), n)) - .collect(); + let unspent_notes = self.get_input_notes(NoteFilter::Unspent).await?; - let mut consumed_output_notes: BTreeMap = self - .store - .get_output_notes(nullifier_filter) - .await? - .into_iter() - .map(|n| { + // Get the sync update from the network + let status = StateSync::new( + self.rpc_api.clone(), + current_block, + has_relevant_notes, + accounts, + note_tags, + unspent_notes, + self.store.build_current_partial_mmr(false).await?, + ) + .sync_state_step() + .await?; + + let (is_last_block, state_sync_update) = if let Some(status) = status { ( - n.nullifier() - .expect("Output notes returned by this query should have nullifiers"), - n, + matches!(status, SyncStatus::SyncedToLastBlock(_)), + status.into_state_sync_update(), ) - }) - .collect(); - - let mut consumed_tracked_input_notes = vec![]; - let mut consumed_tracked_output_notes = vec![]; - - // Committed transactions - for transaction_update in committed_transactions { - let transaction_nullifiers: Vec = consumed_input_notes - .iter() - .filter_map(|(nullifier, note_record)| { - if note_record.is_processing() - && note_record.consumer_transaction_id() - == Some(&transaction_update.transaction_id) - { - Some(nullifier) - } else { - None - } - }) - .cloned() - .collect(); - - for nullifier in transaction_nullifiers { - if let Some(mut input_note_record) = consumed_input_notes.remove(&nullifier) { - if input_note_record.transaction_committed( - transaction_update.transaction_id, - transaction_update.block_num, - )? { - consumed_tracked_input_notes.push(input_note_record); - } - } - } - } - - // Nullified notes - let mut discarded_transactions = vec![]; - for nullifier_update in nullifiers { - let nullifier = nullifier_update.nullifier; - let block_num = nullifier_update.block_num; - - if let Some(mut input_note_record) = consumed_input_notes.remove(&nullifier) { - if input_note_record.is_processing() { - discarded_transactions.push( - *input_note_record - .consumer_transaction_id() - .expect("Processing note should have consumer transaction id"), - ); - } - - if input_note_record.consumed_externally(nullifier, block_num)? { - consumed_tracked_input_notes.push(input_note_record); - } - } - - if let Some(mut output_note_record) = consumed_output_notes.remove(&nullifier) { - if output_note_record.nullifier_received(nullifier, block_num)? { - consumed_tracked_output_notes.push(output_note_record); - } - } - } - - Ok(( - NoteUpdates::new( - vec![], - vec![], - consumed_tracked_input_notes, - consumed_tracked_output_notes, - ), - discarded_transactions, - )) - } - - /// Queries the node for all received notes that aren't being locally tracked in the client. - /// - /// The client can receive metadata for private notes that it's not tracking. In this case, - /// notes are ignored for now as they become useless until details are imported. - async fn fetch_public_note_details( - &mut self, - query_notes: &[NoteId], - block_header: &BlockHeader, - ) -> Result, ClientError> { - if query_notes.is_empty() { - return Ok(vec![]); - } - info!("Getting note details for notes that are not being tracked."); - - let notes_data = self.rpc_api.get_notes_by_id(query_notes).await?; - let mut return_notes = Vec::with_capacity(query_notes.len()); - for note_data in notes_data { - match note_data { - NoteDetails::Private(id, ..) => { - // TODO: Is there any benefit to not ignoring these? In any case we do not have - // the recipient which is mandatory right now. - info!("Note {} is private but the client is not tracking it, ignoring.", id); - }, - NoteDetails::Public(note, inclusion_proof) => { - info!("Retrieved details for Note ID {}.", note.id()); - let inclusion_proof = NoteInclusionProof::new( - block_header.block_num(), - inclusion_proof.note_index, - inclusion_proof.merkle_path, - ) - .map_err(ClientError::NoteError)?; - let metadata = *note.metadata(); - - return_notes.push(InputNoteRecord::new( - note.into(), - self.store.get_current_timestamp(), - CommittedNoteState { - metadata, - inclusion_proof, - block_note_root: block_header.note_root(), - } - .into(), - )) - }, - } - } - Ok(return_notes) - } + } else { + break; + }; - /// Extracts information about transactions for uncommitted transactions that the client is - /// tracking from the received [SyncStateResponse]. - async fn get_transactions_to_commit( - &self, - mut transactions: Vec, - ) -> Result, ClientError> { - // Get current uncommitted transactions - let uncommitted_transaction_ids = self - .store - .get_transactions(TransactionFilter::Uncomitted) - .await? - .into_iter() - .map(|tx| tx.id) - .collect::>(); + let sync_summary: SyncSummary = (&state_sync_update).into(); - transactions.retain(|transaction_update| { - uncommitted_transaction_ids.contains(&transaction_update.transaction_id) - }); + let has_relevant_notes = + self.check_block_relevance(&state_sync_update.note_updates).await?; - Ok(transactions) - } + // Apply received and computed updates to the store + self.store + .apply_state_sync_step(state_sync_update, has_relevant_notes) + .await + .map_err(ClientError::StoreError)?; - async fn get_updated_onchain_accounts( - &mut self, - account_updates: &[(AccountId, Digest)], - current_onchain_accounts: &[AccountHeader], - ) -> Result, ClientError> { - let mut accounts_to_update: Vec = Vec::new(); - for (remote_account_id, remote_account_hash) in account_updates { - // check if this updated account is tracked by the client - let current_account = current_onchain_accounts - .iter() - .find(|acc| *remote_account_id == acc.id() && *remote_account_hash != acc.hash()); + total_sync_summary.combine_with(sync_summary); - if let Some(tracked_account) = current_account { - info!("Public account hash difference detected for account with ID: {}. Fetching node for updates...", tracked_account.id()); - let account_details = self.rpc_api.get_account_update(tracked_account.id()).await?; - if let AccountDetails::Public(account, _) = account_details { - // We should only do the update if it's newer, otherwise we ignore it - if account.nonce().as_int() > tracked_account.nonce().as_int() { - accounts_to_update.push(account); - } - } else { - return Err(RpcError::AccountUpdateForPrivateAccountReceived( - account_details.account_id(), - ) - .into()); - } + if is_last_block { + break; } } - Ok(accounts_to_update) - } - - /// Validates account hash updates and returns a vector with all the offchain account - /// mismatches. - /// - /// Offchain account mismatches happen when the hash account of the local tracked account - /// doesn't match the hash account of the account in the node. This would be an anomaly and may - /// happen for two main reasons: - /// - A different client made a transaction with the account, changing its state. - /// - The local transaction that modified the local state didn't go through, rendering the local - /// account state outdated. - async fn validate_local_account_hashes( - &mut self, - account_updates: &[(AccountId, Digest)], - current_offchain_accounts: &[AccountHeader], - ) -> Result, ClientError> { - let mut mismatched_accounts = vec![]; - - for (remote_account_id, remote_account_hash) in account_updates { - // ensure that if we track that account, it has the same hash - let mismatched_account = current_offchain_accounts - .iter() - .find(|acc| *remote_account_id == acc.id() && *remote_account_hash != acc.hash()); - - // OffChain accounts should always have the latest known state. If we receive a stale - // update we ignore it. - if mismatched_account.is_some() { - let account_by_hash = - self.store.get_account_header_by_hash(*remote_account_hash).await?; + self.update_mmr_data().await?; - if account_by_hash.is_none() { - mismatched_accounts.push((*remote_account_id, *remote_account_hash)); - } - } - } - Ok(mismatched_accounts) + Ok(total_sync_summary) } } diff --git a/crates/rust-client/src/sync/state_sync.rs b/crates/rust-client/src/sync/state_sync.rs index 2fb1b4aaf..b3f99e23b 100644 --- a/crates/rust-client/src/sync/state_sync.rs +++ b/crates/rust-client/src/sync/state_sync.rs @@ -6,7 +6,7 @@ use alloc::{ use miden_objects::{ accounts::{Account, AccountHeader, AccountId}, - crypto::merkle::MmrDelta, + crypto::merkle::{InOrderIndex, MmrDelta, MmrPeaks, PartialMmr}, notes::{NoteId, NoteInclusionProof, NoteTag, Nullifier}, transaction::TransactionId, BlockHeader, Digest, @@ -26,7 +26,7 @@ use crate::{ }, NodeRpcClient, RpcError, }, - store::{input_note_states::CommittedNoteState, InputNoteRecord}, + store::{input_note_states::CommittedNoteState, InputNoteRecord, StoreError}, transactions::TransactionUpdates, ClientError, }; @@ -40,27 +40,17 @@ pub struct StateSyncUpdate { pub note_updates: NoteUpdates, /// Information about transaction changes after the sync. pub transaction_updates: TransactionUpdates, - /// Information to update the local partial MMR. - pub mmr_delta: MmrDelta, + /// New MMR peaks for the locally tracked MMR of the blockchain. + pub new_mmr_peaks: MmrPeaks, + /// New authentications nodes that are meant to be stored in order to authenticate block + /// headers. + pub new_authentication_nodes: Vec<(InOrderIndex, Digest)>, /// Information abount account changes after the sync. pub account_updates: AccountUpdates, /// Tag records that are no longer relevant. pub tags_to_remove: Vec, } -impl StateSyncUpdate { - pub fn new_empty(block_header: BlockHeader) -> Self { - Self { - block_header, - note_updates: NoteUpdates::new(vec![], vec![], vec![], vec![]), - transaction_updates: TransactionUpdates::new(vec![], vec![]), - mmr_delta: MmrDelta { forest: 0, data: Vec::new() }, - account_updates: AccountUpdates::new(vec![], vec![]), - tags_to_remove: vec![], - } - } -} - impl From<&StateSyncUpdate> for SyncSummary { fn from(value: &StateSyncUpdate) -> Self { SyncSummary::new( @@ -97,29 +87,35 @@ impl SyncStatus { pub struct StateSync { rpc_api: Arc, current_block: BlockHeader, + current_block_has_relevant_notes: bool, accounts: Vec, note_tags: Vec, unspent_notes: BTreeMap, changed_notes: BTreeSet, + current_partial_mmr: PartialMmr, } impl StateSync { pub fn new( rpc_api: Arc, current_block: BlockHeader, + current_block_has_relevant_notes: bool, accounts: Vec, note_tags: Vec, unspent_notes: Vec, + current_partial_mmr: PartialMmr, ) -> Self { let unspent_notes = unspent_notes.into_iter().map(|note| (note.id(), note)).collect(); Self { rpc_api, current_block, + current_block_has_relevant_notes, accounts, note_tags, unspent_notes, changed_notes: BTreeSet::new(), + current_partial_mmr, } } @@ -160,24 +156,27 @@ impl StateSync { let tags_to_remove = note_updates .updated_input_notes() .iter() - .filter_map(|note| { - note.is_committed().then(|| { - NoteTagRecord::with_note_source( - note.metadata().expect("Committed note should have metadata").tag(), - note.id(), - ) - }) + .filter(|note| note.is_committed()) + .map(|note| { + NoteTagRecord::with_note_source( + note.metadata().expect("Committed note should have metadata").tag(), + note.id(), + ) }) .collect(); // ACCOUNTS let account_updates = self.account_state_sync(&response.account_hash_updates).await?; + // MMR + let new_authentication_nodes = self.update_partial_mmr(response.mmr_delta).await?; + let update = StateSyncUpdate { block_header: response.block_header, note_updates, transaction_updates, - mmr_delta: response.mmr_delta, + new_mmr_peaks: self.current_partial_mmr.peaks(), + new_authentication_nodes, account_updates, tags_to_remove, /* TODO: I think this can be removed from the update and be inferred * from the note updates */ @@ -296,9 +295,8 @@ impl StateSync { let mut consumed_note_ids: BTreeMap = self .unspent_notes .values() - .filter_map(|n| { - nullifier_filter.contains(&n.nullifier()).then(|| (n.nullifier(), n.id())) - }) + .filter(|&n| nullifier_filter.contains(&n.nullifier())) + .map(|n| (n.nullifier(), n.id())) .collect(); // Modify notes that were being processed by a transaciton that just got committed. These @@ -450,4 +448,27 @@ impl StateSync { } Ok(accounts_to_update) } + + /// Updates the `current_partial_mmr` and returns the authentication nodes for tracked leaves. + pub(crate) async fn update_partial_mmr( + &mut self, + mmr_delta: MmrDelta, + ) -> Result, ClientError> { + // First, apply curent_block to the Mmr + let new_authentication_nodes = self + .current_partial_mmr + .add(self.current_block.hash(), self.current_block_has_relevant_notes) + .into_iter(); + + // Apply the Mmr delta to bring Mmr to forest equal to chain tip + let new_authentication_nodes: Vec<(InOrderIndex, Digest)> = self + .current_partial_mmr + .apply(mmr_delta) + .map_err(StoreError::MmrError)? + .into_iter() + .chain(new_authentication_nodes) + .collect(); + + Ok(new_authentication_nodes) + } } diff --git a/crates/rust-client/src/tests.rs b/crates/rust-client/src/tests.rs index 172b5713e..6ac15aa26 100644 --- a/crates/rust-client/src/tests.rs +++ b/crates/rust-client/src/tests.rs @@ -369,7 +369,7 @@ async fn test_sync_state_mmr() { ); // Try reconstructing the chain_mmr from what's in the database - let partial_mmr = client.build_current_partial_mmr(true).await.unwrap(); + let partial_mmr = client.test_store().build_current_partial_mmr(true).await.unwrap(); assert_eq!(partial_mmr.forest(), 6); assert!(partial_mmr.open(0).unwrap().is_some()); // Account anchor block assert!(partial_mmr.open(1).unwrap().is_some()); diff --git a/crates/rust-client/src/transactions/mod.rs b/crates/rust-client/src/transactions/mod.rs index 477f984f8..f20a270f2 100644 --- a/crates/rust-client/src/transactions/mod.rs +++ b/crates/rust-client/src/transactions/mod.rs @@ -832,7 +832,7 @@ impl Client { let summary = self.sync_state().await?; if summary.block_num != block_num { - let mut current_partial_mmr = self.build_current_partial_mmr(true).await?; + let mut current_partial_mmr = self.store.build_current_partial_mmr(true).await?; self.get_and_store_authenticated_block(block_num, &mut current_partial_mmr) .await?; } From c1b8c4ff0f34603cf0d45f3be080b0b0e513b50a Mon Sep 17 00:00:00 2001 From: tomyrd Date: Thu, 2 Jan 2025 15:53:47 -0300 Subject: [PATCH 05/34] feat: update output notes on sync --- crates/rust-client/src/sync/mod.rs | 6 +- crates/rust-client/src/sync/state_sync.rs | 127 ++++++++++++++-------- 2 files changed, 84 insertions(+), 49 deletions(-) diff --git a/crates/rust-client/src/sync/mod.rs b/crates/rust-client/src/sync/mod.rs index acd8ffc7a..3c7a83d54 100644 --- a/crates/rust-client/src/sync/mod.rs +++ b/crates/rust-client/src/sync/mod.rs @@ -148,7 +148,8 @@ impl Client { let note_tags: Vec = self.store.get_unique_note_tags().await?.into_iter().collect(); - let unspent_notes = self.get_input_notes(NoteFilter::Unspent).await?; + let unspent_input_notes = self.get_input_notes(NoteFilter::Unspent).await?; + let unspent_output_notes = self.get_output_notes(NoteFilter::Unspent).await?; // Get the sync update from the network let status = StateSync::new( @@ -157,7 +158,8 @@ impl Client { has_relevant_notes, accounts, note_tags, - unspent_notes, + unspent_input_notes, + unspent_output_notes, self.store.build_current_partial_mmr(false).await?, ) .sync_state_step() diff --git a/crates/rust-client/src/sync/state_sync.rs b/crates/rust-client/src/sync/state_sync.rs index b3f99e23b..bfe97d01e 100644 --- a/crates/rust-client/src/sync/state_sync.rs +++ b/crates/rust-client/src/sync/state_sync.rs @@ -26,7 +26,7 @@ use crate::{ }, NodeRpcClient, RpcError, }, - store::{input_note_states::CommittedNoteState, InputNoteRecord, StoreError}, + store::{input_note_states::CommittedNoteState, InputNoteRecord, OutputNoteRecord, StoreError}, transactions::TransactionUpdates, ClientError, }; @@ -90,7 +90,8 @@ pub struct StateSync { current_block_has_relevant_notes: bool, accounts: Vec, note_tags: Vec, - unspent_notes: BTreeMap, + unspent_input_notes: BTreeMap, + unspent_output_notes: BTreeMap, changed_notes: BTreeSet, current_partial_mmr: PartialMmr, } @@ -102,10 +103,14 @@ impl StateSync { current_block_has_relevant_notes: bool, accounts: Vec, note_tags: Vec, - unspent_notes: Vec, + unspent_input_notes: Vec, + unspent_output_notes: Vec, current_partial_mmr: PartialMmr, ) -> Self { - let unspent_notes = unspent_notes.into_iter().map(|note| (note.id(), note)).collect(); + let unspent_input_notes = + unspent_input_notes.into_iter().map(|note| (note.id(), note)).collect(); + let unspent_output_notes = + unspent_output_notes.into_iter().map(|note| (note.id(), note)).collect(); Self { rpc_api, @@ -113,7 +118,8 @@ impl StateSync { current_block_has_relevant_notes, accounts, note_tags, - unspent_notes, + unspent_input_notes, + unspent_output_notes, changed_notes: BTreeSet::new(), current_partial_mmr, } @@ -128,7 +134,7 @@ impl StateSync { // (it only returns nullifiers from current_block_num until // response.block_header.block_num()) let nullifiers_tags: Vec = self - .unspent_notes + .unspent_input_notes .values() .map(|note| get_nullifier_prefix(¬e.nullifier())) .collect(); @@ -227,14 +233,20 @@ impl StateSync { let discarded_transactions = self.consumed_note_updates(nullifiers, &committed_transactions).await?; - let modified_notes: Vec = self + let modified_input_notes: Vec = self .changed_notes .iter() - .filter_map(|note_id| self.unspent_notes.remove(note_id)) + .filter_map(|note_id| self.unspent_input_notes.remove(note_id)) .collect(); - //TODO: Add output notes to update - let note_updates = NoteUpdates::new(new_notes, vec![], modified_notes, vec![]); + let modified_output_notes: Vec = self + .changed_notes + .iter() + .filter_map(|note_id| self.unspent_output_notes.remove(note_id)) + .collect(); + + let note_updates = + NoteUpdates::new(new_notes, vec![], modified_input_notes, modified_output_notes); let transaction_updates = TransactionUpdates::new(committed_transactions, discarded_transactions); @@ -260,9 +272,8 @@ impl StateSync { committed_note.merkle_path().clone(), )?; - if let Some(note_record) = self.unspent_notes.get_mut(committed_note.note_id()) { + if let Some(note_record) = self.unspent_input_notes.get_mut(committed_note.note_id()) { // The note belongs to our locally tracked set of input notes - let inclusion_proof_received = note_record .inclusion_proof_received(inclusion_proof.clone(), committed_note.metadata())?; let block_header_received = note_record.block_header_received(*block_header)?; @@ -270,8 +281,19 @@ impl StateSync { if inclusion_proof_received || block_header_received { self.changed_notes.insert(*committed_note.note_id()); } - } else { - // The note is public and we are tracking it, push to the list of IDs to query + } + + if let Some(note_record) = self.unspent_output_notes.get_mut(committed_note.note_id()) { + // The note belongs to our locally tracked set of output notes + if note_record.inclusion_proof_received(inclusion_proof.clone())? { + self.changed_notes.insert(*committed_note.note_id()); + } + } + + if !self.unspent_input_notes.contains_key(committed_note.note_id()) + && !self.unspent_output_notes.contains_key(committed_note.note_id()) + { + // The note totally new to the client new_public_notes.push(*committed_note.note_id()); } } @@ -292,12 +314,22 @@ impl StateSync { ) -> Result, ClientError> { let nullifier_filter: Vec = nullifiers.iter().map(|n| n.nullifier).collect(); - let mut consumed_note_ids: BTreeMap = self - .unspent_notes + let consumed_input_notes = self + .unspent_input_notes .values() .filter(|&n| nullifier_filter.contains(&n.nullifier())) - .map(|n| (n.nullifier(), n.id())) - .collect(); + .map(|n| (n.nullifier(), n.id())); + + let consumed_output_notes = self + .unspent_output_notes + .values() + .filter(|&n| n.nullifier().is_some_and(|n| nullifier_filter.contains(&n))) + .map(|n| { + (n.nullifier().expect("Output notes without nullifier were filtered"), n.id()) + }); + + let mut consumed_note_ids: BTreeMap = + consumed_input_notes.chain(consumed_output_notes).collect(); // Modify notes that were being processed by a transaciton that just got committed. These // notes were consumed internally. @@ -306,7 +338,7 @@ impl StateSync { let transaction_consumed_notes: Vec = consumed_note_ids .iter() .filter_map(|(_, note_id)| { - let note_record = self.unspent_notes.get(note_id)?; + let note_record = self.unspent_input_notes.get(note_id)?; if note_record.is_processing() && note_record.consumer_transaction_id() == Some(&transaction_update.transaction_id) @@ -320,19 +352,16 @@ impl StateSync { .collect(); for note_id in transaction_consumed_notes { - // SAFETY: The note IDs in `consumed_note_ids` were extracted from the - // `unspent_notes` map - let input_note_record = - self.unspent_notes.get_mut(¬e_id).expect("Note should exist"); - - if input_note_record.transaction_committed( - transaction_update.transaction_id, - transaction_update.block_num, - )? { - self.changed_notes.insert(note_id); - - // Remove the note from the list so it's not modified again in the next step - consumed_note_ids.remove(&input_note_record.nullifier()); + if let Some(input_note_record) = self.unspent_input_notes.get_mut(¬e_id) { + if input_note_record.transaction_committed( + transaction_update.transaction_id, + transaction_update.block_num, + )? { + self.changed_notes.insert(note_id); + + // Remove the note from the list so it's not modified again in the next step + consumed_note_ids.remove(&input_note_record.nullifier()); + } } } } @@ -345,23 +374,27 @@ impl StateSync { let block_num = nullifier_update.block_num; if let Some(note_id) = consumed_note_ids.remove(&nullifier) { - // SAFETY: The note IDs in `consumed_note_ids` were extracted from the - // `unspent_notes` map - let input_note_record = - self.unspent_notes.get_mut(¬e_id).expect("Note should exist"); - - if input_note_record.is_processing() { - // The note was being processed by a local transaction but it was nullified - // externally so the transaction should be discarded - discarded_transactions.push( - *input_note_record - .consumer_transaction_id() - .expect("Processing note should have consumer transaction id"), - ); + if let Some(input_note_record) = self.unspent_input_notes.get_mut(¬e_id) { + if input_note_record.is_processing() { + // The input note was being processed by a local transaction but it was + // nullified externally so the transaction should be + // discarded + discarded_transactions.push( + *input_note_record + .consumer_transaction_id() + .expect("Processing note should have consumer transaction id"), + ); + } + + if input_note_record.consumed_externally(nullifier, block_num)? { + self.changed_notes.insert(note_id); + } } - if input_note_record.consumed_externally(nullifier, block_num)? { - self.changed_notes.insert(note_id); + if let Some(output_note_record) = self.unspent_output_notes.get_mut(¬e_id) { + if output_note_record.nullifier_received(nullifier, block_num)? { + self.changed_notes.insert(note_id); + } } } } From 435682bd0f26a86fc94d4f6c0d1f29737cbc41ce Mon Sep 17 00:00:00 2001 From: tomyrd Date: Thu, 2 Jan 2025 17:20:36 -0300 Subject: [PATCH 06/34] feat: check for locked accounts in state sync --- crates/rust-client/src/accounts.rs | 14 +++--- .../src/store/sqlite_store/sync.rs | 27 +++++++++--- crates/rust-client/src/sync/state_sync.rs | 44 +++++++++++++------ 3 files changed, 60 insertions(+), 25 deletions(-) diff --git a/crates/rust-client/src/accounts.rs b/crates/rust-client/src/accounts.rs index b92d9ed30..9dc9ab7a7 100644 --- a/crates/rust-client/src/accounts.rs +++ b/crates/rust-client/src/accounts.rs @@ -166,19 +166,19 @@ impl Client { pub struct AccountUpdates { /// Updated public accounts. updated_public_accounts: Vec, - /// Updated network account hashes for private accounts. - private_account_hashes: Vec<(AccountId, Digest)>, + /// Node account hashes that don't match the tracked information. + mismatched_private_accounts: Vec<(AccountId, Digest)>, } impl AccountUpdates { /// Creates a new instance of `AccountUpdates`. pub fn new( updated_public_accounts: Vec, - private_account_hashes: Vec<(AccountId, Digest)>, + mismatched_private_accounts: Vec<(AccountId, Digest)>, ) -> Self { Self { updated_public_accounts, - private_account_hashes, + mismatched_private_accounts, } } @@ -187,9 +187,9 @@ impl AccountUpdates { &self.updated_public_accounts } - /// Returns updated network account hashes for private accounts. - pub fn private_account_hashes(&self) -> &[(AccountId, Digest)] { - &self.private_account_hashes + /// Returns mismatched private accounts. + pub fn mismatched_private_accounts(&self) -> &[(AccountId, Digest)] { + &self.mismatched_private_accounts } } diff --git a/crates/rust-client/src/store/sqlite_store/sync.rs b/crates/rust-client/src/store/sqlite_store/sync.rs index 541dff009..15aacb415 100644 --- a/crates/rust-client/src/store/sqlite_store/sync.rs +++ b/crates/rust-client/src/store/sqlite_store/sync.rs @@ -7,7 +7,10 @@ use rusqlite::{params, Connection, Transaction}; use super::SqliteStore; use crate::{ store::{ - sqlite_store::{accounts::update_account, notes::apply_note_updates_tx}, + sqlite_store::{ + accounts::{lock_account, update_account}, + notes::apply_note_updates_tx, + }, StoreError, }, sync::{NoteTagRecord, NoteTagSource, StateSyncUpdate}, @@ -103,6 +106,21 @@ impl SqliteStore { tags_to_remove, } = state_sync_update; + let mut locked_accounts = vec![]; + + for (account_id, digest) in account_updates.mismatched_private_accounts() { + // Mismatched digests may be due to stale network data. If the mismatched digest is + // tracked in the db and corresponds to the mismatched account, it means we + // got a past update and shouldn't lock the account. + if let Some(account) = Self::get_account_header_by_hash(conn, *digest)? { + if account.id() == *account_id { + continue; + } + } + + locked_accounts.push(*account_id); + } + let tx = conn.transaction()?; // Update state sync block number @@ -133,10 +151,9 @@ impl SqliteStore { update_account(&tx, account)?; } - // TODO: Lock offchain accounts that have been updated onchain - // for (account_id, _) in updated_accounts.mismatched_offchain_accounts() { - // lock_account(&tx, *account_id)?; - // } + for account_id in locked_accounts { + lock_account(&tx, account_id)?; + } // Commit the updates tx.commit()?; diff --git a/crates/rust-client/src/sync/state_sync.rs b/crates/rust-client/src/sync/state_sync.rs index bfe97d01e..b888fa113 100644 --- a/crates/rust-client/src/sync/state_sync.rs +++ b/crates/rust-client/src/sync/state_sync.rs @@ -3,6 +3,7 @@ use alloc::{ sync::Arc, vec::Vec, }; +use std::println; use miden_objects::{ accounts::{Account, AccountHeader, AccountId}, @@ -64,8 +65,18 @@ impl From<&StateSyncUpdate> for SyncSummary { .iter() .map(|acc| acc.id()) .collect(), - vec![], // TODO add these fields - vec![], + value + .account_updates + .mismatched_private_accounts() + .iter() + .map(|(id, _)| *id) + .collect(), + value + .transaction_updates + .committed_transactions() + .iter() + .map(|t| t.transaction_id) + .collect(), ) } } @@ -133,11 +144,19 @@ impl StateSync { // Note that besides filtering by nullifier prefixes, the node also filters by block number // (it only returns nullifiers from current_block_num until // response.block_header.block_num()) - let nullifiers_tags: Vec = self + let input_note_nullifiers = self .unspent_input_notes .values() - .map(|note| get_nullifier_prefix(¬e.nullifier())) - .collect(); + .map(|note| get_nullifier_prefix(¬e.nullifier())); + + let output_note_nullifiers = self + .unspent_output_notes + .values() + .filter_map(|note| note.nullifier()) + .map(|nullifier| get_nullifier_prefix(&nullifier)); + + let nullifiers_tags: Vec = + input_note_nullifiers.chain(output_note_nullifiers).collect(); let response = self .rpc_api @@ -209,16 +228,17 @@ impl StateSync { .get_updated_onchain_accounts(account_hash_updates, &onchain_accounts) .await?; - let private_account_hashes = account_hash_updates + let mismatched_private_accounts = account_hash_updates .iter() - .filter_map(|(account_id, _)| { + .filter(|(account_id, digest)| { offchain_accounts .iter() - .find(|account| account.id() == *account_id) - .map(|account| (account.id(), account.hash())) + .any(|account| account.id() == *account_id && &account.hash() != digest) }) + .cloned() .collect::>(); - Ok(AccountUpdates::new(updated_onchain_accounts, private_account_hashes)) + + Ok(AccountUpdates::new(updated_onchain_accounts, mismatched_private_accounts)) } async fn note_state_sync( @@ -358,9 +378,6 @@ impl StateSync { transaction_update.block_num, )? { self.changed_notes.insert(note_id); - - // Remove the note from the list so it's not modified again in the next step - consumed_note_ids.remove(&input_note_record.nullifier()); } } } @@ -392,6 +409,7 @@ impl StateSync { } if let Some(output_note_record) = self.unspent_output_notes.get_mut(¬e_id) { + println!("output note consumed externally"); if output_note_record.nullifier_received(nullifier, block_num)? { self.changed_notes.insert(note_id); } From b61262ddaee4ca586c83c90b0a0257ae8ff1c734 Mon Sep 17 00:00:00 2001 From: tomyrd Date: Thu, 2 Jan 2025 18:44:25 -0300 Subject: [PATCH 07/34] doc: improve documentation --- crates/rust-client/src/notes/mod.rs | 21 ++--- .../src/store/sqlite_store/notes.rs | 15 ++-- crates/rust-client/src/sync/state_sync.rs | 83 ++++++++++++++----- 3 files changed, 71 insertions(+), 48 deletions(-) diff --git a/crates/rust-client/src/notes/mod.rs b/crates/rust-client/src/notes/mod.rs index 77216150e..e31dded68 100644 --- a/crates/rust-client/src/notes/mod.rs +++ b/crates/rust-client/src/notes/mod.rs @@ -4,7 +4,7 @@ use alloc::{collections::BTreeSet, string::ToString, vec::Vec}; use miden_lib::transaction::TransactionKernel; -use miden_objects::{accounts::AccountId, crypto::rand::FeltRng}; +use miden_objects::{accounts::AccountId, crypto::rand::FeltRng, transaction::InputNote}; use crate::{ store::{InputNoteRecord, NoteFilter, OutputNoteRecord}, @@ -188,10 +188,8 @@ pub async fn get_input_note_with_id_prefix( /// Contains note changes to apply to the store. pub struct NoteUpdates { - /// A list of new input notes. - new_input_notes: Vec, - /// A list of new output notes. - new_output_notes: Vec, + /// A list of new input notes to be tracked. + new_input_notes: Vec, /// A list of updated input note records corresponding to locally-tracked input notes. updated_input_notes: Vec, /// A list of updated output note records corresponding to locally-tracked output notes. @@ -201,14 +199,12 @@ pub struct NoteUpdates { impl NoteUpdates { /// Creates a [NoteUpdates]. pub fn new( - new_input_notes: Vec, - new_output_notes: Vec, + new_input_notes: Vec, updated_input_notes: Vec, updated_output_notes: Vec, ) -> Self { Self { new_input_notes, - new_output_notes, updated_input_notes, updated_output_notes, } @@ -217,7 +213,6 @@ impl NoteUpdates { /// Combines two [NoteUpdates] into a single one. pub fn combine_with(mut self, other: Self) -> Self { self.new_input_notes.extend(other.new_input_notes); - self.new_output_notes.extend(other.new_output_notes); self.updated_input_notes.extend(other.updated_input_notes); self.updated_output_notes.extend(other.updated_output_notes); @@ -225,15 +220,10 @@ impl NoteUpdates { } /// Returns all new input note records, meant to be tracked by the client. - pub fn new_input_notes(&self) -> &[InputNoteRecord] { + pub fn new_input_notes(&self) -> &[InputNote] { &self.new_input_notes } - /// Returns all new output note records, meant to be tracked by the client. - pub fn new_output_notes(&self) -> &[OutputNoteRecord] { - &self.new_output_notes - } - /// Returns all updated input note records. That is, any input notes that are locally tracked /// and have been updated. pub fn updated_input_notes(&self) -> &[InputNoteRecord] { @@ -251,7 +241,6 @@ impl NoteUpdates { self.updated_input_notes.is_empty() && self.updated_output_notes.is_empty() && self.new_input_notes.is_empty() - && self.new_output_notes.is_empty() } /// Returns the IDs of all notes that have been committed. diff --git a/crates/rust-client/src/store/sqlite_store/notes.rs b/crates/rust-client/src/store/sqlite_store/notes.rs index fe8a59e91..a6c57f63b 100644 --- a/crates/rust-client/src/store/sqlite_store/notes.rs +++ b/crates/rust-client/src/store/sqlite_store/notes.rs @@ -16,6 +16,7 @@ use rusqlite::{named_params, params, params_from_iter, types::Value, Connection, use super::SqliteStore; use crate::{ notes::NoteUpdates, + rpc::generated::note, store::{ note_record::OutputNoteState, InputNoteRecord, InputNoteState, NoteFilter, OutputNoteRecord, StoreError, @@ -598,18 +599,12 @@ pub(crate) fn apply_note_updates_tx( tx: &Transaction, note_updates: &NoteUpdates, ) -> Result<(), StoreError> { - for input_note in - note_updates.new_input_notes().iter().chain(note_updates.updated_input_notes()) - { - upsert_input_note_tx(tx, input_note)?; + for input_note in note_updates.new_input_notes().iter() { + upsert_input_note_tx(tx, &input_note.clone().into())?; } - for output_note in note_updates - .new_output_notes() - .iter() - .chain(note_updates.updated_output_notes()) - { - upsert_output_note_tx(tx, output_note)?; + for input_note_record in note_updates.updated_input_notes().iter() { + upsert_input_note_tx(tx, input_note_record)?; } Ok(()) diff --git a/crates/rust-client/src/sync/state_sync.rs b/crates/rust-client/src/sync/state_sync.rs index b888fa113..df7290188 100644 --- a/crates/rust-client/src/sync/state_sync.rs +++ b/crates/rust-client/src/sync/state_sync.rs @@ -3,13 +3,12 @@ use alloc::{ sync::Arc, vec::Vec, }; -use std::println; use miden_objects::{ accounts::{Account, AccountHeader, AccountId}, crypto::merkle::{InOrderIndex, MmrDelta, MmrPeaks, PartialMmr}, notes::{NoteId, NoteInclusionProof, NoteTag, Nullifier}, - transaction::TransactionId, + transaction::{InputNote, TransactionId}, BlockHeader, Digest, }; use tracing::info; @@ -27,7 +26,7 @@ use crate::{ }, NodeRpcClient, RpcError, }, - store::{input_note_states::CommittedNoteState, InputNoteRecord, OutputNoteRecord, StoreError}, + store::{InputNoteRecord, OutputNoteRecord, StoreError}, transactions::TransactionUpdates, ClientError, }; @@ -81,6 +80,7 @@ impl From<&StateSyncUpdate> for SyncSummary { } } +/// Gives information about the status of the sync process after a step. pub enum SyncStatus { SyncedToLastBlock(StateSyncUpdate), SyncedToBlock(StateSyncUpdate), @@ -95,6 +95,11 @@ impl SyncStatus { } } +/// The state sync components encompasses the client's sync logic. +/// +/// When created it receives the current state of the client's relevant elements (block, accounts, +/// notes, etc). It is then used to requset updates from the node and apply them to the relevant +/// elements. The updates are then returned and can be applied to the store to persist the changes. pub struct StateSync { rpc_api: Arc, current_block: BlockHeader, @@ -108,6 +113,21 @@ pub struct StateSync { } impl StateSync { + /// Creates a new instance of the state sync component. + /// + /// # Arguments + /// + /// * `rpc_api` - The RPC client to use to communicate with the node. + /// * `current_block` - The latest block header tracked by the client. + /// * `current_block_has_relevant_notes` - A flag indicating if the current block has notes that + /// are relevant to the client. + /// * `accounts` - The headers of accounts tracked by the client. + /// * `note_tags` - The note tags to be used in the sync state request. + /// * `unspent_input_notes` - The input notes that haven't been yet consumed and may be changed + /// in the sync process. + /// * `unspent_output_notes` - The output notes that haven't been yet consumed and may be + /// changed in the sync process. + /// * `current_partial_mmr` - The current partial MMR of the client. pub fn new( rpc_api: Arc, current_block: BlockHeader, @@ -136,6 +156,15 @@ impl StateSync { } } + /// Executes a single step of the state sync process, returning the changes that should be + /// applied to the store. + /// + /// A step in this context means a single request to the node to get the next relevant block and + /// the changes that happened in it. This block may not be the last one in the chain and + /// the client may need to call this method multiple times until it reaches the chain tip. + /// Wheter or not the client has reached the chain tip is indicated by the returned + /// [SyncStatus] variant. `None` is returned if the client is already synced with the chain tip + /// and there are no new changes. pub async fn sync_state_step(mut self) -> Result, ClientError> { let current_block_num = self.current_block.block_num(); let account_ids: Vec = self.accounts.iter().map(|acc| acc.id()).collect(); @@ -203,8 +232,7 @@ impl StateSync { new_mmr_peaks: self.current_partial_mmr.peaks(), new_authentication_nodes, account_updates, - tags_to_remove, /* TODO: I think this can be removed from the update and be inferred - * from the note updates */ + tags_to_remove, }; if response.chain_tip == response.block_header.block_num() { @@ -217,6 +245,14 @@ impl StateSync { // HELPERS // -------------------------------------------------------------------------------------------- + /// Compares the state of tracked accounts with the updates received from the node and returns + /// the accounts that need to be updated. + /// + /// When a mismatch is detected, two scenarios are possible: + /// * If the account is public, the component will request the node for the updated account + /// details. + /// * If the account is private it will be marked as mismatched and the client will need to + /// handle it (it could be a stale account state or a reason to lock the account). async fn account_state_sync( &self, account_hash_updates: &[(AccountId, Digest)], @@ -241,6 +277,21 @@ impl StateSync { Ok(AccountUpdates::new(updated_onchain_accounts, mismatched_private_accounts)) } + /// Compares the state of tracked notes with the updates received from the node and returns the + /// note and transaction changes that should be applied to the store. + /// + /// The note changes might include: + /// * New notes that we received from the node and might be relevant to the client. + /// * Tracked expected notes that were committed in the block. + /// * Tracked notes that were being processed by a transaction that got committed. + /// * Tracked notes that were nullified by an external transaction. + /// + /// The transaction changes might include: + /// * Transactions that were committed in the block. Some of these might me tracked by the + /// client + /// and need to be marked as committed. + /// * Local tracked transactions that were discarded because the notes that they were processing + /// were nullified by an another transaction. async fn note_state_sync( &mut self, block_header: &BlockHeader, @@ -265,8 +316,7 @@ impl StateSync { .filter_map(|note_id| self.unspent_output_notes.remove(note_id)) .collect(); - let note_updates = - NoteUpdates::new(new_notes, vec![], modified_input_notes, modified_output_notes); + let note_updates = NoteUpdates::new(new_notes, modified_input_notes, modified_output_notes); let transaction_updates = TransactionUpdates::new(committed_transactions, discarded_transactions); @@ -279,7 +329,7 @@ impl StateSync { &mut self, committed_notes: Vec, block_header: &BlockHeader, - ) -> Result, ClientError> { + ) -> Result, ClientError> { let mut new_public_notes = vec![]; // We'll only pick committed notes that we are tracking as input/output notes. Since the @@ -409,7 +459,6 @@ impl StateSync { } if let Some(output_note_record) = self.unspent_output_notes.get_mut(¬e_id) { - println!("output note consumed externally"); if output_note_record.nullifier_received(nullifier, block_num)? { self.changed_notes.insert(note_id); } @@ -428,7 +477,7 @@ impl StateSync { &self, query_notes: &[NoteId], block_header: &BlockHeader, - ) -> Result, ClientError> { + ) -> Result, ClientError> { if query_notes.is_empty() { return Ok(vec![]); } @@ -451,18 +500,8 @@ impl StateSync { inclusion_proof.merkle_path, ) .map_err(ClientError::NoteError)?; - let metadata = *note.metadata(); - - return_notes.push(InputNoteRecord::new( - note.into(), - None, // TODO: Add timestamp - CommittedNoteState { - metadata, - inclusion_proof, - block_note_root: block_header.note_root(), - } - .into(), - )) + + return_notes.push(InputNote::authenticated(note, inclusion_proof)) }, } } From d54e186dcf819a5dd2cc6cece83b183ed3a414be Mon Sep 17 00:00:00 2001 From: tomyrd Date: Thu, 2 Jan 2025 20:02:41 -0300 Subject: [PATCH 08/34] refactor: revert unnecessary changes --- crates/rust-client/src/notes/import.rs | 4 +- crates/rust-client/src/notes/mod.rs | 21 ++++++--- crates/rust-client/src/store/mod.rs | 40 +---------------- .../src/store/sqlite_store/notes.rs | 15 ++++--- crates/rust-client/src/sync/block_headers.rs | 44 +++++++++++++++++-- crates/rust-client/src/sync/mod.rs | 2 +- crates/rust-client/src/sync/state_sync.rs | 20 +++++---- crates/rust-client/src/tests.rs | 2 +- crates/rust-client/src/transactions/mod.rs | 2 +- 9 files changed, 85 insertions(+), 65 deletions(-) diff --git a/crates/rust-client/src/notes/import.rs b/crates/rust-client/src/notes/import.rs index 58694d144..b1d2f35ea 100644 --- a/crates/rust-client/src/notes/import.rs +++ b/crates/rust-client/src/notes/import.rs @@ -170,7 +170,7 @@ impl Client { note_record.inclusion_proof_received(inclusion_proof, metadata)?; if block_height < current_block_num { - let mut current_partial_mmr = self.store.build_current_partial_mmr(true).await?; + let mut current_partial_mmr = self.build_current_partial_mmr(true).await?; let block_header = self .get_and_store_authenticated_block(block_height, &mut current_partial_mmr) @@ -214,7 +214,7 @@ impl Client { match committed_note_data { Some((metadata, inclusion_proof)) => { - let mut current_partial_mmr = self.store.build_current_partial_mmr(true).await?; + let mut current_partial_mmr = self.build_current_partial_mmr(true).await?; let block_header = self .get_and_store_authenticated_block( inclusion_proof.location().block_num(), diff --git a/crates/rust-client/src/notes/mod.rs b/crates/rust-client/src/notes/mod.rs index e31dded68..77216150e 100644 --- a/crates/rust-client/src/notes/mod.rs +++ b/crates/rust-client/src/notes/mod.rs @@ -4,7 +4,7 @@ use alloc::{collections::BTreeSet, string::ToString, vec::Vec}; use miden_lib::transaction::TransactionKernel; -use miden_objects::{accounts::AccountId, crypto::rand::FeltRng, transaction::InputNote}; +use miden_objects::{accounts::AccountId, crypto::rand::FeltRng}; use crate::{ store::{InputNoteRecord, NoteFilter, OutputNoteRecord}, @@ -188,8 +188,10 @@ pub async fn get_input_note_with_id_prefix( /// Contains note changes to apply to the store. pub struct NoteUpdates { - /// A list of new input notes to be tracked. - new_input_notes: Vec, + /// A list of new input notes. + new_input_notes: Vec, + /// A list of new output notes. + new_output_notes: Vec, /// A list of updated input note records corresponding to locally-tracked input notes. updated_input_notes: Vec, /// A list of updated output note records corresponding to locally-tracked output notes. @@ -199,12 +201,14 @@ pub struct NoteUpdates { impl NoteUpdates { /// Creates a [NoteUpdates]. pub fn new( - new_input_notes: Vec, + new_input_notes: Vec, + new_output_notes: Vec, updated_input_notes: Vec, updated_output_notes: Vec, ) -> Self { Self { new_input_notes, + new_output_notes, updated_input_notes, updated_output_notes, } @@ -213,6 +217,7 @@ impl NoteUpdates { /// Combines two [NoteUpdates] into a single one. pub fn combine_with(mut self, other: Self) -> Self { self.new_input_notes.extend(other.new_input_notes); + self.new_output_notes.extend(other.new_output_notes); self.updated_input_notes.extend(other.updated_input_notes); self.updated_output_notes.extend(other.updated_output_notes); @@ -220,10 +225,15 @@ impl NoteUpdates { } /// Returns all new input note records, meant to be tracked by the client. - pub fn new_input_notes(&self) -> &[InputNote] { + pub fn new_input_notes(&self) -> &[InputNoteRecord] { &self.new_input_notes } + /// Returns all new output note records, meant to be tracked by the client. + pub fn new_output_notes(&self) -> &[OutputNoteRecord] { + &self.new_output_notes + } + /// Returns all updated input note records. That is, any input notes that are locally tracked /// and have been updated. pub fn updated_input_notes(&self) -> &[InputNoteRecord] { @@ -241,6 +251,7 @@ impl NoteUpdates { self.updated_input_notes.is_empty() && self.updated_output_notes.is_empty() && self.new_input_notes.is_empty() + && self.new_output_notes.is_empty() } /// Returns the IDs of all notes that have been committed. diff --git a/crates/rust-client/src/store/mod.rs b/crates/rust-client/src/store/mod.rs index d2a1526eb..b4733ac07 100644 --- a/crates/rust-client/src/store/mod.rs +++ b/crates/rust-client/src/store/mod.rs @@ -10,7 +10,7 @@ use core::fmt::Debug; use async_trait::async_trait; use miden_objects::{ accounts::{Account, AccountCode, AccountHeader, AccountId, AuthSecretKey}, - crypto::merkle::{InOrderIndex, MmrPeaks, PartialMmr}, + crypto::merkle::{InOrderIndex, MmrPeaks}, notes::{NoteId, NoteTag, Nullifier}, BlockHeader, Digest, Word, }; @@ -208,44 +208,6 @@ pub trait Store: Send + Sync { has_client_notes: bool, ) -> Result<(), StoreError>; - /// Builds the current store view of the chain's [PartialMmr]. Because we want to add all new - /// authentication nodes that could come from applying the MMR updates, we need to track all - /// known leaves thus far. - /// - /// As part of the syncing process, we add the current block number so we don't need to - /// track it here. - async fn build_current_partial_mmr( - &self, - include_current_block: bool, - ) -> Result { - let current_block_num = self.get_sync_height().await?; - - let tracked_nodes = self.get_chain_mmr_nodes(ChainMmrNodeFilter::All).await?; - let current_peaks = self.get_chain_mmr_peaks_by_block_num(current_block_num).await?; - - let track_latest = if current_block_num != 0 { - match self.get_block_header_by_num(current_block_num - 1).await { - Ok((_, previous_block_had_notes)) => Ok(previous_block_had_notes), - Err(StoreError::BlockHeaderNotFound(_)) => Ok(false), - Err(err) => Err(err), - }? - } else { - false - }; - - let mut current_partial_mmr = - PartialMmr::from_parts(current_peaks, tracked_nodes, track_latest); - - if include_current_block { - let (current_block, has_client_notes) = - self.get_block_header_by_num(current_block_num).await?; - - current_partial_mmr.add(current_block.hash(), has_client_notes); - } - - Ok(current_partial_mmr) - } - // ACCOUNT // -------------------------------------------------------------------------------------------- diff --git a/crates/rust-client/src/store/sqlite_store/notes.rs b/crates/rust-client/src/store/sqlite_store/notes.rs index a6c57f63b..fe8a59e91 100644 --- a/crates/rust-client/src/store/sqlite_store/notes.rs +++ b/crates/rust-client/src/store/sqlite_store/notes.rs @@ -16,7 +16,6 @@ use rusqlite::{named_params, params, params_from_iter, types::Value, Connection, use super::SqliteStore; use crate::{ notes::NoteUpdates, - rpc::generated::note, store::{ note_record::OutputNoteState, InputNoteRecord, InputNoteState, NoteFilter, OutputNoteRecord, StoreError, @@ -599,12 +598,18 @@ pub(crate) fn apply_note_updates_tx( tx: &Transaction, note_updates: &NoteUpdates, ) -> Result<(), StoreError> { - for input_note in note_updates.new_input_notes().iter() { - upsert_input_note_tx(tx, &input_note.clone().into())?; + for input_note in + note_updates.new_input_notes().iter().chain(note_updates.updated_input_notes()) + { + upsert_input_note_tx(tx, input_note)?; } - for input_note_record in note_updates.updated_input_notes().iter() { - upsert_input_note_tx(tx, input_note_record)?; + for output_note in note_updates + .new_output_notes() + .iter() + .chain(note_updates.updated_output_notes()) + { + upsert_output_note_tx(tx, output_note)?; } Ok(()) diff --git a/crates/rust-client/src/sync/block_headers.rs b/crates/rust-client/src/sync/block_headers.rs index 9ceb38ef6..6e3dcfa14 100644 --- a/crates/rust-client/src/sync/block_headers.rs +++ b/crates/rust-client/src/sync/block_headers.rs @@ -11,7 +11,7 @@ use tracing::warn; use super::NoteUpdates; use crate::{ notes::NoteScreener, - store::{NoteFilter, StoreError}, + store::{ChainMmrNodeFilter, NoteFilter, StoreError}, Client, ClientError, }; @@ -20,7 +20,7 @@ impl Client { /// Updates committed notes with no MMR data. These could be notes that were /// imported with an inclusion proof, but its block header isn't tracked. pub(crate) async fn update_mmr_data(&mut self) -> Result<(), ClientError> { - let mut current_partial_mmr = self.store.build_current_partial_mmr(true).await?; + let mut current_partial_mmr = self.build_current_partial_mmr(true).await?; let mut changed_notes = vec![]; for mut note in self.store.get_input_notes(NoteFilter::Unverified).await? { @@ -101,6 +101,44 @@ impl Client { Ok(false) } + /// Builds the current store view of the chain's [PartialMmr]. Because we want to add all new + /// authentication nodes that could come from applying the MMR updates, we need to track all + /// known leaves thus far. + /// + /// As part of the syncing process, we add the current block number so we don't need to + /// track it here. + pub(crate) async fn build_current_partial_mmr( + &self, + include_current_block: bool, + ) -> Result { + let current_block_num = self.store.get_sync_height().await?; + + let tracked_nodes = self.store.get_chain_mmr_nodes(ChainMmrNodeFilter::All).await?; + let current_peaks = self.store.get_chain_mmr_peaks_by_block_num(current_block_num).await?; + + let track_latest = if current_block_num != 0 { + match self.store.get_block_header_by_num(current_block_num - 1).await { + Ok((_, previous_block_had_notes)) => Ok(previous_block_had_notes), + Err(StoreError::BlockHeaderNotFound(_)) => Ok(false), + Err(err) => Err(ClientError::StoreError(err)), + }? + } else { + false + }; + + let mut current_partial_mmr = + PartialMmr::from_parts(current_peaks, tracked_nodes, track_latest); + + if include_current_block { + let (current_block, has_client_notes) = + self.store.get_block_header_by_num(current_block_num).await?; + + current_partial_mmr.add(current_block.hash(), has_client_notes); + } + + Ok(current_partial_mmr) + } + /// Retrieves and stores a [BlockHeader] by number, and stores its authentication data as well. /// /// If the store already contains MMR data for the requested block number, the request isn't @@ -158,7 +196,7 @@ impl Client { return self.ensure_genesis_in_place().await; } - let mut current_partial_mmr = self.store.build_current_partial_mmr(true).await?; + let mut current_partial_mmr = self.build_current_partial_mmr(true).await?; let anchor_block = self .get_and_store_authenticated_block(epoch_block_number, &mut current_partial_mmr) .await?; diff --git a/crates/rust-client/src/sync/mod.rs b/crates/rust-client/src/sync/mod.rs index 3c7a83d54..5f5188e93 100644 --- a/crates/rust-client/src/sync/mod.rs +++ b/crates/rust-client/src/sync/mod.rs @@ -160,7 +160,7 @@ impl Client { note_tags, unspent_input_notes, unspent_output_notes, - self.store.build_current_partial_mmr(false).await?, + self.build_current_partial_mmr(false).await?, ) .sync_state_step() .await?; diff --git a/crates/rust-client/src/sync/state_sync.rs b/crates/rust-client/src/sync/state_sync.rs index df7290188..671f37d25 100644 --- a/crates/rust-client/src/sync/state_sync.rs +++ b/crates/rust-client/src/sync/state_sync.rs @@ -252,7 +252,7 @@ impl StateSync { /// * If the account is public, the component will request the node for the updated account /// details. /// * If the account is private it will be marked as mismatched and the client will need to - /// handle it (it could be a stale account state or a reason to lock the account). + /// handle it (it could be a stale account state or a reason to lock the account). async fn account_state_sync( &self, account_hash_updates: &[(AccountId, Digest)], @@ -288,10 +288,9 @@ impl StateSync { /// /// The transaction changes might include: /// * Transactions that were committed in the block. Some of these might me tracked by the - /// client - /// and need to be marked as committed. + /// client and need to be marked as committed. /// * Local tracked transactions that were discarded because the notes that they were processing - /// were nullified by an another transaction. + /// were nullified by an another transaction. async fn note_state_sync( &mut self, block_header: &BlockHeader, @@ -316,7 +315,8 @@ impl StateSync { .filter_map(|note_id| self.unspent_output_notes.remove(note_id)) .collect(); - let note_updates = NoteUpdates::new(new_notes, modified_input_notes, modified_output_notes); + let note_updates = + NoteUpdates::new(new_notes, vec![], modified_input_notes, modified_output_notes); let transaction_updates = TransactionUpdates::new(committed_transactions, discarded_transactions); @@ -329,7 +329,7 @@ impl StateSync { &mut self, committed_notes: Vec, block_header: &BlockHeader, - ) -> Result, ClientError> { + ) -> Result, ClientError> { let mut new_public_notes = vec![]; // We'll only pick committed notes that we are tracking as input/output notes. Since the @@ -369,8 +369,12 @@ impl StateSync { } // Query the node for input note data and build the entities - let new_public_notes = - self.fetch_public_note_details(&new_public_notes, block_header).await?; + let new_public_notes = self + .fetch_public_note_details(&new_public_notes, block_header) + .await? + .into_iter() + .map(|note| note.into()) + .collect(); Ok(new_public_notes) } diff --git a/crates/rust-client/src/tests.rs b/crates/rust-client/src/tests.rs index 6ac15aa26..172b5713e 100644 --- a/crates/rust-client/src/tests.rs +++ b/crates/rust-client/src/tests.rs @@ -369,7 +369,7 @@ async fn test_sync_state_mmr() { ); // Try reconstructing the chain_mmr from what's in the database - let partial_mmr = client.test_store().build_current_partial_mmr(true).await.unwrap(); + let partial_mmr = client.build_current_partial_mmr(true).await.unwrap(); assert_eq!(partial_mmr.forest(), 6); assert!(partial_mmr.open(0).unwrap().is_some()); // Account anchor block assert!(partial_mmr.open(1).unwrap().is_some()); diff --git a/crates/rust-client/src/transactions/mod.rs b/crates/rust-client/src/transactions/mod.rs index f20a270f2..477f984f8 100644 --- a/crates/rust-client/src/transactions/mod.rs +++ b/crates/rust-client/src/transactions/mod.rs @@ -832,7 +832,7 @@ impl Client { let summary = self.sync_state().await?; if summary.block_num != block_num { - let mut current_partial_mmr = self.store.build_current_partial_mmr(true).await?; + let mut current_partial_mmr = self.build_current_partial_mmr(true).await?; self.get_and_store_authenticated_block(block_num, &mut current_partial_mmr) .await?; } From a499748709ea37e728d75a55f2d673e1d4864bc9 Mon Sep 17 00:00:00 2001 From: tomyrd Date: Tue, 7 Jan 2025 19:43:16 -0300 Subject: [PATCH 09/34] refactor: move state transitions outside StateSync --- crates/rust-client/src/store/mod.rs | 20 + crates/rust-client/src/sync/block_headers.rs | 32 +- crates/rust-client/src/sync/mod.rs | 340 +++++++++++++- crates/rust-client/src/sync/state_sync.rs | 466 ++++--------------- 4 files changed, 464 insertions(+), 394 deletions(-) diff --git a/crates/rust-client/src/store/mod.rs b/crates/rust-client/src/store/mod.rs index b4733ac07..cdcc84265 100644 --- a/crates/rust-client/src/store/mod.rs +++ b/crates/rust-client/src/store/mod.rs @@ -132,6 +132,26 @@ pub trait Store: Send + Sync { nullifiers } + /// Returns the note IDs of all expected notes. + /// + /// The default implementation of this method uses [Store::get_input_notes] and + /// [Store::get_output_notes]. + async fn get_expected_note_ids(&self) -> Result, StoreError> { + let input_notes = self + .get_input_notes(NoteFilter::Expected) + .await? + .into_iter() + .map(|input_note| input_note.id()); + + let output_notes = self + .get_output_notes(NoteFilter::Expected) + .await? + .into_iter() + .map(|output_note| output_note.id()); + + Ok(input_notes.chain(output_notes).collect()) + } + /// Inserts the provided input notes into the database. If a note with the same ID already /// exists, it will be replaced. async fn upsert_input_notes(&self, notes: &[InputNoteRecord]) -> Result<(), StoreError>; diff --git a/crates/rust-client/src/sync/block_headers.rs b/crates/rust-client/src/sync/block_headers.rs index a2ed8c119..63309fece 100644 --- a/crates/rust-client/src/sync/block_headers.rs +++ b/crates/rust-client/src/sync/block_headers.rs @@ -3,7 +3,11 @@ use alloc::vec::Vec; use crypto::merkle::{InOrderIndex, MmrPeaks, PartialMmr}; use miden_objects::{ block::{block_epoch_from_number, block_num_from_epoch}, - crypto::{self, merkle::MerklePath, rand::FeltRng}, + crypto::{ + self, + merkle::{MerklePath, MmrDelta}, + rand::FeltRng, + }, BlockHeader, Digest, }; use tracing::warn; @@ -208,6 +212,32 @@ impl Client { let current_block_num = self.store.get_sync_height().await?; self.get_epoch_block(current_block_num).await } + + /// Applies changes to the current MMR structure, returns the updated [PartialMmr] and the + /// authentication nodes for leaves we track. + pub(crate) async fn apply_mmr_changes( + &self, + mmr_delta: MmrDelta, + ) -> Result<(MmrPeaks, Vec<(InOrderIndex, Digest)>), ClientError> { + let mut partial_mmr = self.build_current_partial_mmr(false).await?; + let (current_block_header, current_block_has_relevant_notes) = + self.store.get_block_header_by_num(self.get_sync_height().await?).await?; + + // First, apply curent_block to the Mmr + let new_authentication_nodes = partial_mmr + .add(current_block_header.hash(), current_block_has_relevant_notes) + .into_iter(); + + // Apply the Mmr delta to bring Mmr to forest equal to chain tip + let new_authentication_nodes: Vec<(InOrderIndex, Digest)> = partial_mmr + .apply(mmr_delta) + .map_err(StoreError::MmrError)? + .into_iter() + .chain(new_authentication_nodes) + .collect(); + + Ok((partial_mmr.peaks(), new_authentication_nodes)) + } } // UTILS diff --git a/crates/rust-client/src/sync/mod.rs b/crates/rust-client/src/sync/mod.rs index a0e7d1098..0e2c75bf7 100644 --- a/crates/rust-client/src/sync/mod.rs +++ b/crates/rust-client/src/sync/mod.rs @@ -3,17 +3,28 @@ use alloc::vec::Vec; use core::cmp::max; +use std::{boxed::Box, collections::BTreeMap}; use miden_objects::{ accounts::{AccountHeader, AccountId}, - crypto::rand::FeltRng, - notes::{NoteId, NoteTag, Nullifier}, + crypto::{ + merkle::{InOrderIndex, MmrPeaks}, + rand::FeltRng, + }, + notes::{NoteId, NoteInclusionProof, Nullifier}, transaction::TransactionId, + BlockHeader, Digest, }; +use state_sync::fetch_public_note_details; use crate::{ + accounts::AccountUpdates, notes::NoteUpdates, - store::NoteFilter, + rpc::domain::{ + notes::CommittedNote, nullifiers::NullifierUpdate, transactions::TransactionUpdate, + }, + store::{InputNoteRecord, NoteFilter, OutputNoteRecord}, + transactions::TransactionUpdates, Client, ClientError, }; @@ -23,7 +34,56 @@ mod tags; pub use tags::{NoteTagRecord, NoteTagSource}; mod state_sync; -pub use state_sync::{StateSync, StateSyncUpdate, SyncStatus}; +pub use state_sync::{StateSync, SyncStatus}; + +/// Contains all information needed to apply the update in the store after syncing with the node. +pub struct StateSyncUpdate { + /// The new block header, returned as part of the + /// [StateSyncInfo](crate::rpc::domain::sync::StateSyncInfo) + pub block_header: BlockHeader, + /// Information about note changes after the sync. + pub note_updates: NoteUpdates, + /// Information about transaction changes after the sync. + pub transaction_updates: TransactionUpdates, + /// New MMR peaks for the locally tracked MMR of the blockchain. + pub new_mmr_peaks: MmrPeaks, + /// New authentications nodes that are meant to be stored in order to authenticate block + /// headers. + pub new_authentication_nodes: Vec<(InOrderIndex, Digest)>, + /// Information abount account changes after the sync. + pub account_updates: AccountUpdates, + /// Tag records that are no longer relevant. + pub tags_to_remove: Vec, +} + +impl From<&StateSyncUpdate> for SyncSummary { + fn from(value: &StateSyncUpdate) -> Self { + SyncSummary::new( + value.block_header.block_num(), + value.note_updates.new_input_notes().iter().map(|n| n.id()).collect(), + value.note_updates.committed_note_ids().into_iter().collect(), + value.note_updates.consumed_note_ids().into_iter().collect(), + value + .account_updates + .updated_public_accounts() + .iter() + .map(|acc| acc.id()) + .collect(), + value + .account_updates + .mismatched_private_accounts() + .iter() + .map(|(id, _)| *id) + .collect(), + value + .transaction_updates + .committed_transactions() + .iter() + .map(|t| t.transaction_id) + .collect(), + ) + } +} /// Contains stats about the sync operation. pub struct SyncSummary { @@ -138,8 +198,7 @@ impl Client { loop { // Get current state of the client let current_block_num = self.store.get_sync_height().await?; - let (current_block, has_relevant_notes) = - self.store.get_block_header_by_num(current_block_num).await?; + let current_block = self.store.get_block_header_by_num(current_block_num).await?.0; let accounts: Vec = self .store @@ -149,35 +208,61 @@ impl Client { .map(|(acc_header, _)| acc_header) .collect(); - let note_tags: Vec = - self.store.get_unique_note_tags().await?.into_iter().collect(); - - let unspent_input_notes = self.get_input_notes(NoteFilter::Unspent).await?; - let unspent_output_notes = self.get_output_notes(NoteFilter::Unspent).await?; - // Get the sync update from the network + let rpc_clone = self.rpc_api.clone(); let status = StateSync::new( self.rpc_api.clone(), current_block, - has_relevant_notes, accounts, - note_tags, - unspent_input_notes, - unspent_output_notes, - self.build_current_partial_mmr(false).await?, + self.store.get_unique_note_tags().await?.into_iter().collect(), + self.store.get_expected_note_ids().await?, + Box::new(move |note, block_header| { + Box::pin(fetch_public_note_details( + rpc_clone.clone(), + *note.note_id(), + block_header, + )) + }), + self.store.get_unspent_input_note_nullifiers().await?, ) .sync_state_step() .await?; - let (is_last_block, state_sync_update) = if let Some(status) = status { + let (is_last_block, relevant_sync_info) = if let Some(status) = status { ( matches!(status, SyncStatus::SyncedToLastBlock(_)), - status.into_state_sync_update(), + status.into_relevant_sync_info(), ) } else { break; }; + let (note_updates, transaction_updates, tags_to_remove) = self + .note_state_update( + relevant_sync_info.new_notes, + &relevant_sync_info.block_header, + relevant_sync_info.expected_note_inclusions, + relevant_sync_info.nullifiers, + relevant_sync_info.committed_transactions, + ) + .await?; + + let (new_mmr_peaks, new_authentication_nodes) = + self.apply_mmr_changes(relevant_sync_info.mmr_delta).await?; + + let state_sync_update = StateSyncUpdate { + block_header: relevant_sync_info.block_header, + note_updates, + transaction_updates, + new_mmr_peaks, + new_authentication_nodes, + account_updates: AccountUpdates::new( + relevant_sync_info.updated_public_accounts, + relevant_sync_info.mismatched_private_accounts, + ), + tags_to_remove, + }; + let sync_summary: SyncSummary = (&state_sync_update).into(); let has_relevant_notes = @@ -199,6 +284,223 @@ impl Client { Ok(total_sync_summary) } + + // HELPERS + // -------------------------------------------------------------------------------------------- + + /// Returns the [NoteUpdates] containing new public note and committed input/output notes and a + /// list or note tag records to be removed from the store. + async fn committed_note_updates( + &self, + expected_note_inclusions: Vec, + block_header: &BlockHeader, + ) -> Result<(NoteUpdates, Vec), ClientError> { + let relevant_note_filter = NoteFilter::List( + expected_note_inclusions.iter().map(|note| note.note_id()).cloned().collect(), + ); + + let mut committed_input_notes: BTreeMap = self + .store + .get_input_notes(relevant_note_filter.clone()) + .await? + .into_iter() + .map(|n| (n.id(), n)) + .collect(); + + let mut committed_output_notes: BTreeMap = self + .store + .get_output_notes(relevant_note_filter) + .await? + .into_iter() + .map(|n| (n.id(), n)) + .collect(); + + let mut committed_tracked_input_notes = vec![]; + let mut committed_tracked_output_notes = vec![]; + let mut removed_tags = vec![]; + + for committed_note in expected_note_inclusions { + let inclusion_proof = NoteInclusionProof::new( + block_header.block_num(), + committed_note.note_index(), + committed_note.merkle_path().clone(), + )?; + + if let Some(mut note_record) = committed_input_notes.remove(committed_note.note_id()) { + // The note belongs to our locally tracked set of input notes + + let inclusion_proof_received = note_record + .inclusion_proof_received(inclusion_proof.clone(), committed_note.metadata())?; + let block_header_received = note_record.block_header_received(*block_header)?; + + removed_tags.push((¬e_record).try_into()?); + + if inclusion_proof_received || block_header_received { + committed_tracked_input_notes.push(note_record); + } + } + + if let Some(mut note_record) = committed_output_notes.remove(committed_note.note_id()) { + // The note belongs to our locally tracked set of output notes + + if note_record.inclusion_proof_received(inclusion_proof.clone())? { + committed_tracked_output_notes.push(note_record); + } + } + } + + Ok(( + NoteUpdates::new( + vec![], + vec![], + committed_tracked_input_notes, + committed_tracked_output_notes, + ), + removed_tags, + )) + } + + /// Returns the [NoteUpdates] containing consumed input/output notes and a list of IDs of the + /// transactions that were discarded. + async fn consumed_note_updates( + &self, + nullifiers: Vec, + committed_transactions: &[TransactionUpdate], + ) -> Result<(NoteUpdates, Vec), ClientError> { + let nullifier_filter = NoteFilter::Nullifiers( + nullifiers.iter().map(|nullifier_update| nullifier_update.nullifier).collect(), + ); + + let mut consumed_input_notes: BTreeMap = self + .store + .get_input_notes(nullifier_filter.clone()) + .await? + .into_iter() + .map(|n| (n.nullifier(), n)) + .collect(); + + let mut consumed_output_notes: BTreeMap = self + .store + .get_output_notes(nullifier_filter) + .await? + .into_iter() + .map(|n| { + ( + n.nullifier() + .expect("Output notes returned by this query should have nullifiers"), + n, + ) + }) + .collect(); + + let mut consumed_tracked_input_notes = vec![]; + let mut consumed_tracked_output_notes = vec![]; + + // Committed transactions + for transaction_update in committed_transactions { + let transaction_nullifiers: Vec = consumed_input_notes + .iter() + .filter_map(|(nullifier, note_record)| { + if note_record.is_processing() + && note_record.consumer_transaction_id() + == Some(&transaction_update.transaction_id) + { + Some(nullifier) + } else { + None + } + }) + .cloned() + .collect(); + + for nullifier in transaction_nullifiers { + if let Some(mut input_note_record) = consumed_input_notes.remove(&nullifier) { + if input_note_record.transaction_committed( + transaction_update.transaction_id, + transaction_update.block_num, + )? { + consumed_tracked_input_notes.push(input_note_record); + } + } + } + } + + // Nullified notes + let mut discarded_transactions = vec![]; + for nullifier_update in nullifiers { + let nullifier = nullifier_update.nullifier; + let block_num = nullifier_update.block_num; + + if let Some(mut input_note_record) = consumed_input_notes.remove(&nullifier) { + if input_note_record.is_processing() { + discarded_transactions.push( + *input_note_record + .consumer_transaction_id() + .expect("Processing note should have consumer transaction id"), + ); + } + + if input_note_record.consumed_externally(nullifier, block_num)? { + consumed_tracked_input_notes.push(input_note_record); + } + } + + if let Some(mut output_note_record) = consumed_output_notes.remove(&nullifier) { + if output_note_record.nullifier_received(nullifier, block_num)? { + consumed_tracked_output_notes.push(output_note_record); + } + } + } + + Ok(( + NoteUpdates::new( + vec![], + vec![], + consumed_tracked_input_notes, + consumed_tracked_output_notes, + ), + discarded_transactions, + )) + } + + /// Compares the state of tracked notes with the updates received from the node and returns the + /// note and transaction changes that should be applied to the store plus a list of note tag + /// records that are no longer relevant. + /// + /// The note changes might include: + /// * New notes that we received from the node and might be relevant to the client. + /// * Tracked expected notes that were committed in the block. + /// * Tracked notes that were being processed by a transaction that got committed. + /// * Tracked notes that were nullified by an external transaction. + /// + /// The transaction changes might include: + /// * Transactions that were committed in the block. Some of these might me tracked by the + /// client and need to be marked as committed. + /// * Local tracked transactions that were discarded because the notes that they were processing + /// were nullified by an another transaction. + async fn note_state_update( + &self, + new_input_notes: Vec, + block_header: &BlockHeader, + committed_notes: Vec, + nullifiers: Vec, + committed_transactions: Vec, + ) -> Result<(NoteUpdates, TransactionUpdates, Vec), ClientError> { + let (committed_note_updates, tags_to_remove) = + self.committed_note_updates(committed_notes, block_header).await?; + + let (consumed_note_updates, discarded_transactions) = + self.consumed_note_updates(nullifiers, &committed_transactions).await?; + + let note_updates = NoteUpdates::new(new_input_notes, vec![], vec![], vec![]) + .combine_with(committed_note_updates) + .combine_with(consumed_note_updates); + + let transaction_updates = + TransactionUpdates::new(committed_transactions, discarded_transactions); + + Ok((note_updates, transaction_updates, tags_to_remove)) + } } pub(crate) fn get_nullifier_prefix(nullifier: &Nullifier) -> u16 { diff --git a/crates/rust-client/src/sync/state_sync.rs b/crates/rust-client/src/sync/state_sync.rs index af2187167..3999c7191 100644 --- a/crates/rust-client/src/sync/state_sync.rs +++ b/crates/rust-client/src/sync/state_sync.rs @@ -1,97 +1,55 @@ -use alloc::{ - collections::{BTreeMap, BTreeSet}, - sync::Arc, - vec::Vec, -}; +use alloc::{boxed::Box, sync::Arc, vec::Vec}; +use core::{future::Future, pin::Pin}; use miden_objects::{ accounts::{Account, AccountHeader, AccountId}, - crypto::merkle::{InOrderIndex, MmrDelta, MmrPeaks, PartialMmr}, - notes::{NoteId, NoteInclusionProof, NoteTag, Nullifier}, - transaction::TransactionId, + crypto::merkle::MmrDelta, + notes::{NoteId, NoteTag, Nullifier}, BlockHeader, Digest, }; -use tracing::info; -use super::{get_nullifier_prefix, NoteTagRecord, SyncSummary}; +use super::get_nullifier_prefix; use crate::{ - accounts::AccountUpdates, - notes::NoteUpdates, rpc::{ domain::{ notes::CommittedNote, nullifiers::NullifierUpdate, transactions::TransactionUpdate, }, NodeRpcClient, }, - store::{InputNoteRecord, OutputNoteRecord, StoreError}, - transactions::TransactionUpdates, + store::InputNoteRecord, ClientError, }; -/// Contains all information needed to apply the update in the store after syncing with the node. -pub struct StateSyncUpdate { - /// The new block header, returned as part of the - /// [StateSyncInfo](crate::rpc::domain::sync::StateSyncInfo) +pub struct RelevantSyncInfo { pub block_header: BlockHeader, - /// Information about note changes after the sync. - pub note_updates: NoteUpdates, - /// Information about transaction changes after the sync. - pub transaction_updates: TransactionUpdates, - /// New MMR peaks for the locally tracked MMR of the blockchain. - pub new_mmr_peaks: MmrPeaks, - /// New authentications nodes that are meant to be stored in order to authenticate block - /// headers. - pub new_authentication_nodes: Vec<(InOrderIndex, Digest)>, - /// Information abount account changes after the sync. - pub account_updates: AccountUpdates, - /// Tag records that are no longer relevant. - pub tags_to_remove: Vec, -} - -impl From<&StateSyncUpdate> for SyncSummary { - fn from(value: &StateSyncUpdate) -> Self { - SyncSummary::new( - value.block_header.block_num(), - value.note_updates.new_input_notes().iter().map(|n| n.id()).collect(), - value.note_updates.committed_note_ids().into_iter().collect(), - value.note_updates.consumed_note_ids().into_iter().collect(), - value - .account_updates - .updated_public_accounts() - .iter() - .map(|acc| acc.id()) - .collect(), - value - .account_updates - .mismatched_private_accounts() - .iter() - .map(|(id, _)| *id) - .collect(), - value - .transaction_updates - .committed_transactions() - .iter() - .map(|t| t.transaction_id) - .collect(), - ) - } + pub expected_note_inclusions: Vec, + pub new_notes: Vec, + pub nullifiers: Vec, + pub committed_transactions: Vec, + pub updated_public_accounts: Vec, + pub mismatched_private_accounts: Vec<(AccountId, Digest)>, + pub mmr_delta: MmrDelta, } /// Gives information about the status of the sync process after a step. pub enum SyncStatus { - SyncedToLastBlock(StateSyncUpdate), - SyncedToBlock(StateSyncUpdate), + SyncedToLastBlock(RelevantSyncInfo), + SyncedToBlock(RelevantSyncInfo), } impl SyncStatus { - pub fn into_state_sync_update(self) -> StateSyncUpdate { + pub fn into_relevant_sync_info(self) -> RelevantSyncInfo { match self { - SyncStatus::SyncedToLastBlock(update) => update, - SyncStatus::SyncedToBlock(update) => update, + SyncStatus::SyncedToLastBlock(info) => info, + SyncStatus::SyncedToBlock(info) => info, } } } +type NewNoteFilter = Box< + dyn Fn(CommittedNote, BlockHeader) -> Pin>>>, +>; + /// The state sync components encompasses the client's sync logic. /// /// When created it receives the current state of the client's relevant elements (block, accounts, @@ -99,14 +57,12 @@ impl SyncStatus { /// elements. The updates are then returned and can be applied to the store to persist the changes. pub struct StateSync { rpc_api: Arc, + account_states: Vec, current_block: BlockHeader, - current_block_has_relevant_notes: bool, - accounts: Vec, note_tags: Vec, - unspent_input_notes: BTreeMap, - unspent_output_notes: BTreeMap, - changed_notes: BTreeSet, - current_partial_mmr: PartialMmr, + expected_note_ids: Vec, + new_note_filter: NewNoteFilter, + unspent_nullifiers: Vec, } impl StateSync { @@ -128,28 +84,20 @@ impl StateSync { pub fn new( rpc_api: Arc, current_block: BlockHeader, - current_block_has_relevant_notes: bool, - accounts: Vec, + account_states: Vec, note_tags: Vec, - unspent_input_notes: Vec, - unspent_output_notes: Vec, - current_partial_mmr: PartialMmr, + expected_note_ids: Vec, + new_note_filter: NewNoteFilter, + unspent_nullifiers: Vec, ) -> Self { - let unspent_input_notes = - unspent_input_notes.into_iter().map(|note| (note.id(), note)).collect(); - let unspent_output_notes = - unspent_output_notes.into_iter().map(|note| (note.id(), note)).collect(); - Self { rpc_api, current_block, - current_block_has_relevant_notes, - accounts, note_tags, - unspent_input_notes, - unspent_output_notes, - changed_notes: BTreeSet::new(), - current_partial_mmr, + expected_note_ids, + new_note_filter, + account_states, + unspent_nullifiers, } } @@ -162,27 +110,16 @@ impl StateSync { /// Wheter or not the client has reached the chain tip is indicated by the returned /// [SyncStatus] variant. `None` is returned if the client is already synced with the chain tip /// and there are no new changes. - pub async fn sync_state_step(mut self) -> Result, ClientError> { + pub async fn sync_state_step(self) -> Result, ClientError> { let current_block_num = self.current_block.block_num(); - let account_ids: Vec = self.accounts.iter().map(|acc| acc.id()).collect(); + let account_ids: Vec = self.account_states.iter().map(|acc| acc.id()).collect(); // To receive information about added nullifiers, we reduce them to the higher 16 bits // Note that besides filtering by nullifier prefixes, the node also filters by block number // (it only returns nullifiers from current_block_num until // response.block_header.block_num()) - let input_note_nullifiers = self - .unspent_input_notes - .values() - .map(|note| get_nullifier_prefix(¬e.nullifier())); - - let output_note_nullifiers = self - .unspent_output_notes - .values() - .filter_map(|note| note.nullifier()) - .map(|nullifier| get_nullifier_prefix(&nullifier)); - let nullifiers_tags: Vec = - input_note_nullifiers.chain(output_note_nullifiers).collect(); + self.unspent_nullifiers.iter().map(get_nullifier_prefix).collect(); let response = self .rpc_api @@ -194,48 +131,45 @@ impl StateSync { return Ok(None); } - let (note_updates, transaction_updates) = self - .note_state_sync( - &response.block_header, - response.note_inclusions, - response.nullifiers, - response.transactions, - ) - .await?; + let mut expected_note_inclusions = vec![]; + let mut relevant_new_notes = vec![]; - // We can remove tags from notes that got committed - let tags_to_remove = note_updates - .updated_input_notes() - .iter() - .filter(|note| note.is_committed()) - .map(|note| { - NoteTagRecord::with_note_source( - note.metadata().expect("Committed note should have metadata").tag(), - note.id(), - ) - }) - .collect(); + for committed_note in response.note_inclusions { + if self.expected_note_ids.contains(committed_note.note_id()) { + expected_note_inclusions.push(committed_note.clone()); + } else if let Some(new_note) = + (self.new_note_filter)(committed_note, response.block_header).await + { + relevant_new_notes.push(new_note); + } + } - // ACCOUNTS - let account_updates = self.account_state_sync(&response.account_hash_updates).await?; + let (updated_public_accounts, mismatched_private_accounts) = + self.account_state_sync(&response.account_hash_updates).await?; - // MMR - let new_authentication_nodes = self.update_partial_mmr(response.mmr_delta).await?; + let relevant_nullifiers = response + .nullifiers + .into_iter() + .filter(|nullifier_update| { + self.unspent_nullifiers.contains(&nullifier_update.nullifier) + }) + .collect(); - let update = StateSyncUpdate { + let info = RelevantSyncInfo { block_header: response.block_header, - note_updates, - transaction_updates, - new_mmr_peaks: self.current_partial_mmr.peaks(), - new_authentication_nodes, - account_updates, - tags_to_remove, + expected_note_inclusions, + new_notes: relevant_new_notes, + nullifiers: relevant_nullifiers, + committed_transactions: response.transactions, + updated_public_accounts, + mismatched_private_accounts, + mmr_delta: response.mmr_delta, }; if response.chain_tip == response.block_header.block_num() { - Ok(Some(SyncStatus::SyncedToLastBlock(update))) + Ok(Some(SyncStatus::SyncedToLastBlock(info))) } else { - Ok(Some(SyncStatus::SyncedToBlock(update))) + Ok(Some(SyncStatus::SyncedToBlock(info))) } } @@ -253,236 +187,24 @@ impl StateSync { async fn account_state_sync( &self, account_hash_updates: &[(AccountId, Digest)], - ) -> Result { - let (public_accounts, offchain_accounts): (Vec<_>, Vec<_>) = - self.accounts.iter().partition(|account_header| account_header.id().is_public()); + ) -> Result<(Vec, Vec<(AccountId, Digest)>), ClientError> { + let (public_accounts, private_accounts): (Vec<_>, Vec<_>) = + self.account_states.iter().partition(|acc| acc.id().is_public()); let updated_public_accounts = self.get_updated_public_accounts(account_hash_updates, &public_accounts).await?; let mismatched_private_accounts = account_hash_updates .iter() - .filter(|(account_id, digest)| { - offchain_accounts + .filter(|(new_id, new_hash)| { + private_accounts .iter() - .any(|account| account.id() == *account_id && &account.hash() != digest) + .any(|acc| acc.id() == *new_id && acc.hash() != *new_hash) }) .cloned() .collect::>(); - Ok(AccountUpdates::new(updated_public_accounts, mismatched_private_accounts)) - } - - /// Compares the state of tracked notes with the updates received from the node and returns the - /// note and transaction changes that should be applied to the store. - /// - /// The note changes might include: - /// * New notes that we received from the node and might be relevant to the client. - /// * Tracked expected notes that were committed in the block. - /// * Tracked notes that were being processed by a transaction that got committed. - /// * Tracked notes that were nullified by an external transaction. - /// - /// The transaction changes might include: - /// * Transactions that were committed in the block. Some of these might me tracked by the - /// client and need to be marked as committed. - /// * Local tracked transactions that were discarded because the notes that they were processing - /// were nullified by an another transaction. - async fn note_state_sync( - &mut self, - block_header: &BlockHeader, - committed_notes: Vec, - nullifiers: Vec, - committed_transactions: Vec, - ) -> Result<(NoteUpdates, TransactionUpdates), ClientError> { - let new_notes = self.committed_note_updates(committed_notes, block_header).await?; - - let discarded_transactions = - self.consumed_note_updates(nullifiers, &committed_transactions).await?; - - let modified_input_notes: Vec = self - .changed_notes - .iter() - .filter_map(|note_id| self.unspent_input_notes.remove(note_id)) - .collect(); - - let modified_output_notes: Vec = self - .changed_notes - .iter() - .filter_map(|note_id| self.unspent_output_notes.remove(note_id)) - .collect(); - - let note_updates = - NoteUpdates::new(new_notes, vec![], modified_input_notes, modified_output_notes); - let transaction_updates = - TransactionUpdates::new(committed_transactions, discarded_transactions); - - Ok((note_updates, transaction_updates)) - } - - /// Updates the unspent notes with the notes that were committed in the block. Returns the IDs - /// of new public notes that matched the provided tags. - async fn committed_note_updates( - &mut self, - committed_notes: Vec, - block_header: &BlockHeader, - ) -> Result, ClientError> { - let mut new_public_notes = vec![]; - - // We'll only pick committed notes that we are tracking as input/output notes. Since the - // sync response contains notes matching either the provided accounts or the provided tag - // we might get many notes when we only care about a few of those. - for committed_note in committed_notes { - let inclusion_proof = NoteInclusionProof::new( - block_header.block_num(), - committed_note.note_index(), - committed_note.merkle_path().clone(), - )?; - - if let Some(note_record) = self.unspent_input_notes.get_mut(committed_note.note_id()) { - // The note belongs to our locally tracked set of input notes - let inclusion_proof_received = note_record - .inclusion_proof_received(inclusion_proof.clone(), committed_note.metadata())?; - let block_header_received = note_record.block_header_received(*block_header)?; - - if inclusion_proof_received || block_header_received { - self.changed_notes.insert(*committed_note.note_id()); - } - } - - if let Some(note_record) = self.unspent_output_notes.get_mut(committed_note.note_id()) { - // The note belongs to our locally tracked set of output notes - if note_record.inclusion_proof_received(inclusion_proof.clone())? { - self.changed_notes.insert(*committed_note.note_id()); - } - } - - if !self.unspent_input_notes.contains_key(committed_note.note_id()) - && !self.unspent_output_notes.contains_key(committed_note.note_id()) - { - // The note totally new to the client - new_public_notes.push(*committed_note.note_id()); - } - } - - // Query the node for input note data and build the entities - self.fetch_public_note_details(&new_public_notes, block_header).await - } - - /// Updates the unspent notes to nullify those that were consumed (either internally or - /// externally). Returns the IDs of the transactions that were discarded. - async fn consumed_note_updates( - &mut self, - nullifiers: Vec, - committed_transactions: &[TransactionUpdate], - ) -> Result, ClientError> { - let nullifier_filter: Vec = nullifiers.iter().map(|n| n.nullifier).collect(); - - let consumed_input_notes = self - .unspent_input_notes - .values() - .filter(|&n| nullifier_filter.contains(&n.nullifier())) - .map(|n| (n.nullifier(), n.id())); - - let consumed_output_notes = self - .unspent_output_notes - .values() - .filter(|&n| n.nullifier().is_some_and(|n| nullifier_filter.contains(&n))) - .map(|n| { - (n.nullifier().expect("Output notes without nullifier were filtered"), n.id()) - }); - - let mut consumed_note_ids: BTreeMap = - consumed_input_notes.chain(consumed_output_notes).collect(); - - // Modify notes that were being processed by a transaciton that just got committed. These - // notes were consumed internally. - for transaction_update in committed_transactions { - // Get the notes that were being processed by the transaction - let transaction_consumed_notes: Vec = consumed_note_ids - .iter() - .filter_map(|(_, note_id)| { - let note_record = self.unspent_input_notes.get(note_id)?; - if note_record.is_processing() - && note_record.consumer_transaction_id() - == Some(&transaction_update.transaction_id) - { - Some(note_id) - } else { - None - } - }) - .cloned() - .collect(); - - for note_id in transaction_consumed_notes { - if let Some(input_note_record) = self.unspent_input_notes.get_mut(¬e_id) { - if input_note_record.transaction_committed( - transaction_update.transaction_id, - transaction_update.block_num, - )? { - self.changed_notes.insert(note_id); - } - } - } - } - - let mut discarded_transactions = vec![]; - // Modify notes that were nullified and didn't have a committed transaction to consume them - // in the previous step. These notes were consumed externally. - for nullifier_update in nullifiers { - let nullifier = nullifier_update.nullifier; - let block_num = nullifier_update.block_num; - - if let Some(note_id) = consumed_note_ids.remove(&nullifier) { - if let Some(input_note_record) = self.unspent_input_notes.get_mut(¬e_id) { - if input_note_record.is_processing() { - // The input note was being processed by a local transaction but it was - // nullified externally so the transaction should be - // discarded - discarded_transactions.push( - *input_note_record - .consumer_transaction_id() - .expect("Processing note should have consumer transaction id"), - ); - } - - if input_note_record.consumed_externally(nullifier, block_num)? { - self.changed_notes.insert(note_id); - } - } - - if let Some(output_note_record) = self.unspent_output_notes.get_mut(¬e_id) { - if output_note_record.nullifier_received(nullifier, block_num)? { - self.changed_notes.insert(note_id); - } - } - } - } - - Ok(discarded_transactions) - } - - /// Queries the node for all received notes that aren't being locally tracked in the client. - /// - /// The client can receive metadata for private notes that it's not tracking. In this case, - /// notes are ignored for now as they become useless until details are imported. - async fn fetch_public_note_details( - &self, - query_notes: &[NoteId], - block_header: &BlockHeader, - ) -> Result, ClientError> { - if query_notes.is_empty() { - return Ok(vec![]); - } - info!("Getting note details for notes that are not being tracked."); - - let mut return_notes = self.rpc_api.get_public_note_records(query_notes, None).await?; - - for note in return_notes.iter_mut() { - note.block_header_received(*block_header)?; - } - - Ok(return_notes) + Ok((updated_public_accounts, mismatched_private_accounts)) } async fn get_updated_public_accounts( @@ -507,27 +229,23 @@ impl StateSync { .await .map_err(ClientError::RpcError) } +} - /// Updates the `current_partial_mmr` and returns the authentication nodes for tracked leaves. - pub(crate) async fn update_partial_mmr( - &mut self, - mmr_delta: MmrDelta, - ) -> Result, ClientError> { - // First, apply curent_block to the Mmr - let new_authentication_nodes = self - .current_partial_mmr - .add(self.current_block.hash(), self.current_block_has_relevant_notes) - .into_iter(); - - // Apply the Mmr delta to bring Mmr to forest equal to chain tip - let new_authentication_nodes: Vec<(InOrderIndex, Digest)> = self - .current_partial_mmr - .apply(mmr_delta) - .map_err(StoreError::MmrError)? - .into_iter() - .chain(new_authentication_nodes) - .collect(); - - Ok(new_authentication_nodes) +/// Queries the node for the received note that isn't being locally tracked in the client. +/// +/// The client can receive metadata for private notes that it's not tracking. In this case, +/// notes are ignored for now as they become useless until details are imported. +pub(crate) async fn fetch_public_note_details( + rpc_api: Arc, + note_id: NoteId, + block_header: BlockHeader, +) -> Option { + let mut return_notes = rpc_api.get_public_note_records(&[note_id], None).await.ok()?; + + if let Some(mut note) = return_notes.pop() { + note.block_header_received(block_header).ok()?; + Some(note) + } else { + None } } From b0930cf96b2f9413404aa20832e5b9d4d1e404a7 Mon Sep 17 00:00:00 2001 From: tomyrd Date: Tue, 7 Jan 2025 23:35:08 -0300 Subject: [PATCH 10/34] feat: add update callbacks --- crates/rust-client/src/store/mod.rs | 20 - crates/rust-client/src/sync/block_headers.rs | 36 +- crates/rust-client/src/sync/mod.rs | 402 +++----------- crates/rust-client/src/sync/state_sync.rs | 527 ++++++++++++++----- 4 files changed, 473 insertions(+), 512 deletions(-) diff --git a/crates/rust-client/src/store/mod.rs b/crates/rust-client/src/store/mod.rs index cdcc84265..b4733ac07 100644 --- a/crates/rust-client/src/store/mod.rs +++ b/crates/rust-client/src/store/mod.rs @@ -132,26 +132,6 @@ pub trait Store: Send + Sync { nullifiers } - /// Returns the note IDs of all expected notes. - /// - /// The default implementation of this method uses [Store::get_input_notes] and - /// [Store::get_output_notes]. - async fn get_expected_note_ids(&self) -> Result, StoreError> { - let input_notes = self - .get_input_notes(NoteFilter::Expected) - .await? - .into_iter() - .map(|input_note| input_note.id()); - - let output_notes = self - .get_output_notes(NoteFilter::Expected) - .await? - .into_iter() - .map(|output_note| output_note.id()); - - Ok(input_notes.chain(output_notes).collect()) - } - /// Inserts the provided input notes into the database. If a note with the same ID already /// exists, it will be replaced. async fn upsert_input_notes(&self, notes: &[InputNoteRecord]) -> Result<(), StoreError>; diff --git a/crates/rust-client/src/sync/block_headers.rs b/crates/rust-client/src/sync/block_headers.rs index 63309fece..3bd6848d5 100644 --- a/crates/rust-client/src/sync/block_headers.rs +++ b/crates/rust-client/src/sync/block_headers.rs @@ -3,11 +3,7 @@ use alloc::vec::Vec; use crypto::merkle::{InOrderIndex, MmrPeaks, PartialMmr}; use miden_objects::{ block::{block_epoch_from_number, block_num_from_epoch}, - crypto::{ - self, - merkle::{MerklePath, MmrDelta}, - rand::FeltRng, - }, + crypto::{self, merkle::MerklePath, rand::FeltRng}, BlockHeader, Digest, }; use tracing::warn; @@ -23,7 +19,7 @@ use crate::{ impl Client { /// Updates committed notes with no MMR data. These could be notes that were /// imported with an inclusion proof, but its block header isn't tracked. - pub(crate) async fn update_mmr_data(&mut self) -> Result<(), ClientError> { + pub(crate) async fn update_mmr_data(&self) -> Result<(), ClientError> { let mut current_partial_mmr = self.build_current_partial_mmr(true).await?; let mut changed_notes = vec![]; @@ -148,7 +144,7 @@ impl Client { /// If the store already contains MMR data for the requested block number, the request isn't /// done and the stored block header is returned. pub(crate) async fn get_and_store_authenticated_block( - &mut self, + &self, block_num: u32, current_partial_mmr: &mut PartialMmr, ) -> Result { @@ -212,32 +208,6 @@ impl Client { let current_block_num = self.store.get_sync_height().await?; self.get_epoch_block(current_block_num).await } - - /// Applies changes to the current MMR structure, returns the updated [PartialMmr] and the - /// authentication nodes for leaves we track. - pub(crate) async fn apply_mmr_changes( - &self, - mmr_delta: MmrDelta, - ) -> Result<(MmrPeaks, Vec<(InOrderIndex, Digest)>), ClientError> { - let mut partial_mmr = self.build_current_partial_mmr(false).await?; - let (current_block_header, current_block_has_relevant_notes) = - self.store.get_block_header_by_num(self.get_sync_height().await?).await?; - - // First, apply curent_block to the Mmr - let new_authentication_nodes = partial_mmr - .add(current_block_header.hash(), current_block_has_relevant_notes) - .into_iter(); - - // Apply the Mmr delta to bring Mmr to forest equal to chain tip - let new_authentication_nodes: Vec<(InOrderIndex, Digest)> = partial_mmr - .apply(mmr_delta) - .map_err(StoreError::MmrError)? - .into_iter() - .chain(new_authentication_nodes) - .collect(); - - Ok((partial_mmr.peaks(), new_authentication_nodes)) - } } // UTILS diff --git a/crates/rust-client/src/sync/mod.rs b/crates/rust-client/src/sync/mod.rs index 0e2c75bf7..e49eee1fe 100644 --- a/crates/rust-client/src/sync/mod.rs +++ b/crates/rust-client/src/sync/mod.rs @@ -3,30 +3,16 @@ use alloc::vec::Vec; use core::cmp::max; -use std::{boxed::Box, collections::BTreeMap}; +use std::boxed::Box; use miden_objects::{ - accounts::{AccountHeader, AccountId}, - crypto::{ - merkle::{InOrderIndex, MmrPeaks}, - rand::FeltRng, - }, - notes::{NoteId, NoteInclusionProof, Nullifier}, + accounts::AccountId, + crypto::rand::FeltRng, + notes::{NoteId, NoteTag, Nullifier}, transaction::TransactionId, - BlockHeader, Digest, }; -use state_sync::fetch_public_note_details; -use crate::{ - accounts::AccountUpdates, - notes::NoteUpdates, - rpc::domain::{ - notes::CommittedNote, nullifiers::NullifierUpdate, transactions::TransactionUpdate, - }, - store::{InputNoteRecord, NoteFilter, OutputNoteRecord}, - transactions::TransactionUpdates, - Client, ClientError, -}; +use crate::{notes::NoteUpdates, Client, ClientError}; mod block_headers; @@ -34,56 +20,10 @@ mod tags; pub use tags::{NoteTagRecord, NoteTagSource}; mod state_sync; -pub use state_sync::{StateSync, SyncStatus}; - -/// Contains all information needed to apply the update in the store after syncing with the node. -pub struct StateSyncUpdate { - /// The new block header, returned as part of the - /// [StateSyncInfo](crate::rpc::domain::sync::StateSyncInfo) - pub block_header: BlockHeader, - /// Information about note changes after the sync. - pub note_updates: NoteUpdates, - /// Information about transaction changes after the sync. - pub transaction_updates: TransactionUpdates, - /// New MMR peaks for the locally tracked MMR of the blockchain. - pub new_mmr_peaks: MmrPeaks, - /// New authentications nodes that are meant to be stored in order to authenticate block - /// headers. - pub new_authentication_nodes: Vec<(InOrderIndex, Digest)>, - /// Information abount account changes after the sync. - pub account_updates: AccountUpdates, - /// Tag records that are no longer relevant. - pub tags_to_remove: Vec, -} - -impl From<&StateSyncUpdate> for SyncSummary { - fn from(value: &StateSyncUpdate) -> Self { - SyncSummary::new( - value.block_header.block_num(), - value.note_updates.new_input_notes().iter().map(|n| n.id()).collect(), - value.note_updates.committed_note_ids().into_iter().collect(), - value.note_updates.consumed_note_ids().into_iter().collect(), - value - .account_updates - .updated_public_accounts() - .iter() - .map(|acc| acc.id()) - .collect(), - value - .account_updates - .mismatched_private_accounts() - .iter() - .map(|(id, _)| *id) - .collect(), - value - .transaction_updates - .committed_transactions() - .iter() - .map(|t| t.transaction_id) - .collect(), - ) - } -} +pub use state_sync::{ + account_state_sync, committed_note_updates, consumed_note_updates, StateSync, StateSyncUpdate, + SyncStatus, +}; /// Contains stats about the sync operation. pub struct SyncSummary { @@ -192,77 +132,86 @@ impl Client { pub async fn sync_state(&mut self) -> Result { _ = self.ensure_genesis_in_place().await?; + let state_sync = StateSync::new( + self.rpc_api.clone(), + Box::new({ + let store_clone = self.store.clone(); + let rpc_api_clone = self.rpc_api.clone(); + move |committed_notes, block_header| { + Box::pin(committed_note_updates( + store_clone.clone(), + rpc_api_clone.clone(), + committed_notes, + block_header, + )) + } + }), + Box::new({ + let store_clone = self.store.clone(); + move |nullifier_updates, committed_transactions| { + Box::pin(consumed_note_updates( + store_clone.clone(), + nullifier_updates, + committed_transactions, + )) + } + }), + Box::new({ + let store_clone = self.store.clone(); + let rpc_api_clone = self.rpc_api.clone(); + move |account_hash_updates| { + Box::pin(account_state_sync( + store_clone.clone(), + rpc_api_clone.clone(), + account_hash_updates, + )) + } + }), + ); + let current_block_num = self.store.get_sync_height().await?; let mut total_sync_summary = SyncSummary::new_empty(current_block_num); loop { // Get current state of the client let current_block_num = self.store.get_sync_height().await?; - let current_block = self.store.get_block_header_by_num(current_block_num).await?.0; + let (current_block, has_relevant_notes) = + self.store.get_block_header_by_num(current_block_num).await?; - let accounts: Vec = self + let account_ids: Vec = self .store .get_account_headers() .await? .into_iter() - .map(|(acc_header, _)| acc_header) + .map(|(acc_header, _)| acc_header.id()) .collect(); + let note_tags: Vec = + self.store.get_unique_note_tags().await?.into_iter().collect(); + + let unspent_nullifiers = self.store.get_unspent_input_note_nullifiers().await?; + // Get the sync update from the network - let rpc_clone = self.rpc_api.clone(); - let status = StateSync::new( - self.rpc_api.clone(), - current_block, - accounts, - self.store.get_unique_note_tags().await?.into_iter().collect(), - self.store.get_expected_note_ids().await?, - Box::new(move |note, block_header| { - Box::pin(fetch_public_note_details( - rpc_clone.clone(), - *note.note_id(), - block_header, - )) - }), - self.store.get_unspent_input_note_nullifiers().await?, - ) - .sync_state_step() - .await?; + let status = state_sync + .sync_state_step( + current_block, + has_relevant_notes, + self.build_current_partial_mmr(false).await?, + account_ids, + note_tags, + unspent_nullifiers, + ) + .await?; - let (is_last_block, relevant_sync_info) = if let Some(status) = status { + let (is_last_block, state_sync_update) = if let Some(status) = status { ( matches!(status, SyncStatus::SyncedToLastBlock(_)), - status.into_relevant_sync_info(), + status.into_state_sync_update(), ) } else { break; }; - let (note_updates, transaction_updates, tags_to_remove) = self - .note_state_update( - relevant_sync_info.new_notes, - &relevant_sync_info.block_header, - relevant_sync_info.expected_note_inclusions, - relevant_sync_info.nullifiers, - relevant_sync_info.committed_transactions, - ) - .await?; - - let (new_mmr_peaks, new_authentication_nodes) = - self.apply_mmr_changes(relevant_sync_info.mmr_delta).await?; - - let state_sync_update = StateSyncUpdate { - block_header: relevant_sync_info.block_header, - note_updates, - transaction_updates, - new_mmr_peaks, - new_authentication_nodes, - account_updates: AccountUpdates::new( - relevant_sync_info.updated_public_accounts, - relevant_sync_info.mismatched_private_accounts, - ), - tags_to_remove, - }; - let sync_summary: SyncSummary = (&state_sync_update).into(); let has_relevant_notes = @@ -284,223 +233,6 @@ impl Client { Ok(total_sync_summary) } - - // HELPERS - // -------------------------------------------------------------------------------------------- - - /// Returns the [NoteUpdates] containing new public note and committed input/output notes and a - /// list or note tag records to be removed from the store. - async fn committed_note_updates( - &self, - expected_note_inclusions: Vec, - block_header: &BlockHeader, - ) -> Result<(NoteUpdates, Vec), ClientError> { - let relevant_note_filter = NoteFilter::List( - expected_note_inclusions.iter().map(|note| note.note_id()).cloned().collect(), - ); - - let mut committed_input_notes: BTreeMap = self - .store - .get_input_notes(relevant_note_filter.clone()) - .await? - .into_iter() - .map(|n| (n.id(), n)) - .collect(); - - let mut committed_output_notes: BTreeMap = self - .store - .get_output_notes(relevant_note_filter) - .await? - .into_iter() - .map(|n| (n.id(), n)) - .collect(); - - let mut committed_tracked_input_notes = vec![]; - let mut committed_tracked_output_notes = vec![]; - let mut removed_tags = vec![]; - - for committed_note in expected_note_inclusions { - let inclusion_proof = NoteInclusionProof::new( - block_header.block_num(), - committed_note.note_index(), - committed_note.merkle_path().clone(), - )?; - - if let Some(mut note_record) = committed_input_notes.remove(committed_note.note_id()) { - // The note belongs to our locally tracked set of input notes - - let inclusion_proof_received = note_record - .inclusion_proof_received(inclusion_proof.clone(), committed_note.metadata())?; - let block_header_received = note_record.block_header_received(*block_header)?; - - removed_tags.push((¬e_record).try_into()?); - - if inclusion_proof_received || block_header_received { - committed_tracked_input_notes.push(note_record); - } - } - - if let Some(mut note_record) = committed_output_notes.remove(committed_note.note_id()) { - // The note belongs to our locally tracked set of output notes - - if note_record.inclusion_proof_received(inclusion_proof.clone())? { - committed_tracked_output_notes.push(note_record); - } - } - } - - Ok(( - NoteUpdates::new( - vec![], - vec![], - committed_tracked_input_notes, - committed_tracked_output_notes, - ), - removed_tags, - )) - } - - /// Returns the [NoteUpdates] containing consumed input/output notes and a list of IDs of the - /// transactions that were discarded. - async fn consumed_note_updates( - &self, - nullifiers: Vec, - committed_transactions: &[TransactionUpdate], - ) -> Result<(NoteUpdates, Vec), ClientError> { - let nullifier_filter = NoteFilter::Nullifiers( - nullifiers.iter().map(|nullifier_update| nullifier_update.nullifier).collect(), - ); - - let mut consumed_input_notes: BTreeMap = self - .store - .get_input_notes(nullifier_filter.clone()) - .await? - .into_iter() - .map(|n| (n.nullifier(), n)) - .collect(); - - let mut consumed_output_notes: BTreeMap = self - .store - .get_output_notes(nullifier_filter) - .await? - .into_iter() - .map(|n| { - ( - n.nullifier() - .expect("Output notes returned by this query should have nullifiers"), - n, - ) - }) - .collect(); - - let mut consumed_tracked_input_notes = vec![]; - let mut consumed_tracked_output_notes = vec![]; - - // Committed transactions - for transaction_update in committed_transactions { - let transaction_nullifiers: Vec = consumed_input_notes - .iter() - .filter_map(|(nullifier, note_record)| { - if note_record.is_processing() - && note_record.consumer_transaction_id() - == Some(&transaction_update.transaction_id) - { - Some(nullifier) - } else { - None - } - }) - .cloned() - .collect(); - - for nullifier in transaction_nullifiers { - if let Some(mut input_note_record) = consumed_input_notes.remove(&nullifier) { - if input_note_record.transaction_committed( - transaction_update.transaction_id, - transaction_update.block_num, - )? { - consumed_tracked_input_notes.push(input_note_record); - } - } - } - } - - // Nullified notes - let mut discarded_transactions = vec![]; - for nullifier_update in nullifiers { - let nullifier = nullifier_update.nullifier; - let block_num = nullifier_update.block_num; - - if let Some(mut input_note_record) = consumed_input_notes.remove(&nullifier) { - if input_note_record.is_processing() { - discarded_transactions.push( - *input_note_record - .consumer_transaction_id() - .expect("Processing note should have consumer transaction id"), - ); - } - - if input_note_record.consumed_externally(nullifier, block_num)? { - consumed_tracked_input_notes.push(input_note_record); - } - } - - if let Some(mut output_note_record) = consumed_output_notes.remove(&nullifier) { - if output_note_record.nullifier_received(nullifier, block_num)? { - consumed_tracked_output_notes.push(output_note_record); - } - } - } - - Ok(( - NoteUpdates::new( - vec![], - vec![], - consumed_tracked_input_notes, - consumed_tracked_output_notes, - ), - discarded_transactions, - )) - } - - /// Compares the state of tracked notes with the updates received from the node and returns the - /// note and transaction changes that should be applied to the store plus a list of note tag - /// records that are no longer relevant. - /// - /// The note changes might include: - /// * New notes that we received from the node and might be relevant to the client. - /// * Tracked expected notes that were committed in the block. - /// * Tracked notes that were being processed by a transaction that got committed. - /// * Tracked notes that were nullified by an external transaction. - /// - /// The transaction changes might include: - /// * Transactions that were committed in the block. Some of these might me tracked by the - /// client and need to be marked as committed. - /// * Local tracked transactions that were discarded because the notes that they were processing - /// were nullified by an another transaction. - async fn note_state_update( - &self, - new_input_notes: Vec, - block_header: &BlockHeader, - committed_notes: Vec, - nullifiers: Vec, - committed_transactions: Vec, - ) -> Result<(NoteUpdates, TransactionUpdates, Vec), ClientError> { - let (committed_note_updates, tags_to_remove) = - self.committed_note_updates(committed_notes, block_header).await?; - - let (consumed_note_updates, discarded_transactions) = - self.consumed_note_updates(nullifiers, &committed_transactions).await?; - - let note_updates = NoteUpdates::new(new_input_notes, vec![], vec![], vec![]) - .combine_with(committed_note_updates) - .combine_with(consumed_note_updates); - - let transaction_updates = - TransactionUpdates::new(committed_transactions, discarded_transactions); - - Ok((note_updates, transaction_updates, tags_to_remove)) - } } pub(crate) fn get_nullifier_prefix(nullifier: &Nullifier) -> u16 { diff --git a/crates/rust-client/src/sync/state_sync.rs b/crates/rust-client/src/sync/state_sync.rs index 3999c7191..42c921172 100644 --- a/crates/rust-client/src/sync/state_sync.rs +++ b/crates/rust-client/src/sync/state_sync.rs @@ -1,53 +1,113 @@ use alloc::{boxed::Box, sync::Arc, vec::Vec}; use core::{future::Future, pin::Pin}; +use std::collections::BTreeMap; use miden_objects::{ accounts::{Account, AccountHeader, AccountId}, - crypto::merkle::MmrDelta, - notes::{NoteId, NoteTag, Nullifier}, + crypto::merkle::{InOrderIndex, MmrDelta, MmrPeaks, PartialMmr}, + notes::{NoteId, NoteInclusionProof, NoteTag, Nullifier}, BlockHeader, Digest, }; +use tracing::info; -use super::get_nullifier_prefix; +use super::{get_nullifier_prefix, NoteTagRecord, SyncSummary}; use crate::{ + accounts::AccountUpdates, + notes::NoteUpdates, rpc::{ domain::{ notes::CommittedNote, nullifiers::NullifierUpdate, transactions::TransactionUpdate, }, NodeRpcClient, }, - store::InputNoteRecord, + store::{InputNoteRecord, NoteFilter, OutputNoteRecord, Store, StoreError}, + transactions::TransactionUpdates, ClientError, }; -pub struct RelevantSyncInfo { +/// Contains all information needed to apply the update in the store after syncing with the node. +pub struct StateSyncUpdate { + /// The new block header, returned as part of the + /// [StateSyncInfo](crate::rpc::domain::sync::StateSyncInfo) pub block_header: BlockHeader, - pub expected_note_inclusions: Vec, - pub new_notes: Vec, - pub nullifiers: Vec, - pub committed_transactions: Vec, - pub updated_public_accounts: Vec, - pub mismatched_private_accounts: Vec<(AccountId, Digest)>, - pub mmr_delta: MmrDelta, + /// Information about note changes after the sync. + pub note_updates: NoteUpdates, + /// Information about transaction changes after the sync. + pub transaction_updates: TransactionUpdates, + /// New MMR peaks for the locally tracked MMR of the blockchain. + pub new_mmr_peaks: MmrPeaks, + /// New authentications nodes that are meant to be stored in order to authenticate block + /// headers. + pub new_authentication_nodes: Vec<(InOrderIndex, Digest)>, + /// Information abount account changes after the sync. + pub account_updates: AccountUpdates, + /// Tag records that are no longer relevant. + pub tags_to_remove: Vec, +} + +impl From<&StateSyncUpdate> for SyncSummary { + fn from(value: &StateSyncUpdate) -> Self { + SyncSummary::new( + value.block_header.block_num(), + value.note_updates.new_input_notes().iter().map(|n| n.id()).collect(), + value.note_updates.committed_note_ids().into_iter().collect(), + value.note_updates.consumed_note_ids().into_iter().collect(), + value + .account_updates + .updated_public_accounts() + .iter() + .map(|acc| acc.id()) + .collect(), + value + .account_updates + .mismatched_private_accounts() + .iter() + .map(|(id, _)| *id) + .collect(), + value + .transaction_updates + .committed_transactions() + .iter() + .map(|t| t.transaction_id) + .collect(), + ) + } } /// Gives information about the status of the sync process after a step. pub enum SyncStatus { - SyncedToLastBlock(RelevantSyncInfo), - SyncedToBlock(RelevantSyncInfo), + SyncedToLastBlock(StateSyncUpdate), + SyncedToBlock(StateSyncUpdate), } impl SyncStatus { - pub fn into_relevant_sync_info(self) -> RelevantSyncInfo { + pub fn into_state_sync_update(self) -> StateSyncUpdate { match self { - SyncStatus::SyncedToLastBlock(info) => info, - SyncStatus::SyncedToBlock(info) => info, + SyncStatus::SyncedToLastBlock(update) => update, + SyncStatus::SyncedToBlock(update) => update, } } } -type NewNoteFilter = Box< - dyn Fn(CommittedNote, BlockHeader) -> Pin>>>, +type NoteInclusionUpdate = Box< + dyn Fn( + Vec, + BlockHeader, + ) -> Pin>>>, +>; + +type NewNullifierUpdate = Box< + dyn Fn( + Vec, + Vec, + ) + -> Pin>>>, +>; + +type AccountHashUpdate = Box< + dyn Fn( + Vec<(AccountId, Digest)>, + ) -> Pin>>>, >; /// The state sync components encompasses the client's sync logic. @@ -57,12 +117,9 @@ type NewNoteFilter = Box< /// elements. The updates are then returned and can be applied to the store to persist the changes. pub struct StateSync { rpc_api: Arc, - account_states: Vec, - current_block: BlockHeader, - note_tags: Vec, - expected_note_ids: Vec, - new_note_filter: NewNoteFilter, - unspent_nullifiers: Vec, + note_inclusion_update: NoteInclusionUpdate, + new_nullifier_update: NewNullifierUpdate, + account_hash_update: AccountHashUpdate, } impl StateSync { @@ -83,21 +140,15 @@ impl StateSync { /// * `current_partial_mmr` - The current partial MMR of the client. pub fn new( rpc_api: Arc, - current_block: BlockHeader, - account_states: Vec, - note_tags: Vec, - expected_note_ids: Vec, - new_note_filter: NewNoteFilter, - unspent_nullifiers: Vec, + note_inclusion_update: NoteInclusionUpdate, + new_nullifier_update: NewNullifierUpdate, + account_hash_update: AccountHashUpdate, ) -> Self { Self { rpc_api, - current_block, - note_tags, - expected_note_ids, - new_note_filter, - account_states, - unspent_nullifiers, + note_inclusion_update, + new_nullifier_update, + account_hash_update, } } @@ -110,20 +161,27 @@ impl StateSync { /// Wheter or not the client has reached the chain tip is indicated by the returned /// [SyncStatus] variant. `None` is returned if the client is already synced with the chain tip /// and there are no new changes. - pub async fn sync_state_step(self) -> Result, ClientError> { - let current_block_num = self.current_block.block_num(); - let account_ids: Vec = self.account_states.iter().map(|acc| acc.id()).collect(); + pub async fn sync_state_step( + &self, + current_block: BlockHeader, + current_block_has_relevant_notes: bool, + current_partial_mmr: PartialMmr, + account_ids: Vec, + note_tags: Vec, + unspent_nullifiers: Vec, + ) -> Result, ClientError> { + let current_block_num = current_block.block_num(); // To receive information about added nullifiers, we reduce them to the higher 16 bits // Note that besides filtering by nullifier prefixes, the node also filters by block number // (it only returns nullifiers from current_block_num until // response.block_header.block_num()) let nullifiers_tags: Vec = - self.unspent_nullifiers.iter().map(get_nullifier_prefix).collect(); + unspent_nullifiers.iter().map(get_nullifier_prefix).collect(); let response = self .rpc_api - .sync_state(current_block_num, &account_ids, &self.note_tags, &nullifiers_tags) + .sync_state(current_block_num, &account_ids, ¬e_tags, &nullifiers_tags) .await?; // We don't need to continue if the chain has not advanced, there are no new changes @@ -131,121 +189,342 @@ impl StateSync { return Ok(None); } - let mut expected_note_inclusions = vec![]; - let mut relevant_new_notes = vec![]; + let account_updates = (self.account_hash_update)(response.account_hash_updates).await?; - for committed_note in response.note_inclusions { - if self.expected_note_ids.contains(committed_note.note_id()) { - expected_note_inclusions.push(committed_note.clone()); - } else if let Some(new_note) = - (self.new_note_filter)(committed_note, response.block_header).await - { - relevant_new_notes.push(new_note); - } - } + let committed_note_updates = + (self.note_inclusion_update)(response.note_inclusions, response.block_header).await?; - let (updated_public_accounts, mismatched_private_accounts) = - self.account_state_sync(&response.account_hash_updates).await?; + let (consumed_note_updates, transaction_updates) = + (self.new_nullifier_update)(response.nullifiers, response.transactions).await?; - let relevant_nullifiers = response - .nullifiers - .into_iter() - .filter(|nullifier_update| { - self.unspent_nullifiers.contains(&nullifier_update.nullifier) + // We can remove tags from notes that got committed + let tags_to_remove = committed_note_updates + .updated_input_notes() + .iter() + .filter(|note| note.is_committed()) + .map(|note| { + NoteTagRecord::with_note_source( + note.metadata().expect("Committed note should have metadata").tag(), + note.id(), + ) }) .collect(); - let info = RelevantSyncInfo { + let (new_mmr_peaks, new_authentication_nodes) = self + .apply_mmr_changes( + current_block, + current_block_has_relevant_notes, + current_partial_mmr, + response.mmr_delta, + ) + .await?; + + let update = StateSyncUpdate { block_header: response.block_header, - expected_note_inclusions, - new_notes: relevant_new_notes, - nullifiers: relevant_nullifiers, - committed_transactions: response.transactions, - updated_public_accounts, - mismatched_private_accounts, - mmr_delta: response.mmr_delta, + note_updates: committed_note_updates.combine_with(consumed_note_updates), + transaction_updates, + new_mmr_peaks, + new_authentication_nodes, + account_updates, + tags_to_remove, }; if response.chain_tip == response.block_header.block_num() { - Ok(Some(SyncStatus::SyncedToLastBlock(info))) + Ok(Some(SyncStatus::SyncedToLastBlock(update))) } else { - Ok(Some(SyncStatus::SyncedToBlock(info))) + Ok(Some(SyncStatus::SyncedToBlock(update))) } } // HELPERS // -------------------------------------------------------------------------------------------- - /// Compares the state of tracked accounts with the updates received from the node and returns - /// the accounts that need to be updated. - /// - /// When a mismatch is detected, two scenarios are possible: - /// * If the account is public, the component will request the node for the updated account - /// details. - /// * If the account is private it will be marked as mismatched and the client will need to - /// handle it (it could be a stale account state or a reason to lock the account). - async fn account_state_sync( + /// Applies changes to the current MMR structure, returns the updated [MmrPeaks] and the + /// authentication nodes for leaves we track. + pub(crate) async fn apply_mmr_changes( &self, - account_hash_updates: &[(AccountId, Digest)], - ) -> Result<(Vec, Vec<(AccountId, Digest)>), ClientError> { - let (public_accounts, private_accounts): (Vec<_>, Vec<_>) = - self.account_states.iter().partition(|acc| acc.id().is_public()); + current_block: BlockHeader, + current_block_has_relevant_notes: bool, + mut current_partial_mmr: PartialMmr, + mmr_delta: MmrDelta, + ) -> Result<(MmrPeaks, Vec<(InOrderIndex, Digest)>), ClientError> { + // First, apply curent_block to the Mmr + let new_authentication_nodes = current_partial_mmr + .add(current_block.hash(), current_block_has_relevant_notes) + .into_iter(); + + // Apply the Mmr delta to bring Mmr to forest equal to chain tip + let new_authentication_nodes: Vec<(InOrderIndex, Digest)> = current_partial_mmr + .apply(mmr_delta) + .map_err(StoreError::MmrError)? + .into_iter() + .chain(new_authentication_nodes) + .collect(); + + Ok((current_partial_mmr.peaks(), new_authentication_nodes)) + } +} + +/// Returns the [NoteUpdates] containing new public note and committed input/output notes and a +/// list or note tag records to be removed from the store. +pub async fn committed_note_updates( + store: Arc, + rpc_api: Arc, + committed_notes: Vec, + block_header: BlockHeader, +) -> Result { + // We'll only pick committed notes that we are tracking as input/output notes. Since the + // sync response contains notes matching either the provided accounts or the provided tag + // we might get many notes when we only care about a few of those. + let relevant_note_filter = + NoteFilter::List(committed_notes.iter().map(|note| note.note_id()).cloned().collect()); + + let mut committed_input_notes: BTreeMap = store + .get_input_notes(relevant_note_filter.clone()) + .await? + .into_iter() + .map(|n| (n.id(), n)) + .collect(); + + let mut committed_output_notes: BTreeMap = store + .get_output_notes(relevant_note_filter) + .await? + .into_iter() + .map(|n| (n.id(), n)) + .collect(); + + let mut new_public_notes = vec![]; + let mut committed_tracked_input_notes = vec![]; + let mut committed_tracked_output_notes = vec![]; + + for committed_note in committed_notes { + let inclusion_proof = NoteInclusionProof::new( + block_header.block_num(), + committed_note.note_index(), + committed_note.merkle_path().clone(), + )?; + + if let Some(mut note_record) = committed_input_notes.remove(committed_note.note_id()) { + // The note belongs to our locally tracked set of input notes + + let inclusion_proof_received = note_record + .inclusion_proof_received(inclusion_proof.clone(), committed_note.metadata())?; + let block_header_received = note_record.block_header_received(block_header)?; + + if inclusion_proof_received || block_header_received { + committed_tracked_input_notes.push(note_record); + } + } + + if let Some(mut note_record) = committed_output_notes.remove(committed_note.note_id()) { + // The note belongs to our locally tracked set of output notes + + if note_record.inclusion_proof_received(inclusion_proof.clone())? { + committed_tracked_output_notes.push(note_record); + } + } + + if !committed_input_notes.contains_key(committed_note.note_id()) + && !committed_output_notes.contains_key(committed_note.note_id()) + { + // The note is public and we are not tracking it, push to the list of IDs to query + new_public_notes.push(*committed_note.note_id()); + } + } + + // Query the node for input note data and build the entities + let new_public_notes = + fetch_public_note_details(store, rpc_api, &new_public_notes, &block_header).await?; + + Ok(NoteUpdates::new( + new_public_notes, + vec![], + committed_tracked_input_notes, + committed_tracked_output_notes, + )) +} - let updated_public_accounts = - self.get_updated_public_accounts(account_hash_updates, &public_accounts).await?; +/// Queries the node for all received notes that aren't being locally tracked in the client. +/// +/// The client can receive metadata for private notes that it's not tracking. In this case, +/// notes are ignored for now as they become useless until details are imported. +async fn fetch_public_note_details( + store: Arc, + rpc_api: Arc, + query_notes: &[NoteId], + block_header: &BlockHeader, +) -> Result, ClientError> { + if query_notes.is_empty() { + return Ok(vec![]); + } + info!("Getting note details for notes that are not being tracked."); - let mismatched_private_accounts = account_hash_updates + let mut return_notes = rpc_api + .get_public_note_records(query_notes, store.get_current_timestamp()) + .await?; + + for note in return_notes.iter_mut() { + note.block_header_received(*block_header)?; + } + + Ok(return_notes) +} + +/// Returns the [NoteUpdates] containing consumed input/output notes and a list of IDs of the +/// transactions that were discarded. +pub async fn consumed_note_updates( + store: Arc, + nullifiers: Vec, + committed_transactions: Vec, +) -> Result<(NoteUpdates, TransactionUpdates), ClientError> { + let nullifier_filter = NoteFilter::Nullifiers( + nullifiers.iter().map(|nullifier_update| nullifier_update.nullifier).collect(), + ); + + let mut consumed_input_notes: BTreeMap = store + .get_input_notes(nullifier_filter.clone()) + .await? + .into_iter() + .map(|n| (n.nullifier(), n)) + .collect(); + + let mut consumed_output_notes: BTreeMap = store + .get_output_notes(nullifier_filter) + .await? + .into_iter() + .map(|n| { + ( + n.nullifier() + .expect("Output notes returned by this query should have nullifiers"), + n, + ) + }) + .collect(); + + let mut consumed_tracked_input_notes = vec![]; + let mut consumed_tracked_output_notes = vec![]; + + // Committed transactions + for transaction_update in committed_transactions.iter() { + let transaction_nullifiers: Vec = consumed_input_notes .iter() - .filter(|(new_id, new_hash)| { - private_accounts - .iter() - .any(|acc| acc.id() == *new_id && acc.hash() != *new_hash) + .filter_map(|(nullifier, note_record)| { + if note_record.is_processing() + && note_record.consumer_transaction_id() + == Some(&transaction_update.transaction_id) + { + Some(nullifier) + } else { + None + } }) .cloned() - .collect::>(); + .collect(); - Ok((updated_public_accounts, mismatched_private_accounts)) + for nullifier in transaction_nullifiers { + if let Some(mut input_note_record) = consumed_input_notes.remove(&nullifier) { + if input_note_record.transaction_committed( + transaction_update.transaction_id, + transaction_update.block_num, + )? { + consumed_tracked_input_notes.push(input_note_record); + } + } + } } - async fn get_updated_public_accounts( - &self, - account_updates: &[(AccountId, Digest)], - current_public_accounts: &[&AccountHeader], - ) -> Result, ClientError> { - let mut mismatched_public_accounts = vec![]; - - for (id, hash) in account_updates { - // check if this updated account is tracked by the client - if let Some(account) = current_public_accounts - .iter() - .find(|acc| *id == acc.id() && *hash != acc.hash()) - { - mismatched_public_accounts.push(*account); + // Nullified notes + let mut discarded_transactions = vec![]; + for nullifier_update in nullifiers { + let nullifier = nullifier_update.nullifier; + let block_num = nullifier_update.block_num; + + if let Some(mut input_note_record) = consumed_input_notes.remove(&nullifier) { + if input_note_record.is_processing() { + discarded_transactions.push( + *input_note_record + .consumer_transaction_id() + .expect("Processing note should have consumer transaction id"), + ); + } + + if input_note_record.consumed_externally(nullifier, block_num)? { + consumed_tracked_input_notes.push(input_note_record); } } - self.rpc_api - .get_updated_public_accounts(&mismatched_public_accounts) - .await - .map_err(ClientError::RpcError) + if let Some(mut output_note_record) = consumed_output_notes.remove(&nullifier) { + if output_note_record.nullifier_received(nullifier, block_num)? { + consumed_tracked_output_notes.push(output_note_record); + } + } } + + Ok(( + NoteUpdates::new( + vec![], + vec![], + consumed_tracked_input_notes, + consumed_tracked_output_notes, + ), + TransactionUpdates::new(committed_transactions, discarded_transactions), + )) } -/// Queries the node for the received note that isn't being locally tracked in the client. +/// Compares the state of tracked accounts with the updates received from the node and returns +/// the accounts that need to be updated. /// -/// The client can receive metadata for private notes that it's not tracking. In this case, -/// notes are ignored for now as they become useless until details are imported. -pub(crate) async fn fetch_public_note_details( +/// When a mismatch is detected, two scenarios are possible: +/// * If the account is public, the component will request the node for the updated account details. +/// * If the account is private it will be marked as mismatched and the client will need to handle +/// it (it could be a stale account state or a reason to lock the account). +pub async fn account_state_sync( + store: Arc, rpc_api: Arc, - note_id: NoteId, - block_header: BlockHeader, -) -> Option { - let mut return_notes = rpc_api.get_public_note_records(&[note_id], None).await.ok()?; - - if let Some(mut note) = return_notes.pop() { - note.block_header_received(block_header).ok()?; - Some(note) - } else { - None + account_hash_updates: Vec<(AccountId, Digest)>, +) -> Result { + let (public_accounts, private_accounts): (Vec<_>, Vec<_>) = store + .get_account_headers() + .await? + .into_iter() + .map(|(acc, _)| acc) + .partition(|acc| acc.id().is_public()); + + let updated_public_accounts = + get_updated_public_accounts(rpc_api, &account_hash_updates, &public_accounts).await?; + + let mismatched_private_accounts = account_hash_updates + .iter() + .filter(|(new_id, new_hash)| { + private_accounts + .iter() + .any(|acc| acc.id() == *new_id && acc.hash() != *new_hash) + }) + .cloned() + .collect::>(); + + Ok(AccountUpdates::new(updated_public_accounts, mismatched_private_accounts)) +} + +async fn get_updated_public_accounts( + rpc_api: Arc, + account_updates: &[(AccountId, Digest)], + current_public_accounts: &[AccountHeader], +) -> Result, ClientError> { + let mut mismatched_public_accounts = vec![]; + + for (id, hash) in account_updates { + // check if this updated account is tracked by the client + if let Some(account) = current_public_accounts + .iter() + .find(|acc| *id == acc.id() && *hash != acc.hash()) + { + mismatched_public_accounts.push(account); + } } + + rpc_api + .get_updated_public_accounts(&mismatched_public_accounts) + .await + .map_err(ClientError::RpcError) } From 170cc855714f521527d1410caf9fd0ce11651fde Mon Sep 17 00:00:00 2001 From: tomyrd Date: Wed, 8 Jan 2025 17:55:22 -0300 Subject: [PATCH 11/34] refactor: change callbacks to deal with individual elements --- crates/rust-client/src/notes/mod.rs | 13 +- crates/rust-client/src/sync/mod.rs | 33 +- crates/rust-client/src/sync/state_sync.rs | 486 ++++++++++----------- crates/rust-client/src/transactions/mod.rs | 12 + tests/integration/main.rs | 1 + 5 files changed, 274 insertions(+), 271 deletions(-) diff --git a/crates/rust-client/src/notes/mod.rs b/crates/rust-client/src/notes/mod.rs index 77216150e..b7a87189d 100644 --- a/crates/rust-client/src/notes/mod.rs +++ b/crates/rust-client/src/notes/mod.rs @@ -214,14 +214,21 @@ impl NoteUpdates { } } + pub fn new_empty() -> Self { + Self { + new_input_notes: Vec::new(), + new_output_notes: Vec::new(), + updated_input_notes: Vec::new(), + updated_output_notes: Vec::new(), + } + } + /// Combines two [NoteUpdates] into a single one. - pub fn combine_with(mut self, other: Self) -> Self { + pub fn combine_with(&mut self, other: Self) { self.new_input_notes.extend(other.new_input_notes); self.new_output_notes.extend(other.new_output_notes); self.updated_input_notes.extend(other.updated_input_notes); self.updated_output_notes.extend(other.updated_output_notes); - - self } /// Returns all new input note records, meant to be tracked by the client. diff --git a/crates/rust-client/src/sync/mod.rs b/crates/rust-client/src/sync/mod.rs index e49eee1fe..ebf216454 100644 --- a/crates/rust-client/src/sync/mod.rs +++ b/crates/rust-client/src/sync/mod.rs @@ -21,7 +21,7 @@ pub use tags::{NoteTagRecord, NoteTagSource}; mod state_sync; pub use state_sync::{ - account_state_sync, committed_note_updates, consumed_note_updates, StateSync, StateSyncUpdate, + on_note_received, on_nullifier_received, on_transaction_committed, StateSync, StateSyncUpdate, SyncStatus, }; @@ -136,35 +136,20 @@ impl Client { self.rpc_api.clone(), Box::new({ let store_clone = self.store.clone(); - let rpc_api_clone = self.rpc_api.clone(); move |committed_notes, block_header| { - Box::pin(committed_note_updates( - store_clone.clone(), - rpc_api_clone.clone(), - committed_notes, - block_header, - )) + Box::pin(on_note_received(store_clone.clone(), committed_notes, block_header)) } }), Box::new({ let store_clone = self.store.clone(); - move |nullifier_updates, committed_transactions| { - Box::pin(consumed_note_updates( - store_clone.clone(), - nullifier_updates, - committed_transactions, - )) + move |transaction_update| { + Box::pin(on_transaction_committed(store_clone.clone(), transaction_update)) } }), Box::new({ let store_clone = self.store.clone(); - let rpc_api_clone = self.rpc_api.clone(); - move |account_hash_updates| { - Box::pin(account_state_sync( - store_clone.clone(), - rpc_api_clone.clone(), - account_hash_updates, - )) + move |nullifier_update| { + Box::pin(on_nullifier_received(store_clone.clone(), nullifier_update)) } }), ); @@ -178,12 +163,12 @@ impl Client { let (current_block, has_relevant_notes) = self.store.get_block_header_by_num(current_block_num).await?; - let account_ids: Vec = self + let accounts = self .store .get_account_headers() .await? .into_iter() - .map(|(acc_header, _)| acc_header.id()) + .map(|(acc_header, _)| acc_header) .collect(); let note_tags: Vec = @@ -197,7 +182,7 @@ impl Client { current_block, has_relevant_notes, self.build_current_partial_mmr(false).await?, - account_ids, + accounts, note_tags, unspent_nullifiers, ) diff --git a/crates/rust-client/src/sync/state_sync.rs b/crates/rust-client/src/sync/state_sync.rs index 42c921172..994de6686 100644 --- a/crates/rust-client/src/sync/state_sync.rs +++ b/crates/rust-client/src/sync/state_sync.rs @@ -1,6 +1,6 @@ use alloc::{boxed::Box, sync::Arc, vec::Vec}; use core::{future::Future, pin::Pin}; -use std::collections::BTreeMap; +use std::vec; use miden_objects::{ accounts::{Account, AccountHeader, AccountId}, @@ -20,7 +20,7 @@ use crate::{ }, NodeRpcClient, }, - store::{InputNoteRecord, NoteFilter, OutputNoteRecord, Store, StoreError}, + store::{InputNoteRecord, NoteFilter, Store, StoreError}, transactions::TransactionUpdates, ClientError, }; @@ -89,25 +89,25 @@ impl SyncStatus { } } -type NoteInclusionUpdate = Box< +type OnNoteReceived = Box< dyn Fn( - Vec, + CommittedNote, BlockHeader, - ) -> Pin>>>, + ) -> Pin), ClientError>>>>, >; -type NewNullifierUpdate = Box< +type OnTransactionCommitted = Box< dyn Fn( - Vec, - Vec, + TransactionUpdate, ) -> Pin>>>, >; -type AccountHashUpdate = Box< +type OnNullifierReceived = Box< dyn Fn( - Vec<(AccountId, Digest)>, - ) -> Pin>>>, + NullifierUpdate, + ) + -> Pin>>>, >; /// The state sync components encompasses the client's sync logic. @@ -117,9 +117,9 @@ type AccountHashUpdate = Box< /// elements. The updates are then returned and can be applied to the store to persist the changes. pub struct StateSync { rpc_api: Arc, - note_inclusion_update: NoteInclusionUpdate, - new_nullifier_update: NewNullifierUpdate, - account_hash_update: AccountHashUpdate, + on_note_received: OnNoteReceived, + on_committed_transaction: OnTransactionCommitted, + on_nullifier_received: OnNullifierReceived, } impl StateSync { @@ -140,15 +140,15 @@ impl StateSync { /// * `current_partial_mmr` - The current partial MMR of the client. pub fn new( rpc_api: Arc, - note_inclusion_update: NoteInclusionUpdate, - new_nullifier_update: NewNullifierUpdate, - account_hash_update: AccountHashUpdate, + on_note_received: OnNoteReceived, + on_committed_transaction: OnTransactionCommitted, + on_nullifier_received: OnNullifierReceived, ) -> Self { Self { rpc_api, - note_inclusion_update, - new_nullifier_update, - account_hash_update, + on_note_received, + on_committed_transaction, + on_nullifier_received, } } @@ -166,11 +166,12 @@ impl StateSync { current_block: BlockHeader, current_block_has_relevant_notes: bool, current_partial_mmr: PartialMmr, - account_ids: Vec, + accounts: Vec, note_tags: Vec, unspent_nullifiers: Vec, ) -> Result, ClientError> { let current_block_num = current_block.block_num(); + let account_ids: Vec = accounts.iter().map(|acc| acc.id()).collect(); // To receive information about added nullifiers, we reduce them to the higher 16 bits // Note that besides filtering by nullifier prefixes, the node also filters by block number @@ -179,7 +180,7 @@ impl StateSync { let nullifiers_tags: Vec = unspent_nullifiers.iter().map(get_nullifier_prefix).collect(); - let response = self + let mut response = self .rpc_api .sync_state(current_block_num, &account_ids, ¬e_tags, &nullifiers_tags) .await?; @@ -189,16 +190,26 @@ impl StateSync { return Ok(None); } - let account_updates = (self.account_hash_update)(response.account_hash_updates).await?; + let account_updates = + self.account_state_sync(&accounts, &response.account_hash_updates).await?; - let committed_note_updates = - (self.note_inclusion_update)(response.note_inclusions, response.block_header).await?; + let mut note_updates = NoteUpdates::new_empty(); + + let mut public_note_ids = vec![]; + for committed_note in response.note_inclusions { + let (new_note_update, new_note_ids) = + (self.on_note_received)(committed_note, response.block_header).await?; + note_updates.combine_with(new_note_update); + public_note_ids.extend(new_note_ids); + } - let (consumed_note_updates, transaction_updates) = - (self.new_nullifier_update)(response.nullifiers, response.transactions).await?; + let new_public_notes = + self.fetch_public_note_details(&public_note_ids, &response.block_header).await?; + + note_updates.combine_with(NoteUpdates::new(new_public_notes, vec![], vec![], vec![])); // We can remove tags from notes that got committed - let tags_to_remove = committed_note_updates + let tags_to_remove = note_updates .updated_input_notes() .iter() .filter(|note| note.is_committed()) @@ -210,6 +221,31 @@ impl StateSync { }) .collect(); + let mut transaction_updates = TransactionUpdates::new_empty(); + + for transaction_update in response.transactions { + let (new_note_update, new_transaction_update) = + (self.on_committed_transaction)(transaction_update).await?; + + // Remove nullifiers if they were consumed by the transaction + response.nullifiers.retain(|nullifier| { + !new_note_update + .updated_input_notes() + .iter() + .any(|note| note.nullifier() == nullifier.nullifier) + }); + + note_updates.combine_with(new_note_update); + transaction_updates.combine_with(new_transaction_update); + } + + for nullifier_update in response.nullifiers { + let (new_note_update, new_transaction_update) = + (self.on_nullifier_received)(nullifier_update).await?; + note_updates.combine_with(new_note_update); + transaction_updates.combine_with(new_transaction_update); + } + let (new_mmr_peaks, new_authentication_nodes) = self .apply_mmr_changes( current_block, @@ -221,7 +257,7 @@ impl StateSync { let update = StateSyncUpdate { block_header: response.block_header, - note_updates: committed_note_updates.combine_with(consumed_note_updates), + note_updates, transaction_updates, new_mmr_peaks, new_authentication_nodes, @@ -263,268 +299,230 @@ impl StateSync { Ok((current_partial_mmr.peaks(), new_authentication_nodes)) } -} -/// Returns the [NoteUpdates] containing new public note and committed input/output notes and a -/// list or note tag records to be removed from the store. -pub async fn committed_note_updates( - store: Arc, - rpc_api: Arc, - committed_notes: Vec, - block_header: BlockHeader, -) -> Result { - // We'll only pick committed notes that we are tracking as input/output notes. Since the - // sync response contains notes matching either the provided accounts or the provided tag - // we might get many notes when we only care about a few of those. - let relevant_note_filter = - NoteFilter::List(committed_notes.iter().map(|note| note.note_id()).cloned().collect()); - - let mut committed_input_notes: BTreeMap = store - .get_input_notes(relevant_note_filter.clone()) - .await? - .into_iter() - .map(|n| (n.id(), n)) - .collect(); - - let mut committed_output_notes: BTreeMap = store - .get_output_notes(relevant_note_filter) - .await? - .into_iter() - .map(|n| (n.id(), n)) - .collect(); + // HELPERS + // -------------------------------------------------------------------------------------------- - let mut new_public_notes = vec![]; - let mut committed_tracked_input_notes = vec![]; - let mut committed_tracked_output_notes = vec![]; + /// Compares the state of tracked accounts with the updates received from the node and returns + /// the accounts that need to be updated. + /// + /// When a mismatch is detected, two scenarios are possible: + /// * If the account is public, the component will request the node for the updated account + /// details. + /// * If the account is private it will be marked as mismatched and the client will need to + /// handle it (it could be a stale account state or a reason to lock the account). + async fn account_state_sync( + &self, + accounts: &[AccountHeader], + account_hash_updates: &[(AccountId, Digest)], + ) -> Result { + let (public_accounts, offchain_accounts): (Vec<_>, Vec<_>) = + accounts.iter().partition(|account_header| account_header.id().is_public()); - for committed_note in committed_notes { - let inclusion_proof = NoteInclusionProof::new( - block_header.block_num(), - committed_note.note_index(), - committed_note.merkle_path().clone(), - )?; + let updated_public_accounts = + self.get_updated_public_accounts(account_hash_updates, &public_accounts).await?; - if let Some(mut note_record) = committed_input_notes.remove(committed_note.note_id()) { - // The note belongs to our locally tracked set of input notes + let mismatched_private_accounts = account_hash_updates + .iter() + .filter(|(account_id, digest)| { + offchain_accounts + .iter() + .any(|account| account.id() == *account_id && &account.hash() != digest) + }) + .cloned() + .collect::>(); - let inclusion_proof_received = note_record - .inclusion_proof_received(inclusion_proof.clone(), committed_note.metadata())?; - let block_header_received = note_record.block_header_received(block_header)?; + Ok(AccountUpdates::new(updated_public_accounts, mismatched_private_accounts)) + } - if inclusion_proof_received || block_header_received { - committed_tracked_input_notes.push(note_record); + async fn get_updated_public_accounts( + &self, + account_updates: &[(AccountId, Digest)], + current_public_accounts: &[&AccountHeader], + ) -> Result, ClientError> { + let mut mismatched_public_accounts = vec![]; + + for (id, hash) in account_updates { + // check if this updated account is tracked by the client + if let Some(account) = current_public_accounts + .iter() + .find(|acc| *id == acc.id() && *hash != acc.hash()) + { + mismatched_public_accounts.push(*account); } } - if let Some(mut note_record) = committed_output_notes.remove(committed_note.note_id()) { - // The note belongs to our locally tracked set of output notes + self.rpc_api + .get_updated_public_accounts(&mismatched_public_accounts) + .await + .map_err(ClientError::RpcError) + } - if note_record.inclusion_proof_received(inclusion_proof.clone())? { - committed_tracked_output_notes.push(note_record); - } + /// Queries the node for all received notes that aren't being locally tracked in the client. + /// + /// The client can receive metadata for private notes that it's not tracking. In this case, + /// notes are ignored for now as they become useless until details are imported. + async fn fetch_public_note_details( + &self, + query_notes: &[NoteId], + block_header: &BlockHeader, + ) -> Result, ClientError> { + if query_notes.is_empty() { + return Ok(vec![]); } + info!("Getting note details for notes that are not being tracked."); - if !committed_input_notes.contains_key(committed_note.note_id()) - && !committed_output_notes.contains_key(committed_note.note_id()) - { - // The note is public and we are not tracking it, push to the list of IDs to query - new_public_notes.push(*committed_note.note_id()); - } - } + let mut return_notes = self.rpc_api.get_public_note_records(query_notes, None).await?; - // Query the node for input note data and build the entities - let new_public_notes = - fetch_public_note_details(store, rpc_api, &new_public_notes, &block_header).await?; + for note in return_notes.iter_mut() { + note.block_header_received(*block_header)?; + } - Ok(NoteUpdates::new( - new_public_notes, - vec![], - committed_tracked_input_notes, - committed_tracked_output_notes, - )) + Ok(return_notes) + } } -/// Queries the node for all received notes that aren't being locally tracked in the client. -/// -/// The client can receive metadata for private notes that it's not tracking. In this case, -/// notes are ignored for now as they become useless until details are imported. -async fn fetch_public_note_details( +pub async fn on_note_received( store: Arc, - rpc_api: Arc, - query_notes: &[NoteId], - block_header: &BlockHeader, -) -> Result, ClientError> { - if query_notes.is_empty() { - return Ok(vec![]); + committed_note: CommittedNote, + block_header: BlockHeader, +) -> Result<(NoteUpdates, Vec), ClientError> { + let inclusion_proof = NoteInclusionProof::new( + block_header.block_num(), + committed_note.note_index(), + committed_note.merkle_path().clone(), + )?; + + let mut updated_input_notes = vec![]; + let mut updated_output_notes = vec![]; + let mut new_note_ids = vec![]; + + if let Some(mut input_note_record) = store + .get_input_notes(NoteFilter::List(vec![*committed_note.note_id()])) + .await? + .pop() + { + // The note belongs to our locally tracked set of input notes + + let inclusion_proof_received = input_note_record + .inclusion_proof_received(inclusion_proof.clone(), committed_note.metadata())?; + let block_header_received = input_note_record.block_header_received(block_header)?; + + if inclusion_proof_received || block_header_received { + updated_input_notes.push(input_note_record); + } } - info!("Getting note details for notes that are not being tracked."); - let mut return_notes = rpc_api - .get_public_note_records(query_notes, store.get_current_timestamp()) - .await?; + if let Some(mut output_note_record) = store + .get_output_notes(NoteFilter::List(vec![*committed_note.note_id()])) + .await? + .pop() + { + // The note belongs to our locally tracked set of output notes - for note in return_notes.iter_mut() { - note.block_header_received(*block_header)?; + if output_note_record.inclusion_proof_received(inclusion_proof.clone())? { + updated_output_notes.push(output_note_record); + } } - Ok(return_notes) + if updated_input_notes.is_empty() && updated_output_notes.is_empty() { + // The note is public and we are not tracking it, push to the list of IDs to query + new_note_ids.push(*committed_note.note_id()); + } + + Ok(( + NoteUpdates::new(vec![], vec![], updated_input_notes, updated_output_notes), + new_note_ids, + )) } -/// Returns the [NoteUpdates] containing consumed input/output notes and a list of IDs of the -/// transactions that were discarded. -pub async fn consumed_note_updates( +pub async fn on_transaction_committed( store: Arc, - nullifiers: Vec, - committed_transactions: Vec, + transaction_update: TransactionUpdate, ) -> Result<(NoteUpdates, TransactionUpdates), ClientError> { - let nullifier_filter = NoteFilter::Nullifiers( - nullifiers.iter().map(|nullifier_update| nullifier_update.nullifier).collect(), - ); - - let mut consumed_input_notes: BTreeMap = store - .get_input_notes(nullifier_filter.clone()) - .await? + // TODO: This could be improved if we add a filter to get only notes that are being processed by + // a specific transaction + let processing_notes = store.get_input_notes(NoteFilter::Processing).await?; + let consumed_input_notes: Vec = processing_notes .into_iter() - .map(|n| (n.nullifier(), n)) - .collect(); - - let mut consumed_output_notes: BTreeMap = store - .get_output_notes(nullifier_filter) - .await? - .into_iter() - .map(|n| { - ( - n.nullifier() - .expect("Output notes returned by this query should have nullifiers"), - n, - ) + .filter(|note_record| { + note_record.consumer_transaction_id() == Some(&transaction_update.transaction_id) }) .collect(); - let mut consumed_tracked_input_notes = vec![]; - let mut consumed_tracked_output_notes = vec![]; - - // Committed transactions - for transaction_update in committed_transactions.iter() { - let transaction_nullifiers: Vec = consumed_input_notes - .iter() - .filter_map(|(nullifier, note_record)| { - if note_record.is_processing() - && note_record.consumer_transaction_id() - == Some(&transaction_update.transaction_id) - { - Some(nullifier) - } else { - None - } - }) - .cloned() - .collect(); + let consumed_output_notes = store + .get_output_notes(NoteFilter::Nullifiers( + consumed_input_notes.iter().map(|n| n.nullifier()).collect(), + )) + .await?; - for nullifier in transaction_nullifiers { - if let Some(mut input_note_record) = consumed_input_notes.remove(&nullifier) { - if input_note_record.transaction_committed( - transaction_update.transaction_id, - transaction_update.block_num, - )? { - consumed_tracked_input_notes.push(input_note_record); - } - } + let mut updated_input_notes = vec![]; + let mut updated_output_notes = vec![]; + for mut input_note_record in consumed_input_notes { + if input_note_record.transaction_committed( + transaction_update.transaction_id, + transaction_update.block_num, + )? { + updated_input_notes.push(input_note_record); } } - // Nullified notes - let mut discarded_transactions = vec![]; - for nullifier_update in nullifiers { - let nullifier = nullifier_update.nullifier; - let block_num = nullifier_update.block_num; - - if let Some(mut input_note_record) = consumed_input_notes.remove(&nullifier) { - if input_note_record.is_processing() { - discarded_transactions.push( - *input_note_record - .consumer_transaction_id() - .expect("Processing note should have consumer transaction id"), - ); - } - - if input_note_record.consumed_externally(nullifier, block_num)? { - consumed_tracked_input_notes.push(input_note_record); - } - } - - if let Some(mut output_note_record) = consumed_output_notes.remove(&nullifier) { - if output_note_record.nullifier_received(nullifier, block_num)? { - consumed_tracked_output_notes.push(output_note_record); - } + for mut output_note_record in consumed_output_notes { + // SAFETY: Output notes were queried from a nullifier list and should have a nullifier + let nullifier = output_note_record.nullifier().unwrap(); + if output_note_record.nullifier_received(nullifier, transaction_update.block_num)? { + updated_output_notes.push(output_note_record); } } Ok(( - NoteUpdates::new( - vec![], - vec![], - consumed_tracked_input_notes, - consumed_tracked_output_notes, - ), - TransactionUpdates::new(committed_transactions, discarded_transactions), + NoteUpdates::new(vec![], vec![], updated_input_notes, updated_output_notes), + TransactionUpdates::new(vec![transaction_update], vec![]), )) } -/// Compares the state of tracked accounts with the updates received from the node and returns -/// the accounts that need to be updated. -/// -/// When a mismatch is detected, two scenarios are possible: -/// * If the account is public, the component will request the node for the updated account details. -/// * If the account is private it will be marked as mismatched and the client will need to handle -/// it (it could be a stale account state or a reason to lock the account). -pub async fn account_state_sync( +pub async fn on_nullifier_received( store: Arc, - rpc_api: Arc, - account_hash_updates: Vec<(AccountId, Digest)>, -) -> Result { - let (public_accounts, private_accounts): (Vec<_>, Vec<_>) = store - .get_account_headers() - .await? - .into_iter() - .map(|(acc, _)| acc) - .partition(|acc| acc.id().is_public()); - - let updated_public_accounts = - get_updated_public_accounts(rpc_api, &account_hash_updates, &public_accounts).await?; + nullifier_update: NullifierUpdate, +) -> Result<(NoteUpdates, TransactionUpdates), ClientError> { + let mut discarded_transactions = vec![]; + let mut updated_input_notes = vec![]; + let mut updated_output_notes = vec![]; - let mismatched_private_accounts = account_hash_updates - .iter() - .filter(|(new_id, new_hash)| { - private_accounts - .iter() - .any(|acc| acc.id() == *new_id && acc.hash() != *new_hash) - }) - .cloned() - .collect::>(); + if let Some(mut input_note_record) = store + .get_input_notes(NoteFilter::Nullifiers(vec![nullifier_update.nullifier])) + .await? + .pop() + { + if input_note_record.is_processing() { + discarded_transactions.push( + *input_note_record + .consumer_transaction_id() + .expect("Processing note should have consumer transaction id"), + ); + } - Ok(AccountUpdates::new(updated_public_accounts, mismatched_private_accounts)) -} + if input_note_record + .consumed_externally(nullifier_update.nullifier, nullifier_update.block_num)? + { + updated_input_notes.push(input_note_record); + } + } -async fn get_updated_public_accounts( - rpc_api: Arc, - account_updates: &[(AccountId, Digest)], - current_public_accounts: &[AccountHeader], -) -> Result, ClientError> { - let mut mismatched_public_accounts = vec![]; - - for (id, hash) in account_updates { - // check if this updated account is tracked by the client - if let Some(account) = current_public_accounts - .iter() - .find(|acc| *id == acc.id() && *hash != acc.hash()) + if let Some(mut output_note_record) = store + .get_output_notes(NoteFilter::Nullifiers(vec![nullifier_update.nullifier])) + .await? + .pop() + { + if output_note_record + .nullifier_received(nullifier_update.nullifier, nullifier_update.block_num)? { - mismatched_public_accounts.push(account); + updated_output_notes.push(output_note_record); } } - rpc_api - .get_updated_public_accounts(&mismatched_public_accounts) - .await - .map_err(ClientError::RpcError) + Ok(( + NoteUpdates::new(vec![], vec![], updated_input_notes, updated_output_notes), + TransactionUpdates::new(vec![], discarded_transactions), + )) } diff --git a/crates/rust-client/src/transactions/mod.rs b/crates/rust-client/src/transactions/mod.rs index 78c9fb886..05fe34259 100644 --- a/crates/rust-client/src/transactions/mod.rs +++ b/crates/rust-client/src/transactions/mod.rs @@ -75,6 +75,18 @@ impl TransactionUpdates { } } + pub fn new_empty() -> Self { + Self { + committed_transactions: vec![], + discarded_transactions: vec![], + } + } + + pub fn combine_with(&mut self, other: Self) { + self.committed_transactions.extend(other.committed_transactions); + self.discarded_transactions.extend(other.discarded_transactions); + } + pub fn committed_transactions(&self) -> &[TransactionUpdate] { &self.committed_transactions } diff --git a/tests/integration/main.rs b/tests/integration/main.rs index 8e405b485..282448f0d 100644 --- a/tests/integration/main.rs +++ b/tests/integration/main.rs @@ -309,6 +309,7 @@ async fn test_p2idr_transfer_consumed_by_target() { // Check that the note is consumed by the target account let input_note = client.get_input_note(note.id()).await.unwrap(); + println!("input note state: {:?}", input_note.state()); assert!(matches!(input_note.state(), InputNoteState::ConsumedAuthenticatedLocal { .. })); if let InputNoteState::ConsumedAuthenticatedLocal(ConsumedAuthenticatedLocalNoteState { submission_data, From 599a801578c608667fccd5d99c7b8009f48264ad Mon Sep 17 00:00:00 2001 From: tomyrd Date: Wed, 8 Jan 2025 23:03:20 -0300 Subject: [PATCH 12/34] chore: improve code structure and documentation --- crates/rust-client/src/sync/state_sync.rs | 214 +++++++++++++++------- 1 file changed, 143 insertions(+), 71 deletions(-) diff --git a/crates/rust-client/src/sync/state_sync.rs b/crates/rust-client/src/sync/state_sync.rs index 994de6686..631aead5a 100644 --- a/crates/rust-client/src/sync/state_sync.rs +++ b/crates/rust-client/src/sync/state_sync.rs @@ -25,6 +25,9 @@ use crate::{ ClientError, }; +// STATE SYNC UPDATE +// ================================================================================================ + /// Contains all information needed to apply the update in the store after syncing with the node. pub struct StateSyncUpdate { /// The new block header, returned as part of the @@ -89,6 +92,13 @@ impl SyncStatus { } } +// SYNC CALLBACKS +// ================================================================================================ + +/// Callback to be executed when a new note inclusion is received in the sync response. It receives +/// the committed note received from the node and the block header in which the note was included. +/// It returns the note updates that should be applied to the store and a list of public note IDs +/// that should be queried from the node and start being tracked. type OnNoteReceived = Box< dyn Fn( CommittedNote, @@ -96,6 +106,9 @@ type OnNoteReceived = Box< ) -> Pin), ClientError>>>>, >; +/// Callback to be executed when a transaction is marked committed in the sync response. It receives +/// the transaction update received from the node. It returns the note updates and transaction +/// updates that should be applied to the store as a result of the transaction being committed. type OnTransactionCommitted = Box< dyn Fn( TransactionUpdate, @@ -103,6 +116,11 @@ type OnTransactionCommitted = Box< -> Pin>>>, >; +/// Callback to be executed when a nullifier is received in the sync response. If a note was +/// consumed by a committed transaction provided in the [OnTransactionCommitted] callback, its +/// nullifier will not be passed to this callback. It receives the nullifier update received from +/// the node. It returns the note updates and transaction updates that should be applied to the +/// store as a result of the nullifier being received. type OnNullifierReceived = Box< dyn Fn( NullifierUpdate, @@ -110,15 +128,22 @@ type OnNullifierReceived = Box< -> Pin>>>, >; +// STATE SYNC +// ================================================================================================ + /// The state sync components encompasses the client's sync logic. /// /// When created it receives the current state of the client's relevant elements (block, accounts, /// notes, etc). It is then used to requset updates from the node and apply them to the relevant /// elements. The updates are then returned and can be applied to the store to persist the changes. pub struct StateSync { + /// The RPC client used to communicate with the node. rpc_api: Arc, + /// Callback to be executed when a new note inclusion is received. on_note_received: OnNoteReceived, + /// Callback to be executed when a transaction is committed. on_committed_transaction: OnTransactionCommitted, + /// Callback to be executed when a nullifier is received. on_nullifier_received: OnNullifierReceived, } @@ -127,17 +152,10 @@ impl StateSync { /// /// # Arguments /// - /// * `rpc_api` - The RPC client to use to communicate with the node. - /// * `current_block` - The latest block header tracked by the client. - /// * `current_block_has_relevant_notes` - A flag indicating if the current block has notes that - /// are relevant to the client. - /// * `accounts` - The headers of accounts tracked by the client. - /// * `note_tags` - The note tags to be used in the sync state request. - /// * `unspent_input_notes` - The input notes that haven't been yet consumed and may be changed - /// in the sync process. - /// * `unspent_output_notes` - The output notes that haven't been yet consumed and may be - /// changed in the sync process. - /// * `current_partial_mmr` - The current partial MMR of the client. + /// * `rpc_api` - The RPC client used to communicate with the node. + /// * `on_note_received` - A callback to be executed when a new note inclusion is received. + /// * `on_committed_transaction` - A callback to be executed when a transaction is committed. + /// * `on_nullifier_received` - A callback to be executed when a nullifier is received. pub fn new( rpc_api: Arc, on_note_received: OnNoteReceived, @@ -161,6 +179,15 @@ impl StateSync { /// Wheter or not the client has reached the chain tip is indicated by the returned /// [SyncStatus] variant. `None` is returned if the client is already synced with the chain tip /// and there are no new changes. + /// + /// # Arguments + /// * `current_block` - The latest tracked block header. + /// * `current_block_has_relevant_notes` - A flag indicating if the current block has notes that + /// are relevant to the client. + /// * `current_partial_mmr` - The current partial MMR. + /// * `accounts` - The headers of tracked accounts. + /// * `note_tags` - The note tags to be used in the sync state request. + /// * `unspent_nullifiers` - The nullifiers of tracked notes that haven't been consumed. pub async fn sync_state_step( &self, current_block: BlockHeader, @@ -180,7 +207,7 @@ impl StateSync { let nullifiers_tags: Vec = unspent_nullifiers.iter().map(get_nullifier_prefix).collect(); - let mut response = self + let response = self .rpc_api .sync_state(current_block_num, &account_ids, ¬e_tags, &nullifiers_tags) .await?; @@ -193,58 +220,14 @@ impl StateSync { let account_updates = self.account_state_sync(&accounts, &response.account_hash_updates).await?; - let mut note_updates = NoteUpdates::new_empty(); - - let mut public_note_ids = vec![]; - for committed_note in response.note_inclusions { - let (new_note_update, new_note_ids) = - (self.on_note_received)(committed_note, response.block_header).await?; - note_updates.combine_with(new_note_update); - public_note_ids.extend(new_note_ids); - } - - let new_public_notes = - self.fetch_public_note_details(&public_note_ids, &response.block_header).await?; - - note_updates.combine_with(NoteUpdates::new(new_public_notes, vec![], vec![], vec![])); - - // We can remove tags from notes that got committed - let tags_to_remove = note_updates - .updated_input_notes() - .iter() - .filter(|note| note.is_committed()) - .map(|note| { - NoteTagRecord::with_note_source( - note.metadata().expect("Committed note should have metadata").tag(), - note.id(), - ) - }) - .collect(); - - let mut transaction_updates = TransactionUpdates::new_empty(); - - for transaction_update in response.transactions { - let (new_note_update, new_transaction_update) = - (self.on_committed_transaction)(transaction_update).await?; - - // Remove nullifiers if they were consumed by the transaction - response.nullifiers.retain(|nullifier| { - !new_note_update - .updated_input_notes() - .iter() - .any(|note| note.nullifier() == nullifier.nullifier) - }); - - note_updates.combine_with(new_note_update); - transaction_updates.combine_with(new_transaction_update); - } - - for nullifier_update in response.nullifiers { - let (new_note_update, new_transaction_update) = - (self.on_nullifier_received)(nullifier_update).await?; - note_updates.combine_with(new_note_update); - transaction_updates.combine_with(new_transaction_update); - } + let (note_updates, transaction_updates, tags_to_remove) = self + .note_state_sync( + response.note_inclusions, + response.transactions, + response.nullifiers, + response.block_header, + ) + .await?; let (new_mmr_peaks, new_authentication_nodes) = self .apply_mmr_changes( @@ -284,12 +267,13 @@ impl StateSync { mut current_partial_mmr: PartialMmr, mmr_delta: MmrDelta, ) -> Result<(MmrPeaks, Vec<(InOrderIndex, Digest)>), ClientError> { - // First, apply curent_block to the Mmr + // First, apply curent_block to the MMR. This is needed as the MMR delta received from the + // node doesn't contain the request block itself. let new_authentication_nodes = current_partial_mmr .add(current_block.hash(), current_block_has_relevant_notes) .into_iter(); - // Apply the Mmr delta to bring Mmr to forest equal to chain tip + // Apply the MMR delta to bring MMR to forest equal to chain tip let new_authentication_nodes: Vec<(InOrderIndex, Digest)> = current_partial_mmr .apply(mmr_delta) .map_err(StoreError::MmrError)? @@ -300,9 +284,6 @@ impl StateSync { Ok((current_partial_mmr.peaks(), new_authentication_nodes)) } - // HELPERS - // -------------------------------------------------------------------------------------------- - /// Compares the state of tracked accounts with the updates received from the node and returns /// the accounts that need to be updated. /// @@ -335,6 +316,8 @@ impl StateSync { Ok(AccountUpdates::new(updated_public_accounts, mismatched_private_accounts)) } + /// Queries the node for the latest state of the public accounts that don't match the current + /// state of the client. async fn get_updated_public_accounts( &self, account_updates: &[(AccountId, Digest)], @@ -358,6 +341,85 @@ impl StateSync { .map_err(ClientError::RpcError) } + /// Applies the changes received from the sync response to the notes and transactions tracked + /// by the client. It returns the updates that should be applied to the store. + /// + /// This method uses the callbacks provided to the [StateSync] component to apply the changes. + /// + /// The note changes might include: + /// * New notes that we received from the node and might be relevant to the client. + /// * Tracked expected notes that were committed in the block. + /// * Tracked notes that were being processed by a transaction that got committed. + /// * Tracked notes that were nullified by an external transaction. + /// + /// The transaction changes might include: + /// * Transactions that were committed in the block. Some of these might me tracked by the + /// client and need to be marked as committed. + /// * Local tracked transactions that were discarded because the notes that they were processing + /// were nullified by an another transaction. + async fn note_state_sync( + &self, + note_inclusions: Vec, + transactions: Vec, + mut nullifiers: Vec, + block_header: BlockHeader, + ) -> Result<(NoteUpdates, TransactionUpdates, Vec), ClientError> { + let mut note_updates = NoteUpdates::new_empty(); + let mut public_note_ids = vec![]; + + for committed_note in note_inclusions { + let (new_note_update, new_note_ids) = + (self.on_note_received)(committed_note, block_header).await?; + note_updates.combine_with(new_note_update); + public_note_ids.extend(new_note_ids); + } + + let new_public_notes = + self.fetch_public_note_details(&public_note_ids, &block_header).await?; + + note_updates.combine_with(NoteUpdates::new(new_public_notes, vec![], vec![], vec![])); + + // We can remove tags from notes that got committed + let tags_to_remove = note_updates + .updated_input_notes() + .iter() + .filter(|note| note.is_committed()) + .map(|note| { + NoteTagRecord::with_note_source( + note.metadata().expect("Committed note should have metadata").tag(), + note.id(), + ) + }) + .collect(); + + let mut transaction_updates = TransactionUpdates::new_empty(); + + for transaction_update in transactions { + let (new_note_update, new_transaction_update) = + (self.on_committed_transaction)(transaction_update).await?; + + // Remove nullifiers if they were consumed by the transaction + nullifiers.retain(|nullifier| { + !new_note_update + .updated_input_notes() + .iter() + .any(|note| note.nullifier() == nullifier.nullifier) + }); + + note_updates.combine_with(new_note_update); + transaction_updates.combine_with(new_transaction_update); + } + + for nullifier_update in nullifiers { + let (new_note_update, new_transaction_update) = + (self.on_nullifier_received)(nullifier_update).await?; + note_updates.combine_with(new_note_update); + transaction_updates.combine_with(new_transaction_update); + } + + Ok((note_updates, transaction_updates, tags_to_remove)) + } + /// Queries the node for all received notes that aren't being locally tracked in the client. /// /// The client can receive metadata for private notes that it's not tracking. In this case, @@ -382,6 +444,12 @@ impl StateSync { } } +// DEFAULT CALLBACK IMPLEMENTATIONS +// ================================================================================================ + +/// Default implementation of the [OnNoteReceived] callback. It queries the store for the committed +/// note and updates the note records accordingly. If the note is not being tracked, it returns the +/// note ID to be queried from the node so it can be queried from the node and tracked. pub async fn on_note_received( store: Arc, committed_note: CommittedNote, @@ -403,7 +471,6 @@ pub async fn on_note_received( .pop() { // The note belongs to our locally tracked set of input notes - let inclusion_proof_received = input_note_record .inclusion_proof_received(inclusion_proof.clone(), committed_note.metadata())?; let block_header_received = input_note_record.block_header_received(block_header)?; @@ -419,7 +486,6 @@ pub async fn on_note_received( .pop() { // The note belongs to our locally tracked set of output notes - if output_note_record.inclusion_proof_received(inclusion_proof.clone())? { updated_output_notes.push(output_note_record); } @@ -436,6 +502,9 @@ pub async fn on_note_received( )) } +/// Default implementation of the [OnTransactionCommitted] callback. It queries the store for the +/// input notes that were consumed by the transaction and updates the note records accordingly. It +/// also returns the committed transaction update to be applied to the store. pub async fn on_transaction_committed( store: Arc, transaction_update: TransactionUpdate, @@ -481,6 +550,9 @@ pub async fn on_transaction_committed( )) } +/// Default implementation of the [OnNullifierReceived] callback. It queries the store for the notes +/// that match the nullifier and updates the note records accordingly. It also returns the +/// transactions that should be discarded as they weren't committed when the nullifier was received. pub async fn on_nullifier_received( store: Arc, nullifier_update: NullifierUpdate, From 2645e7abe35e1f3771660d10c0576eef06ab0f88 Mon Sep 17 00:00:00 2001 From: tomyrd Date: Wed, 8 Jan 2025 23:26:51 -0300 Subject: [PATCH 13/34] fix: update web store --- crates/rust-client/src/accounts.rs | 4 +-- crates/rust-client/src/rpc/mod.rs | 16 +++++----- .../rust-client/src/rpc/tonic_client/mod.rs | 6 ++-- crates/rust-client/src/store/mod.rs | 2 +- crates/rust-client/src/store/web_store/mod.rs | 8 +++-- .../src/store/web_store/sync/mod.rs | 29 ++++++++++++------- crates/rust-client/src/sync/mod.rs | 3 +- crates/rust-client/src/sync/state_sync.rs | 1 - crates/web-client/test/mocha.global.setup.mjs | 6 +++- tests/integration/main.rs | 1 - 10 files changed, 45 insertions(+), 31 deletions(-) diff --git a/crates/rust-client/src/accounts.rs b/crates/rust-client/src/accounts.rs index 9dc9ab7a7..87a92c6e1 100644 --- a/crates/rust-client/src/accounts.rs +++ b/crates/rust-client/src/accounts.rs @@ -182,12 +182,12 @@ impl AccountUpdates { } } - /// Returns updated public accounts. + /// Returns the updated public accounts. pub fn updated_public_accounts(&self) -> &[Account] { &self.updated_public_accounts } - /// Returns mismatched private accounts. + /// Returns the mismatched private accounts. pub fn mismatched_private_accounts(&self) -> &[(AccountId, Digest)] { &self.mismatched_private_accounts } diff --git a/crates/rust-client/src/rpc/mod.rs b/crates/rust-client/src/rpc/mod.rs index c34614bdf..438f246b8 100644 --- a/crates/rust-client/src/rpc/mod.rs +++ b/crates/rust-client/src/rpc/mod.rs @@ -73,7 +73,7 @@ pub trait NodeRpcClient { /// Given a block number, fetches the block header corresponding to that height from the node /// using the `/GetBlockHeaderByNumber` endpoint. /// If `include_mmr_proof` is set to true and the function returns an `Ok`, the second value - /// of the return tuple should always be Some(MmrProof) + /// of the return tuple should always be Some(MmrProof). /// /// When `None` is provided, returns info regarding the latest block. async fn get_block_header_by_number( @@ -82,7 +82,7 @@ pub trait NodeRpcClient { include_mmr_proof: bool, ) -> Result<(BlockHeader, Option), RpcError>; - /// Fetches note-related data for a list of [NoteId] using the `/GetNotesById` rpc endpoint + /// Fetches note-related data for a list of [NoteId] using the `/GetNotesById` rpc endpoint. /// /// For any NoteType::Private note, the return data is only the /// [miden_objects::notes::NoteMetadata], whereas for NoteType::Onchain notes, the return @@ -90,15 +90,15 @@ pub trait NodeRpcClient { async fn get_notes_by_id(&self, note_ids: &[NoteId]) -> Result, RpcError>; /// Fetches info from the node necessary to perform a state sync using the - /// `/SyncState` RPC endpoint + /// `/SyncState` RPC endpoint. /// /// - `block_num` is the last block number known by the client. The returned [StateSyncInfo] /// should contain data starting from the next block, until the first block which contains a /// note of matching the requested tag, or the chain tip if there are no notes. - /// - `account_ids` is a list of account ids and determines the accounts the client is + /// - `account_ids` is a list of account IDs and determines the accounts the client is /// interested in and should receive account updates of. /// - `note_tags` is a list of tags used to filter the notes the client is interested in, which - /// serves as a "note group" filter. Notice that you can't filter by a specific note id + /// serves as a "note group" filter. Notice that you can't filter by a specific note ID. /// - `nullifiers_tags` similar to `note_tags`, is a list of tags used to filter the nullifiers /// corresponding to some notes the client is interested in. async fn sync_state( @@ -110,9 +110,9 @@ pub trait NodeRpcClient { ) -> Result; /// Fetches the current state of an account from the node using the `/GetAccountDetails` RPC - /// endpoint + /// endpoint. /// - /// - `account_id` is the id of the wanted account. + /// - `account_id` is the ID of the wanted account. async fn get_account_update(&self, account_id: AccountId) -> Result; async fn sync_notes( @@ -136,7 +136,7 @@ pub trait NodeRpcClient { include_headers: bool, ) -> Result; - /// Fetches the commit height where the nullifier was consumed. If the nullifier is not found, + /// Fetches the commit height where the nullifier was consumed. If the nullifier isn't found, /// then `None` is returned. /// /// The default implementation of this method uses [NodeRpcClient::check_nullifiers_by_prefix]. diff --git a/crates/rust-client/src/rpc/tonic_client/mod.rs b/crates/rust-client/src/rpc/tonic_client/mod.rs index 4008db265..a5228cc88 100644 --- a/crates/rust-client/src/rpc/tonic_client/mod.rs +++ b/crates/rust-client/src/rpc/tonic_client/mod.rs @@ -42,7 +42,7 @@ pub struct TonicRpcClient { impl TonicRpcClient { /// Returns a new instance of [TonicRpcClient] that'll do calls to the provided [Endpoint] with - /// the given timeout in milliseconds + /// the given timeout in milliseconds. pub async fn new(endpoint: Endpoint, timeout_ms: u64) -> Result { let endpoint = tonic::transport::Endpoint::try_from(endpoint.to_string()) .map_err(|err| RpcError::ConnectionError(err.to_string()))? @@ -210,10 +210,10 @@ impl NodeRpcClient for TonicRpcClient { /// /// This function will return an error if: /// - /// - There was an error sending the request to the node + /// - There was an error sending the request to the node. /// - The answer had a `None` for one of the expected fields (account, summary, account_hash, /// details). - /// - There is an error during [Account] deserialization + /// - There is an error during [Account] deserialization. async fn get_account_update(&self, account_id: AccountId) -> Result { let request = GetAccountDetailsRequest { account_id: Some(account_id.into()) }; diff --git a/crates/rust-client/src/store/mod.rs b/crates/rust-client/src/store/mod.rs index b4733ac07..515c52e41 100644 --- a/crates/rust-client/src/store/mod.rs +++ b/crates/rust-client/src/store/mod.rs @@ -324,7 +324,7 @@ pub trait Store: Send + Sync { /// - Updating the corresponding tracked input/output notes. /// - Removing note tags that are no longer relevant. /// - Updating transactions in the store, marking as `committed` or `discarded`. - /// - Applies the MMR delta to the store. + /// - Storing new MMR authentication nodes. /// - Updating the tracked on-chain accounts. async fn apply_state_sync_step( &self, diff --git a/crates/rust-client/src/store/web_store/mod.rs b/crates/rust-client/src/store/web_store/mod.rs index ee4498413..68a878b84 100644 --- a/crates/rust-client/src/store/web_store/mod.rs +++ b/crates/rust-client/src/store/web_store/mod.rs @@ -65,8 +65,12 @@ impl Store for WebStore { self.get_sync_height().await } - async fn apply_state_sync(&self, state_sync_update: StateSyncUpdate) -> Result<(), StoreError> { - self.apply_state_sync(state_sync_update).await + async fn apply_state_sync_step( + &self, + state_sync_update: StateSyncUpdate, + block_has_relevant_notes: bool, + ) -> Result<(), StoreError> { + self.apply_state_sync_step(state_sync_update, block_has_relevant_notes).await } // TRANSACTIONS diff --git a/crates/rust-client/src/store/web_store/sync/mod.rs b/crates/rust-client/src/store/web_store/sync/mod.rs index 0f4643ced..6cde5d501 100644 --- a/crates/rust-client/src/store/web_store/sync/mod.rs +++ b/crates/rust-client/src/store/web_store/sync/mod.rs @@ -97,20 +97,18 @@ impl WebStore { Ok(removed_tags) } - pub(super) async fn apply_state_sync( + pub(super) async fn apply_state_sync_step( &self, state_sync_update: StateSyncUpdate, + block_has_relevant_notes: bool, ) -> Result<(), StoreError> { let StateSyncUpdate { block_header, note_updates, - transactions_to_commit: committed_transactions, + transaction_updates, //TODO: Add support for discarded transactions in web store new_mmr_peaks, new_authentication_nodes, - updated_accounts, - block_has_relevant_notes, - transactions_to_discard: _transactions_to_discard, /* TODO: Add support for discarded - * transactions in web store */ + account_updates, tags_to_remove, } = state_sync_update; @@ -147,22 +145,33 @@ impl WebStore { .collect(); // Serialize data for updating committed transactions - let transactions_to_commit_block_nums_as_str = committed_transactions + let transactions_to_commit_block_nums_as_str = transaction_updates + .committed_transactions() .iter() .map(|tx_update| tx_update.block_num.to_string()) .collect(); - let transactions_to_commit_as_str: Vec = committed_transactions + let transactions_to_commit_as_str: Vec = transaction_updates + .committed_transactions() .iter() .map(|tx_update| tx_update.transaction_id.to_string()) .collect(); // TODO: LOP INTO idxdb_apply_state_sync call // Update public accounts on the db that have been updated onchain - for account in updated_accounts.updated_public_accounts() { + for account in account_updates.updated_public_accounts() { update_account(&account.clone()).await.unwrap(); } - for (account_id, _) in updated_accounts.mismatched_offchain_accounts() { + for (account_id, digest) in account_updates.mismatched_private_accounts() { + // Mismatched digests may be due to stale network data. If the mismatched digest is + // tracked in the db and corresponds to the mismatched account, it means we + // got a past update and shouldn't lock the account. + if let Some(account) = self.get_account_header_by_hash(*digest).await? { + if account.id() == *account_id { + continue; + } + } + lock_account(account_id).await.unwrap(); } diff --git a/crates/rust-client/src/sync/mod.rs b/crates/rust-client/src/sync/mod.rs index ebf216454..8983709e4 100644 --- a/crates/rust-client/src/sync/mod.rs +++ b/crates/rust-client/src/sync/mod.rs @@ -1,9 +1,8 @@ //! Provides the client APIs for synchronizing the client's local state with the Miden //! rollup network. It ensures that the client maintains a valid, up-to-date view of the chain. -use alloc::vec::Vec; +use alloc::{boxed::Box, vec::Vec}; use core::cmp::max; -use std::boxed::Box; use miden_objects::{ accounts::AccountId, diff --git a/crates/rust-client/src/sync/state_sync.rs b/crates/rust-client/src/sync/state_sync.rs index 631aead5a..2713b1f41 100644 --- a/crates/rust-client/src/sync/state_sync.rs +++ b/crates/rust-client/src/sync/state_sync.rs @@ -1,6 +1,5 @@ use alloc::{boxed::Box, sync::Arc, vec::Vec}; use core::{future::Future, pin::Pin}; -use std::vec; use miden_objects::{ accounts::{Account, AccountHeader, AccountId}, diff --git a/crates/web-client/test/mocha.global.setup.mjs b/crates/web-client/test/mocha.global.setup.mjs index 2d0e984fe..2ba05826c 100644 --- a/crates/web-client/test/mocha.global.setup.mjs +++ b/crates/web-client/test/mocha.global.setup.mjs @@ -30,7 +30,11 @@ before(async () => { shell: process.platform == "win32", }); - browser = await puppeteer.launch({ headless: true, protocolTimeout: 360000 }); + browser = await puppeteer.launch({ + headless: true, + protocolTimeout: 360000, + args: ["--no-sandbox", "--disable-setuid-sandbox"], + }); testingPage = await browser.newPage(); await testingPage.goto(TEST_SERVER); diff --git a/tests/integration/main.rs b/tests/integration/main.rs index 282448f0d..8e405b485 100644 --- a/tests/integration/main.rs +++ b/tests/integration/main.rs @@ -309,7 +309,6 @@ async fn test_p2idr_transfer_consumed_by_target() { // Check that the note is consumed by the target account let input_note = client.get_input_note(note.id()).await.unwrap(); - println!("input note state: {:?}", input_note.state()); assert!(matches!(input_note.state(), InputNoteState::ConsumedAuthenticatedLocal { .. })); if let InputNoteState::ConsumedAuthenticatedLocal(ConsumedAuthenticatedLocalNoteState { submission_data, From 2f260076a45232117a6ca581318c8e3a1f66f778 Mon Sep 17 00:00:00 2001 From: tomyrd Date: Wed, 8 Jan 2025 23:37:21 -0300 Subject: [PATCH 14/34] chore: update CHANGELOG --- CHANGELOG.md | 1 + crates/rust-client/src/sync/mod.rs | 4 ++-- crates/rust-client/src/sync/state_sync.rs | 6 +++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ae20cc4d..9c11a1efa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ ### Changes +* [BREAKING] Refactored the sync process to use a new `SyncState` component (#650). * Refactored RPC functions and structs to improve code quality (#616). * [BREAKING] Added support for new two `Felt` account ID (#639). * [BREAKING] Removed unnecessary methods from `Client` (#631). diff --git a/crates/rust-client/src/sync/mod.rs b/crates/rust-client/src/sync/mod.rs index 8983709e4..d06e47c18 100644 --- a/crates/rust-client/src/sync/mod.rs +++ b/crates/rust-client/src/sync/mod.rs @@ -20,8 +20,8 @@ pub use tags::{NoteTagRecord, NoteTagSource}; mod state_sync; pub use state_sync::{ - on_note_received, on_nullifier_received, on_transaction_committed, StateSync, StateSyncUpdate, - SyncStatus, + on_note_received, on_nullifier_received, on_transaction_committed, OnNoteReceived, + OnNullifierReceived, OnTransactionCommitted, StateSync, StateSyncUpdate, SyncStatus, }; /// Contains stats about the sync operation. diff --git a/crates/rust-client/src/sync/state_sync.rs b/crates/rust-client/src/sync/state_sync.rs index 2713b1f41..58a9ef05d 100644 --- a/crates/rust-client/src/sync/state_sync.rs +++ b/crates/rust-client/src/sync/state_sync.rs @@ -98,7 +98,7 @@ impl SyncStatus { /// the committed note received from the node and the block header in which the note was included. /// It returns the note updates that should be applied to the store and a list of public note IDs /// that should be queried from the node and start being tracked. -type OnNoteReceived = Box< +pub type OnNoteReceived = Box< dyn Fn( CommittedNote, BlockHeader, @@ -108,7 +108,7 @@ type OnNoteReceived = Box< /// Callback to be executed when a transaction is marked committed in the sync response. It receives /// the transaction update received from the node. It returns the note updates and transaction /// updates that should be applied to the store as a result of the transaction being committed. -type OnTransactionCommitted = Box< +pub type OnTransactionCommitted = Box< dyn Fn( TransactionUpdate, ) @@ -120,7 +120,7 @@ type OnTransactionCommitted = Box< /// nullifier will not be passed to this callback. It receives the nullifier update received from /// the node. It returns the note updates and transaction updates that should be applied to the /// store as a result of the nullifier being received. -type OnNullifierReceived = Box< +pub type OnNullifierReceived = Box< dyn Fn( NullifierUpdate, ) From b524978c5bca7e26965f30332a01eede09a92d54 Mon Sep 17 00:00:00 2001 From: Ignacio Amigo Date: Tue, 14 Jan 2025 12:58:14 -0300 Subject: [PATCH 15/34] fix: Various stylistic fixes and refactors --- crates/rust-client/src/notes/mod.rs | 25 ++++++++-------- crates/rust-client/src/notes/note_screener.rs | 2 ++ crates/rust-client/src/store/mod.rs | 8 ++++- crates/rust-client/src/sync/block_headers.rs | 9 ++---- crates/rust-client/src/sync/mod.rs | 16 ++++------ crates/rust-client/src/sync/state_sync.rs | 30 ++++++++++++------- crates/rust-client/src/transactions/mod.rs | 28 ++++++++--------- 7 files changed, 64 insertions(+), 54 deletions(-) diff --git a/crates/rust-client/src/notes/mod.rs b/crates/rust-client/src/notes/mod.rs index ed933efd7..24307316c 100644 --- a/crates/rust-client/src/notes/mod.rs +++ b/crates/rust-client/src/notes/mod.rs @@ -179,6 +179,7 @@ pub async fn get_input_note_with_id_prefix( // ------------------------------------------------------------------------------------------------ /// Contains note changes to apply to the store. +#[derive(Clone, Debug, Default)] pub struct NoteUpdates { /// A list of new input notes. new_input_notes: Vec, @@ -206,17 +207,8 @@ impl NoteUpdates { } } - pub fn new_empty() -> Self { - Self { - new_input_notes: Vec::new(), - new_output_notes: Vec::new(), - updated_input_notes: Vec::new(), - updated_output_notes: Vec::new(), - } - } - /// Combines two [NoteUpdates] into a single one. - pub fn combine_with(&mut self, other: Self) { + pub fn extend(&mut self, other: Self) { self.new_input_notes.extend(other.new_input_notes); self.new_output_notes.extend(other.new_output_notes); self.updated_input_notes.extend(other.updated_input_notes); @@ -253,7 +245,16 @@ impl NoteUpdates { && self.new_output_notes.is_empty() } - /// Returns the IDs of all notes that have been committed. + /// Returns any note that has been committed into the chain in this update (either new or + /// already locally tracked) + pub fn committed_input_notes(&self) -> impl Iterator { + self.updated_input_notes + .iter() + .chain(self.new_input_notes.iter()) + .filter(|note| note.is_committed()) + } + + /// Returns the IDs of all notes that have been committed (previously locally tracked). pub fn committed_note_ids(&self) -> BTreeSet { let committed_output_note_ids = self .updated_output_notes @@ -268,7 +269,7 @@ impl NoteUpdates { BTreeSet::from_iter(committed_input_note_ids.chain(committed_output_note_ids)) } - /// Returns the IDs of all notes that have been consumed + /// Returns the IDs of all notes that have been consumed. pub fn consumed_note_ids(&self) -> BTreeSet { let consumed_output_note_ids = self .updated_output_notes diff --git a/crates/rust-client/src/notes/note_screener.rs b/crates/rust-client/src/notes/note_screener.rs index 0571c1b9e..c2226336f 100644 --- a/crates/rust-client/src/notes/note_screener.rs +++ b/crates/rust-client/src/notes/note_screener.rs @@ -57,6 +57,8 @@ impl NoteScreener { /// Does a fast check for known scripts (P2ID, P2IDR, SWAP). We're currently /// unable to execute notes that aren't committed so a slow check for other scripts is /// currently not available. + /// + /// If relevance can't be determined, the screener defaults to setting the note as consumable. pub async fn check_relevance( &self, note: &Note, diff --git a/crates/rust-client/src/store/mod.rs b/crates/rust-client/src/store/mod.rs index ad0965eea..882a26e0a 100644 --- a/crates/rust-client/src/store/mod.rs +++ b/crates/rust-client/src/store/mod.rs @@ -302,8 +302,14 @@ pub trait Store: Send + Sync { /// - Updating the corresponding tracked input/output notes. /// - Removing note tags that are no longer relevant. /// - Updating transactions in the store, marking as `committed` or `discarded`. + /// - In turn, validating private account's state transitions. If a private account's hash + /// locally does not match the `StateSyncUpdate` information, the account may be locked. /// - Storing new MMR authentication nodes. - /// - Updating the tracked on-chain accounts. + /// - Updating the tracked public accounts. + /// + /// A [StateSyncUpdate] corresponds to an update from a single `SyncState` RPC call. For the + /// client to be up to date against the current chain tip, multiple calls may be performed, and + /// so multiple store updates could happen sequentially. async fn apply_state_sync_step( &self, state_sync_update: StateSyncUpdate, diff --git a/crates/rust-client/src/sync/block_headers.rs b/crates/rust-client/src/sync/block_headers.rs index 3b633246c..edc10ae70 100644 --- a/crates/rust-client/src/sync/block_headers.rs +++ b/crates/rust-client/src/sync/block_headers.rs @@ -82,12 +82,9 @@ impl Client { let note_screener = NoteScreener::new(self.store.clone()); // Find all relevant Input Notes using the note checker - for input_note in note_updates - .updated_input_notes() - .iter() - .chain(note_updates.new_input_notes().iter()) - .filter(|note| note.is_committed()) - { + for input_note in note_updates.committed_input_notes() { + // TODO: Map the below error into a better representation (ie, we expected to be able + // to convert here) if !note_screener .check_relevance(&input_note.try_into().map_err(ClientError::NoteRecordError)?) .await? diff --git a/crates/rust-client/src/sync/mod.rs b/crates/rust-client/src/sync/mod.rs index 51edaf8c9..3651ee702 100644 --- a/crates/rust-client/src/sync/mod.rs +++ b/crates/rust-client/src/sync/mod.rs @@ -109,8 +109,8 @@ impl Client { self.store.get_sync_height().await.map_err(|err| err.into()) } - /// Syncs the client's state with the current state of the Miden network. Returns the block - /// number the client has been synced to. + /// Syncs the client's state with the current state of the Miden network and returns a + /// [SyncSummary] corresponding to the local state update. /// /// The sync process is done in multiple steps: /// 1. A request is sent to the node to get the state updates. This request includes tracked @@ -124,7 +124,7 @@ impl Client { /// can be consumed by accounts the client is tracking (this is checked by the /// [crate::notes::NoteScreener]) /// 5. Transactions are updated with their new states. - /// 6. Tracked public accounts are updated and off-chain accounts are validated against the node + /// 6. Tracked public accounts are updated and private accounts are validated against the node /// state. /// 7. The MMR is updated with the new peaks and authentication nodes. /// 8. All updates are applied to the store to be persisted. @@ -190,13 +190,9 @@ impl Client { ) .await?; - let (is_last_block, state_sync_update) = if let Some(status) = status { - ( - matches!(status, SyncStatus::SyncedToLastBlock(_)), - status.into_state_sync_update(), - ) - } else { - break; + let (is_last_block, state_sync_update): (bool, StateSyncUpdate) = match status { + Some(s) => (s.is_last_block(), s.into()), + None => break, }; let sync_summary: SyncSummary = (&state_sync_update).into(); diff --git a/crates/rust-client/src/sync/state_sync.rs b/crates/rust-client/src/sync/state_sync.rs index 58a9ef05d..5d80ab3b1 100644 --- a/crates/rust-client/src/sync/state_sync.rs +++ b/crates/rust-client/src/sync/state_sync.rs @@ -83,7 +83,16 @@ pub enum SyncStatus { } impl SyncStatus { - pub fn into_state_sync_update(self) -> StateSyncUpdate { + pub fn is_last_block(&self) -> bool { + match self { + SyncStatus::SyncedToLastBlock(_) => true, + _ => false, + } + } +} + +impl Into for SyncStatus { + fn into(self) -> StateSyncUpdate { match self { SyncStatus::SyncedToLastBlock(update) => update, SyncStatus::SyncedToBlock(update) => update, @@ -182,7 +191,8 @@ impl StateSync { /// # Arguments /// * `current_block` - The latest tracked block header. /// * `current_block_has_relevant_notes` - A flag indicating if the current block has notes that - /// are relevant to the client. + /// are relevant to the client. This is used to determine whether new MMR authentication nodes + /// are stored for this block. /// * `current_partial_mmr` - The current partial MMR. /// * `accounts` - The headers of tracked accounts. /// * `note_tags` - The note tags to be used in the sync state request. @@ -363,20 +373,20 @@ impl StateSync { mut nullifiers: Vec, block_header: BlockHeader, ) -> Result<(NoteUpdates, TransactionUpdates, Vec), ClientError> { - let mut note_updates = NoteUpdates::new_empty(); + let mut note_updates = NoteUpdates::default(); let mut public_note_ids = vec![]; for committed_note in note_inclusions { let (new_note_update, new_note_ids) = (self.on_note_received)(committed_note, block_header).await?; - note_updates.combine_with(new_note_update); + note_updates.extend(new_note_update); public_note_ids.extend(new_note_ids); } let new_public_notes = self.fetch_public_note_details(&public_note_ids, &block_header).await?; - note_updates.combine_with(NoteUpdates::new(new_public_notes, vec![], vec![], vec![])); + note_updates.extend(NoteUpdates::new(new_public_notes, vec![], vec![], vec![])); // We can remove tags from notes that got committed let tags_to_remove = note_updates @@ -391,7 +401,7 @@ impl StateSync { }) .collect(); - let mut transaction_updates = TransactionUpdates::new_empty(); + let mut transaction_updates = TransactionUpdates::default(); for transaction_update in transactions { let (new_note_update, new_transaction_update) = @@ -405,15 +415,15 @@ impl StateSync { .any(|note| note.nullifier() == nullifier.nullifier) }); - note_updates.combine_with(new_note_update); - transaction_updates.combine_with(new_transaction_update); + note_updates.extend(new_note_update); + transaction_updates.extend(new_transaction_update); } for nullifier_update in nullifiers { let (new_note_update, new_transaction_update) = (self.on_nullifier_received)(nullifier_update).await?; - note_updates.combine_with(new_note_update); - transaction_updates.combine_with(new_transaction_update); + note_updates.extend(new_note_update); + transaction_updates.extend(new_transaction_update); } Ok((note_updates, transaction_updates, tags_to_remove)) diff --git a/crates/rust-client/src/transactions/mod.rs b/crates/rust-client/src/transactions/mod.rs index 558ab565c..90cc49820 100644 --- a/crates/rust-client/src/transactions/mod.rs +++ b/crates/rust-client/src/transactions/mod.rs @@ -53,18 +53,20 @@ pub use miden_tx::{DataStoreError, TransactionExecutorError}; pub use script_builder::TransactionScriptBuilderError; // TRANSACTION UPDATES -// -------------------------------------------------------------------------------------------- +// ================================================================================================ /// Contains transaction changes to apply to the store. +#[derive(Debug, Clone, Default)] pub struct TransactionUpdates { - /// Transaction updates for any transaction that was committed between the sync request's - /// block number and the response's block number. + /// Transaction updates for any transaction that was committed between the sync request's block + /// number and the response's block number. committed_transactions: Vec, /// Transaction IDs for any transactions that were discarded in the sync. discarded_transactions: Vec, } impl TransactionUpdates { + /// Creates a new [TransactionUpdates] pub fn new( committed_transactions: Vec, discarded_transactions: Vec, @@ -75,29 +77,25 @@ impl TransactionUpdates { } } - pub fn new_empty() -> Self { - Self { - committed_transactions: vec![], - discarded_transactions: vec![], - } - } - - pub fn combine_with(&mut self, other: Self) { + /// Extends the transaction update information with `other`. + pub fn extend(&mut self, other: Self) { self.committed_transactions.extend(other.committed_transactions); self.discarded_transactions.extend(other.discarded_transactions); } + /// Returns a reference to committed transactions. pub fn committed_transactions(&self) -> &[TransactionUpdate] { &self.committed_transactions } + /// Returns a reference to discarded transactions. pub fn discarded_transactions(&self) -> &[TransactionId] { &self.discarded_transactions } } // TRANSACTION RESULT -// -------------------------------------------------------------------------------------------- +// ================================================================================================ /// Represents the result of executing a transaction by the client. /// @@ -203,7 +201,7 @@ impl From for ExecutedTransaction { } // TRANSACTION RECORD -// -------------------------------------------------------------------------------------------- +// ================================================================================================ /// Describes a transaction that has been executed and is being tracked on the Client. /// @@ -273,7 +271,7 @@ impl fmt::Display for TransactionStatus { } // TRANSACTION STORE UPDATE -// -------------------------------------------------------------------------------------------- +// ================================================================================================ /// Represents the changes that need to be applied to the client store as a result of a /// transaction execution. @@ -601,7 +599,7 @@ impl Client { // HELPERS // -------------------------------------------------------------------------------------------- - /// Helper to get the account outgoing assets. + /// Helper to get the account outgoing asset. /// /// Any outgoing assets resulting from executing note scripts but not present in expected output /// notes wouldn't be included. From 2ff1c919951a9127ff02f8e7e8821f44ff581690 Mon Sep 17 00:00:00 2001 From: Ignacio Amigo Date: Tue, 14 Jan 2025 13:01:19 -0300 Subject: [PATCH 16/34] refactor: move function outside of impl --- crates/rust-client/src/sync/state_sync.rs | 67 +++++++++++------------ 1 file changed, 32 insertions(+), 35 deletions(-) diff --git a/crates/rust-client/src/sync/state_sync.rs b/crates/rust-client/src/sync/state_sync.rs index 5d80ab3b1..ead44a290 100644 --- a/crates/rust-client/src/sync/state_sync.rs +++ b/crates/rust-client/src/sync/state_sync.rs @@ -238,14 +238,13 @@ impl StateSync { ) .await?; - let (new_mmr_peaks, new_authentication_nodes) = self - .apply_mmr_changes( - current_block, - current_block_has_relevant_notes, - current_partial_mmr, - response.mmr_delta, - ) - .await?; + let (new_mmr_peaks, new_authentication_nodes) = apply_mmr_changes( + current_block, + current_block_has_relevant_notes, + current_partial_mmr, + response.mmr_delta, + ) + .await?; let update = StateSyncUpdate { block_header: response.block_header, @@ -266,33 +265,6 @@ impl StateSync { // HELPERS // -------------------------------------------------------------------------------------------- - - /// Applies changes to the current MMR structure, returns the updated [MmrPeaks] and the - /// authentication nodes for leaves we track. - pub(crate) async fn apply_mmr_changes( - &self, - current_block: BlockHeader, - current_block_has_relevant_notes: bool, - mut current_partial_mmr: PartialMmr, - mmr_delta: MmrDelta, - ) -> Result<(MmrPeaks, Vec<(InOrderIndex, Digest)>), ClientError> { - // First, apply curent_block to the MMR. This is needed as the MMR delta received from the - // node doesn't contain the request block itself. - let new_authentication_nodes = current_partial_mmr - .add(current_block.hash(), current_block_has_relevant_notes) - .into_iter(); - - // Apply the MMR delta to bring MMR to forest equal to chain tip - let new_authentication_nodes: Vec<(InOrderIndex, Digest)> = current_partial_mmr - .apply(mmr_delta) - .map_err(StoreError::MmrError)? - .into_iter() - .chain(new_authentication_nodes) - .collect(); - - Ok((current_partial_mmr.peaks(), new_authentication_nodes)) - } - /// Compares the state of tracked accounts with the updates received from the node and returns /// the accounts that need to be updated. /// @@ -511,6 +483,31 @@ pub async fn on_note_received( )) } +/// Applies changes to the current MMR structure, returns the updated [MmrPeaks] and the +/// authentication nodes for leaves we track. +pub(crate) async fn apply_mmr_changes( + current_block: BlockHeader, + current_block_has_relevant_notes: bool, + mut current_partial_mmr: PartialMmr, + mmr_delta: MmrDelta, +) -> Result<(MmrPeaks, Vec<(InOrderIndex, Digest)>), ClientError> { + // First, apply curent_block to the MMR. This is needed as the MMR delta received from the + // node doesn't contain the request block itself. + let new_authentication_nodes = current_partial_mmr + .add(current_block.hash(), current_block_has_relevant_notes) + .into_iter(); + + // Apply the MMR delta to bring MMR to forest equal to chain tip + let new_authentication_nodes: Vec<(InOrderIndex, Digest)> = current_partial_mmr + .apply(mmr_delta) + .map_err(StoreError::MmrError)? + .into_iter() + .chain(new_authentication_nodes) + .collect(); + + Ok((current_partial_mmr.peaks(), new_authentication_nodes)) +} + /// Default implementation of the [OnTransactionCommitted] callback. It queries the store for the /// input notes that were consumed by the transaction and updates the note records accordingly. It /// also returns the committed transaction update to be applied to the store. From da532db10d6fc61b76f5e5096aa8755fe2155764 Mon Sep 17 00:00:00 2001 From: Ignacio Amigo Date: Tue, 14 Jan 2025 14:01:18 -0300 Subject: [PATCH 17/34] chore: fix clippy suggestiosn --- crates/rust-client/src/sync/state_sync.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/crates/rust-client/src/sync/state_sync.rs b/crates/rust-client/src/sync/state_sync.rs index ead44a290..410ae761e 100644 --- a/crates/rust-client/src/sync/state_sync.rs +++ b/crates/rust-client/src/sync/state_sync.rs @@ -84,16 +84,13 @@ pub enum SyncStatus { impl SyncStatus { pub fn is_last_block(&self) -> bool { - match self { - SyncStatus::SyncedToLastBlock(_) => true, - _ => false, - } + matches!(self, SyncStatus::SyncedToLastBlock(_)) } } -impl Into for SyncStatus { - fn into(self) -> StateSyncUpdate { - match self { +impl From for StateSyncUpdate { + fn from(value: SyncStatus) -> StateSyncUpdate { + match value { SyncStatus::SyncedToLastBlock(update) => update, SyncStatus::SyncedToBlock(update) => update, } From 268c9f5bf73d2ef47635918ac967f686733b19b2 Mon Sep 17 00:00:00 2001 From: Ignacio Amigo Date: Wed, 15 Jan 2025 17:32:32 -0300 Subject: [PATCH 18/34] chore: Merge next --- crates/rust-client/src/mock.rs | 2 +- crates/rust-client/src/rpc/mod.rs | 2 +- crates/rust-client/src/rpc/tonic_client/mod.rs | 2 +- crates/rust-client/src/rpc/web_tonic_client/mod.rs | 2 +- crates/rust-client/src/transactions/mod.rs | 5 ++--- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/crates/rust-client/src/mock.rs b/crates/rust-client/src/mock.rs index 421c3f64d..9597db69f 100644 --- a/crates/rust-client/src/mock.rs +++ b/crates/rust-client/src/mock.rs @@ -285,7 +285,7 @@ impl NodeRpcClient for MockRpcApi { } async fn get_account_proofs( - &mut self, + &self, _account_ids: &BTreeSet, _code_commitments: Vec, ) -> Result { diff --git a/crates/rust-client/src/rpc/mod.rs b/crates/rust-client/src/rpc/mod.rs index a355bf6e0..21c36ae48 100644 --- a/crates/rust-client/src/rpc/mod.rs +++ b/crates/rust-client/src/rpc/mod.rs @@ -130,7 +130,7 @@ pub trait NodeRpcClient { /// to prevent unnecessary data fetching. Returns the block number and the FPI account data. If /// one of the tracked accounts is not found in the node, the method will return an error. async fn get_account_proofs( - &mut self, + &self, account_storage_requests: &BTreeSet, known_account_codes: Vec, ) -> Result; diff --git a/crates/rust-client/src/rpc/tonic_client/mod.rs b/crates/rust-client/src/rpc/tonic_client/mod.rs index 2bd98d3d1..092b9b753 100644 --- a/crates/rust-client/src/rpc/tonic_client/mod.rs +++ b/crates/rust-client/src/rpc/tonic_client/mod.rs @@ -269,7 +269,7 @@ impl NodeRpcClient for TonicRpcClient { /// - The answer had a `None` for one of the expected fields. /// - There is an error during storage deserialization. async fn get_account_proofs( - &mut self, + &self, account_requests: &BTreeSet, known_account_codes: Vec, ) -> Result { diff --git a/crates/rust-client/src/rpc/web_tonic_client/mod.rs b/crates/rust-client/src/rpc/web_tonic_client/mod.rs index fdd2824e6..be132474c 100644 --- a/crates/rust-client/src/rpc/web_tonic_client/mod.rs +++ b/crates/rust-client/src/rpc/web_tonic_client/mod.rs @@ -231,7 +231,7 @@ impl NodeRpcClient for WebTonicRpcClient { /// - The answer had a `None` for one of the expected fields. /// - There is an error during storage deserialization. async fn get_account_proofs( - &mut self, + &self, account_requests: &BTreeSet, known_account_codes: Vec, ) -> Result { diff --git a/crates/rust-client/src/transactions/mod.rs b/crates/rust-client/src/transactions/mod.rs index 8731bc7a3..252d846b5 100644 --- a/crates/rust-client/src/transactions/mod.rs +++ b/crates/rust-client/src/transactions/mod.rs @@ -28,8 +28,7 @@ use tracing::info; use super::{Client, FeltRng}; use crate::{ notes::{NoteScreener, NoteUpdates}, - rpc::domain::accounts::AccountProof, - rpc::domain::transactions::TransactionUpdate, + rpc::domain::{accounts::AccountProof, transactions::TransactionUpdate}, store::{ input_note_states::ExpectedNoteState, InputNoteRecord, InputNoteState, NoteFilter, OutputNoteRecord, StoreError, TransactionFilter, @@ -796,7 +795,7 @@ impl Client { // Fetch account proofs let (block_num, account_proofs) = - self.rpc_api.get_account_proofs(&foreign_accounts, known_account_codes).await?; + self.rpc_api.get_account_proofs(&foreign_accounts, known_account_codes).await?; let mut account_proofs: BTreeMap = account_proofs.into_iter().map(|proof| (proof.account_id(), proof)).collect(); From 77d91cea8d7c9ec92743c2ac731a0d85c1056558 Mon Sep 17 00:00:00 2001 From: Ignacio Amigo Date: Wed, 15 Jan 2025 17:43:05 -0300 Subject: [PATCH 19/34] chore: Format --- crates/rust-client/src/transactions/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/rust-client/src/transactions/mod.rs b/crates/rust-client/src/transactions/mod.rs index 252d846b5..88874fd3f 100644 --- a/crates/rust-client/src/transactions/mod.rs +++ b/crates/rust-client/src/transactions/mod.rs @@ -795,7 +795,7 @@ impl Client { // Fetch account proofs let (block_num, account_proofs) = - self.rpc_api.get_account_proofs(&foreign_accounts, known_account_codes).await?; + self.rpc_api.get_account_proofs(&foreign_accounts, known_account_codes).await?; let mut account_proofs: BTreeMap = account_proofs.into_iter().map(|proof| (proof.account_id(), proof)).collect(); From 6b63d59df21acc949c82e749255b2a11dc7727ef Mon Sep 17 00:00:00 2001 From: tomyrd Date: Mon, 27 Jan 2025 13:14:15 -0300 Subject: [PATCH 20/34] review: naming and sections --- crates/rust-client/src/lib.rs | 2 +- crates/rust-client/src/rpc/mod.rs | 2 +- crates/rust-client/src/sync/state_sync.rs | 62 ++++++++++++----------- 3 files changed, 35 insertions(+), 31 deletions(-) diff --git a/crates/rust-client/src/lib.rs b/crates/rust-client/src/lib.rs index 68b5b5732..c3dc50faa 100644 --- a/crates/rust-client/src/lib.rs +++ b/crates/rust-client/src/lib.rs @@ -80,7 +80,7 @@ //! // Instantiate the client using a Tonic RPC client //! let endpoint = Endpoint::new("https".into(), "localhost".into(), Some(57291)); //! let client: Client = Client::new( -//! Box::new(TonicRpcClient::new(endpoint, 10_000)), +//! Arc::new(TonicRpcClient::new(endpoint, 10_000).await?), //! rng, //! store, //! Arc::new(authenticator), diff --git a/crates/rust-client/src/rpc/mod.rs b/crates/rust-client/src/rpc/mod.rs index aa4dab254..8e31dfc5d 100644 --- a/crates/rust-client/src/rpc/mod.rs +++ b/crates/rust-client/src/rpc/mod.rs @@ -21,7 +21,7 @@ //! # async fn main() -> Result<(), Box> { //! // Create a Tonic RPC client instance (assumes default endpoint configuration). //! let endpoint = Endpoint::new("https".into(), "localhost".into(), Some(57291)); -//! let mut rpc_client = TonicRpcClient::new(endpoint, 1000); +//! let mut rpc_client = TonicRpcClient::new(endpoint, 1000).await?; //! //! // Fetch the latest block header (by passing None). //! let (block_header, mmr_proof) = rpc_client.get_block_header_by_number(None, true).await?; diff --git a/crates/rust-client/src/sync/state_sync.rs b/crates/rust-client/src/sync/state_sync.rs index f5bd5625a..1b6336bbe 100644 --- a/crates/rust-client/src/sync/state_sync.rs +++ b/crates/rust-client/src/sync/state_sync.rs @@ -146,7 +146,7 @@ pub struct StateSync { /// Callback to be executed when a new note inclusion is received. on_note_received: OnNoteReceived, /// Callback to be executed when a transaction is committed. - on_committed_transaction: OnTransactionCommitted, + on_transaction_committed: OnTransactionCommitted, /// Callback to be executed when a nullifier is received. on_nullifier_received: OnNullifierReceived, } @@ -163,13 +163,13 @@ impl StateSync { pub fn new( rpc_api: Arc, on_note_received: OnNoteReceived, - on_committed_transaction: OnTransactionCommitted, + on_transaction_committed: OnTransactionCommitted, on_nullifier_received: OnNullifierReceived, ) -> Self { Self { rpc_api, on_note_received, - on_committed_transaction, + on_transaction_committed, on_nullifier_received, } } @@ -261,6 +261,7 @@ impl StateSync { // HELPERS // -------------------------------------------------------------------------------------------- + /// Compares the state of tracked accounts with the updates received from the node and returns /// the accounts that need to be updated. /// @@ -373,7 +374,7 @@ impl StateSync { for transaction_update in transactions { let (new_note_update, new_transaction_update) = - (self.on_committed_transaction)(transaction_update).await?; + (self.on_transaction_committed)(transaction_update).await?; // Remove nullifiers if they were consumed by the transaction nullifiers.retain(|nullifier| { @@ -421,6 +422,34 @@ impl StateSync { } } +// HELPERS +// ================================================================================================ + +/// Applies changes to the current MMR structure, returns the updated [MmrPeaks] and the +/// authentication nodes for leaves we track. +pub(crate) async fn apply_mmr_changes( + current_block: BlockHeader, + current_block_has_relevant_notes: bool, + mut current_partial_mmr: PartialMmr, + mmr_delta: MmrDelta, +) -> Result<(MmrPeaks, Vec<(InOrderIndex, Digest)>), ClientError> { + // First, apply curent_block to the MMR. This is needed as the MMR delta received from the + // node doesn't contain the request block itself. + let new_authentication_nodes = current_partial_mmr + .add(current_block.hash(), current_block_has_relevant_notes) + .into_iter(); + + // Apply the MMR delta to bring MMR to forest equal to chain tip + let new_authentication_nodes: Vec<(InOrderIndex, Digest)> = current_partial_mmr + .apply(mmr_delta) + .map_err(StoreError::MmrError)? + .into_iter() + .chain(new_authentication_nodes) + .collect(); + + Ok((current_partial_mmr.peaks(), new_authentication_nodes)) +} + // DEFAULT CALLBACK IMPLEMENTATIONS // ================================================================================================ @@ -479,31 +508,6 @@ pub async fn on_note_received( )) } -/// Applies changes to the current MMR structure, returns the updated [MmrPeaks] and the -/// authentication nodes for leaves we track. -pub(crate) async fn apply_mmr_changes( - current_block: BlockHeader, - current_block_has_relevant_notes: bool, - mut current_partial_mmr: PartialMmr, - mmr_delta: MmrDelta, -) -> Result<(MmrPeaks, Vec<(InOrderIndex, Digest)>), ClientError> { - // First, apply curent_block to the MMR. This is needed as the MMR delta received from the - // node doesn't contain the request block itself. - let new_authentication_nodes = current_partial_mmr - .add(current_block.hash(), current_block_has_relevant_notes) - .into_iter(); - - // Apply the MMR delta to bring MMR to forest equal to chain tip - let new_authentication_nodes: Vec<(InOrderIndex, Digest)> = current_partial_mmr - .apply(mmr_delta) - .map_err(StoreError::MmrError)? - .into_iter() - .chain(new_authentication_nodes) - .collect(); - - Ok((current_partial_mmr.peaks(), new_authentication_nodes)) -} - /// Default implementation of the [OnTransactionCommitted] callback. It queries the store for the /// input notes that were consumed by the transaction and updates the note records accordingly. It /// also returns the committed transaction update to be applied to the store. From e2f9dc74dc126e714a028d5431e2c355416f1663 Mon Sep 17 00:00:00 2001 From: Tomas Rodriguez Dala <43424983+tomyrd@users.noreply.github.com> Date: Fri, 7 Feb 2025 18:39:56 -0300 Subject: [PATCH 21/34] refactor: remove steps from `StateSync` interface (#709) * feat: remove steps from state sync interface * feat: add block relevance in `StateSyncUpdate` * fix: web client build * chore: update docs * add failing test related to TODOs * fix: add nullifiers for new untracked notes in each step * use `Arc>` in callbacks * feat: reduce the number of callbacks * remove `NoteUpdates` as callback parameter * refactor callbacks * update docs * change callbacks and move state transitions * fix: web store * refator: use partial mmr with current block in sync --- Cargo.lock | 119 ++-- bin/miden-cli/src/commands/sync.rs | 3 +- crates/rust-client/src/account.rs | 7 +- crates/rust-client/src/note/import.rs | 4 +- crates/rust-client/src/note/mod.rs | 130 ++-- .../rust-client/src/rpc/domain/nullifier.rs | 1 + crates/rust-client/src/store/mod.rs | 10 +- .../rust-client/src/store/sqlite_store/mod.rs | 8 +- .../src/store/sqlite_store/note.rs | 10 +- .../src/store/sqlite_store/sync.rs | 32 +- .../src/store/web_store/js/sync.js | 40 +- crates/rust-client/src/store/web_store/mod.rs | 8 +- .../src/store/web_store/note/utils.rs | 10 +- .../src/store/web_store/sync/js_bindings.rs | 9 +- .../src/store/web_store/sync/mod.rs | 46 +- crates/rust-client/src/sync/block_header.rs | 71 +-- crates/rust-client/src/sync/mod.rs | 130 ++-- crates/rust-client/src/sync/state_sync.rs | 553 ++++++++---------- crates/rust-client/src/tests.rs | 2 +- crates/rust-client/src/transaction/mod.rs | 11 +- crates/web-client/package-lock.json | 4 +- crates/web-client/src/models/sync_summary.rs | 4 - tests/integration/main.rs | 4 +- tests/integration/onchain_tests.rs | 26 +- 24 files changed, 613 insertions(+), 629 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ac21401d0..be3e4ae55 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -328,9 +328,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.11.1" +version = "1.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "786a307d683a5bf92e6fd5fd69a7eb613751668d1d8d67d802846dfe367c62c8" +checksum = "531a9155a481e2ee699d4f98f43c0ca4ff8ee1bfd55c31e9e98fb29d2b176fe0" dependencies = [ "memchr", "regex-automata 0.4.9", @@ -339,9 +339,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.16.0" +version = "3.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" [[package]] name = "bytemuck" @@ -474,9 +474,9 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b80225097f2e5ae4e7179dd2266824648f3e2f49d9134d584b76389d31c4c3" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" dependencies = [ "libc", ] @@ -825,10 +825,22 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "wasm-bindgen", ] +[[package]] +name = "getrandom" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.13.3+wasi-0.2.2", + "windows-targets 0.52.6", +] + [[package]] name = "gimli" version = "0.31.1" @@ -837,9 +849,9 @@ checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "glob" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" [[package]] name = "h2" @@ -944,9 +956,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.9.5" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" +checksum = "f2d708df4e7140240a16cd6ab0ab65c972d7433ab77819ea693fde9c43811e2a" [[package]] name = "httpdate" @@ -956,9 +968,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "1.5.2" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "256fb8d4bd6413123cc9d91832d78325c48ff41677595be797d90f42969beae0" +checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80" dependencies = [ "bytes", "futures-channel", @@ -1426,9 +1438,9 @@ dependencies = [ [[package]] name = "miden-crypto" -version = "0.13.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06bf3ad2a85f3f8f0da73b6357c77e482b1ceb36cacda8a2d85caae3bd1f702" +checksum = "1945918276152bd9b8e8434643ad24d4968e075b68a5ed03927b53ac75490a79" dependencies = [ "blake3", "cc", @@ -1511,11 +1523,11 @@ dependencies = [ [[package]] name = "miden-objects" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4622f71888d4641577d145dd561165824b9b9293fd0be2306326f11bcf39e67d" +checksum = "8fe3f10d0e3787176f0803be2ecb4646f3a17fe10af45a50736c8d079a3c94d8" dependencies = [ - "getrandom", + "getrandom 0.2.15", "miden-assembly", "miden-core", "miden-crypto", @@ -1577,9 +1589,9 @@ dependencies = [ [[package]] name = "miden-rpc-proto" -version = "0.7.0" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c2edb867ffc4cd908bb98b48f326016e36158129faba39aef574a0f33433b60" +checksum = "e68f00e0e97ba6bc65896e5ca2bb0995d59b14fd8f0389916dc2ee792f353f50" [[package]] name = "miden-stdlib" @@ -1674,7 +1686,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" dependencies = [ "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.52.0", ] @@ -1901,18 +1913,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.7" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be57f64e946e500c8ee36ef6331845d40a93055567ec57e8fae13efd33759b95" +checksum = "1e2ec53ad785f4d35dac0adea7f7dc6f1bb277ad84a680c7afefeae05d1f5916" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.7" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" +checksum = "d56a66c0c55993aa927429d0f8a0abfd74f084e4d9c192cffed01e418d83eefb" dependencies = [ "proc-macro2", "quote", @@ -1921,9 +1933,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" [[package]] name = "pin-utils" @@ -2148,7 +2160,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom 0.2.15", ] [[package]] @@ -2195,7 +2207,7 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ - "getrandom", + "getrandom 0.2.15", "libredox", "thiserror 1.0.69", ] @@ -2303,9 +2315,9 @@ checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4" [[package]] name = "ryu" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +checksum = "6ea1a2d0a644769cc99faa24c3ad26b379b786fe7c36fd3c546254801650e6dd" [[package]] name = "same-file" @@ -2354,9 +2366,9 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.216" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" dependencies = [ "serde_derive", ] @@ -2374,9 +2386,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.216" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", @@ -2385,9 +2397,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.137" +version = "1.0.138" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "930cfb6e6abf99298aaad7d29abbef7a9999a9a8806a40088f55f0dcec03146b" +checksum = "d434192e7da787e94a6ea7e9670b26a036d0ca41e0b7efb2676dd32bae872949" dependencies = [ "itoa", "memchr", @@ -2565,12 +2577,13 @@ checksum = "42a4d50cdb458045afc8131fd91b64904da29548bcb63c7236e0844936c13078" [[package]] name = "tempfile" -version = "3.14.0" +version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" +checksum = "38c246215d7d24f48ae091a2902398798e05d978b24315d6efbc00ede9a8bb91" dependencies = [ "cfg-if", "fastrand", + "getrandom 0.3.1", "once_cell", "rustix", "windows-sys 0.59.0", @@ -2992,9 +3005,9 @@ dependencies = [ [[package]] name = "unicode-ident" -version = "1.0.15" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11cd88e12b17c6494200a9c1b683a04fcac9573ed74cd1b62aeb2727c5592243" +checksum = "a210d160f08b701c8721ba1c726c11662f877ea6b7094007e1ca9a1041945034" [[package]] name = "unicode-linebreak" @@ -3032,7 +3045,7 @@ version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3758f5e68192bb96cc8f9b7e2c2cfdabb435499a28499a42f8f984092adad4b" dependencies = [ - "getrandom", + "getrandom 0.2.15", "serde", ] @@ -3097,6 +3110,15 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasi" +version = "0.13.3+wasi-0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2" +dependencies = [ + "wit-bindgen-rt", +] + [[package]] name = "wasm-bindgen" version = "0.2.100" @@ -3447,9 +3469,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.6.24" +version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d71a593cc5c42ad7876e2c1fda56f314f3754c084128833e64f1345ff8a03a" +checksum = "ad699df48212c6cc6eb4435f35500ac6fd3b9913324f938aea302022ce19d310" dependencies = [ "memchr", ] @@ -3556,6 +3578,15 @@ dependencies = [ "winter-utils", ] +[[package]] +name = "wit-bindgen-rt" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" +dependencies = [ + "bitflags", +] + [[package]] name = "yansi" version = "1.0.1" diff --git a/bin/miden-cli/src/commands/sync.rs b/bin/miden-cli/src/commands/sync.rs index da5b763d8..cc6f1e7b8 100644 --- a/bin/miden-cli/src/commands/sync.rs +++ b/bin/miden-cli/src/commands/sync.rs @@ -12,8 +12,7 @@ impl SyncCmd { let new_details = client.sync_state().await?; println!("State synced to block {}", new_details.block_num); - println!("New public notes: {}", new_details.received_notes.len()); - println!("Tracked notes updated: {}", new_details.committed_notes.len()); + println!("Committed notes: {}", new_details.committed_notes.len()); println!("Tracked notes consumed: {}", new_details.consumed_notes.len()); println!("Tracked accounts updated: {}", new_details.updated_accounts.len()); println!("Locked accounts: {}", new_details.locked_accounts.len()); diff --git a/crates/rust-client/src/account.rs b/crates/rust-client/src/account.rs index 60ceb6981..4618aaa81 100644 --- a/crates/rust-client/src/account.rs +++ b/crates/rust-client/src/account.rs @@ -256,7 +256,7 @@ impl Client { // ACCOUNT UPDATES // ================================================================================================ -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] /// Contains account changes to apply to the store. pub struct AccountUpdates { /// Updated public accounts. @@ -286,6 +286,11 @@ impl AccountUpdates { pub fn mismatched_private_accounts(&self) -> &[(AccountId, Digest)] { &self.mismatched_private_accounts } + + pub fn extend(&mut self, other: AccountUpdates) { + self.updated_public_accounts.extend(other.updated_public_accounts); + self.mismatched_private_accounts.extend(other.mismatched_private_accounts); + } } // TESTS diff --git a/crates/rust-client/src/note/import.rs b/crates/rust-client/src/note/import.rs index 3e716d518..a18cee577 100644 --- a/crates/rust-client/src/note/import.rs +++ b/crates/rust-client/src/note/import.rs @@ -175,7 +175,7 @@ impl Client { note_record.inclusion_proof_received(inclusion_proof, metadata)?; if block_height < current_block_num { - let mut current_partial_mmr = self.build_current_partial_mmr(true).await?; + let mut current_partial_mmr = self.build_current_partial_mmr().await?; let block_header = self .get_and_store_authenticated_block(block_height, &mut current_partial_mmr) @@ -219,7 +219,7 @@ impl Client { match committed_note_data { Some((metadata, inclusion_proof)) => { - let mut current_partial_mmr = self.build_current_partial_mmr(true).await?; + let mut current_partial_mmr = self.build_current_partial_mmr().await?; let block_header = self .get_and_store_authenticated_block( inclusion_proof.location().block_num(), diff --git a/crates/rust-client/src/note/mod.rs b/crates/rust-client/src/note/mod.rs index 433db9570..4f5c0db2f 100644 --- a/crates/rust-client/src/note/mod.rs +++ b/crates/rust-client/src/note/mod.rs @@ -56,7 +56,11 @@ //! For more details on the API and error handling, see the documentation for the specific functions //! and types in this module. -use alloc::{collections::BTreeSet, string::ToString, vec::Vec}; +use alloc::{ + collections::{BTreeMap, BTreeSet}, + string::ToString, + vec::Vec, +}; use miden_lib::transaction::TransactionKernel; use miden_objects::{account::AccountId, crypto::rand::FeltRng}; @@ -244,89 +248,61 @@ pub async fn get_input_note_with_id_prefix( /// Contains note changes to apply to the store. #[derive(Clone, Debug, Default)] pub struct NoteUpdates { - /// A list of new input notes. - new_input_notes: Vec, - /// A list of new output notes. - new_output_notes: Vec, - /// A list of updated input note records corresponding to locally-tracked input notes. - updated_input_notes: Vec, - /// A list of updated output note records corresponding to locally-tracked output notes. - updated_output_notes: Vec, + /// A map of new and updated input note records to be upserted in the store. + updated_input_notes: BTreeMap, + /// A map of updated output note records to be upserted in the store. + updated_output_notes: BTreeMap, } impl NoteUpdates { /// Creates a [NoteUpdates]. pub fn new( - new_input_notes: Vec, - new_output_notes: Vec, updated_input_notes: Vec, updated_output_notes: Vec, ) -> Self { Self { - new_input_notes, - new_output_notes, - updated_input_notes, - updated_output_notes, + updated_input_notes: updated_input_notes + .into_iter() + .map(|note| (note.id(), note)) + .collect(), + updated_output_notes: updated_output_notes + .into_iter() + .map(|note| (note.id(), note)) + .collect(), } } - /// Combines two [NoteUpdates] into a single one. - pub fn extend(&mut self, other: Self) { - self.new_input_notes.extend(other.new_input_notes); - self.new_output_notes.extend(other.new_output_notes); - self.updated_input_notes.extend(other.updated_input_notes); - self.updated_output_notes.extend(other.updated_output_notes); - } - - /// Returns all new input note records, meant to be tracked by the client. - pub fn new_input_notes(&self) -> &[InputNoteRecord] { - &self.new_input_notes - } - - /// Returns all new output note records, meant to be tracked by the client. - pub fn new_output_notes(&self) -> &[OutputNoteRecord] { - &self.new_output_notes + /// Returns all input note records that have been updated. + pub fn updated_input_notes(&self) -> impl Iterator { + self.updated_input_notes.values() } - /// Returns all updated input note records. That is, any input notes that are locally tracked - /// and have been updated. - pub fn updated_input_notes(&self) -> &[InputNoteRecord] { - &self.updated_input_notes - } - - /// Returns all updated output note records. That is, any output notes that are locally tracked - /// and have been updated. - pub fn updated_output_notes(&self) -> &[OutputNoteRecord] { - &self.updated_output_notes + /// Returns all updated output note records that have been updated. + pub fn updated_output_notes(&self) -> impl Iterator { + self.updated_output_notes.values() } /// Returns whether no new note-related information has been retrieved. pub fn is_empty(&self) -> bool { - self.updated_input_notes.is_empty() - && self.updated_output_notes.is_empty() - && self.new_input_notes.is_empty() - && self.new_output_notes.is_empty() + self.updated_input_notes.is_empty() && self.updated_output_notes.is_empty() } /// Returns any note that has been committed into the chain in this update (either new or /// already locally tracked) pub fn committed_input_notes(&self) -> impl Iterator { - self.updated_input_notes - .iter() - .chain(self.new_input_notes.iter()) - .filter(|note| note.is_committed()) + self.updated_input_notes.values().filter(|note| note.is_committed()) } /// Returns the IDs of all notes that have been committed (previously locally tracked). pub fn committed_note_ids(&self) -> BTreeSet { let committed_output_note_ids = self .updated_output_notes - .iter() + .values() .filter_map(|note_record| note_record.is_committed().then_some(note_record.id())); let committed_input_note_ids = self .updated_input_notes - .iter() + .values() .filter_map(|note_record| note_record.is_committed().then_some(note_record.id())); BTreeSet::from_iter(committed_input_note_ids.chain(committed_output_note_ids)) @@ -336,14 +312,62 @@ impl NoteUpdates { pub fn consumed_note_ids(&self) -> BTreeSet { let consumed_output_note_ids = self .updated_output_notes - .iter() + .values() .filter_map(|note_record| note_record.is_consumed().then_some(note_record.id())); let consumed_input_note_ids = self .updated_input_notes - .iter() + .values() .filter_map(|note_record| note_record.is_consumed().then_some(note_record.id())); BTreeSet::from_iter(consumed_input_note_ids.chain(consumed_output_note_ids)) } + + pub fn extend(&mut self, other: NoteUpdates) { + self.updated_input_notes.extend(other.updated_input_notes); + self.updated_output_notes.extend(other.updated_output_notes); + } + + pub fn insert_updates( + &mut self, + input_note: Option, + output_note: Option, + ) { + if let Some(input_note) = input_note { + self.updated_input_notes.insert(input_note.id(), input_note); + } + if let Some(output_note) = output_note { + self.updated_output_notes.insert(output_note.id(), output_note); + } + } + + /// Returns a mutable reference to the input note record with the provided ID if it exists. + pub fn get_input_note_by_id(&mut self, note_id: NoteId) -> Option<&mut InputNoteRecord> { + self.updated_input_notes.get_mut(¬e_id) + } + + /// Returns a mutable reference to the output note record with the provided ID if it exists. + pub fn get_output_note_by_id(&mut self, note_id: NoteId) -> Option<&mut OutputNoteRecord> { + self.updated_output_notes.get_mut(¬e_id) + } + + /// Returns a mutable reference to the input note record with the provided nullifier if it + /// exists. + pub fn get_input_note_by_nullifier( + &mut self, + nullifier: Nullifier, + ) -> Option<&mut InputNoteRecord> { + self.updated_input_notes.values_mut().find(|note| note.nullifier() == nullifier) + } + + /// Returns a mutable reference to the output note record with the provided nullifier if it + /// exists. + pub fn get_output_note_by_nullifier( + &mut self, + nullifier: Nullifier, + ) -> Option<&mut OutputNoteRecord> { + self.updated_output_notes + .values_mut() + .find(|note| note.nullifier() == Some(nullifier)) + } } diff --git a/crates/rust-client/src/rpc/domain/nullifier.rs b/crates/rust-client/src/rpc/domain/nullifier.rs index 9834fbdb5..345508063 100644 --- a/crates/rust-client/src/rpc/domain/nullifier.rs +++ b/crates/rust-client/src/rpc/domain/nullifier.rs @@ -6,6 +6,7 @@ use crate::rpc::{errors::RpcConversionError, generated::digest::Digest}; // ================================================================================================ /// Represents a note that was consumed in the node at a certain block. +#[derive(Debug, Clone)] pub struct NullifierUpdate { /// The nullifier of the consumed note. pub nullifier: Nullifier, diff --git a/crates/rust-client/src/store/mod.rs b/crates/rust-client/src/store/mod.rs index 0872074a7..463fa7b77 100644 --- a/crates/rust-client/src/store/mod.rs +++ b/crates/rust-client/src/store/mod.rs @@ -327,15 +327,7 @@ pub trait Store: Send + Sync { /// locally does not match the `StateSyncUpdate` information, the account may be locked. /// - Storing new MMR authentication nodes. /// - Updating the tracked public accounts. - /// - /// A [StateSyncUpdate] corresponds to an update from a single `SyncState` RPC call. For the - /// client to be up to date against the current chain tip, multiple calls may be performed, and - /// so multiple store updates could happen sequentially. - async fn apply_state_sync_step( - &self, - state_sync_update: StateSyncUpdate, - new_block_has_relevant_notes: bool, - ) -> Result<(), StoreError>; + async fn apply_state_sync(&self, state_sync_update: StateSyncUpdate) -> Result<(), StoreError>; } // CHAIN MMR NODE FILTER diff --git a/crates/rust-client/src/store/sqlite_store/mod.rs b/crates/rust-client/src/store/sqlite_store/mod.rs index 576cb171a..92d9715ac 100644 --- a/crates/rust-client/src/store/sqlite_store/mod.rs +++ b/crates/rust-client/src/store/sqlite_store/mod.rs @@ -144,13 +144,9 @@ impl Store for SqliteStore { self.interact_with_connection(SqliteStore::get_sync_height).await } - async fn apply_state_sync_step( - &self, - state_sync_update: StateSyncUpdate, - block_has_relevant_notes: bool, - ) -> Result<(), StoreError> { + async fn apply_state_sync(&self, state_sync_update: StateSyncUpdate) -> Result<(), StoreError> { self.interact_with_connection(move |conn| { - SqliteStore::apply_state_sync_step(conn, state_sync_update, block_has_relevant_notes) + SqliteStore::apply_state_sync(conn, state_sync_update) }) .await } diff --git a/crates/rust-client/src/store/sqlite_store/note.rs b/crates/rust-client/src/store/sqlite_store/note.rs index bd6a831ba..ce017c67d 100644 --- a/crates/rust-client/src/store/sqlite_store/note.rs +++ b/crates/rust-client/src/store/sqlite_store/note.rs @@ -587,17 +587,11 @@ pub(crate) fn apply_note_updates_tx( tx: &Transaction, note_updates: &NoteUpdates, ) -> Result<(), StoreError> { - for input_note in - note_updates.new_input_notes().iter().chain(note_updates.updated_input_notes()) - { + for input_note in note_updates.updated_input_notes() { upsert_input_note_tx(tx, input_note)?; } - for output_note in note_updates - .new_output_notes() - .iter() - .chain(note_updates.updated_output_notes()) - { + for output_note in note_updates.updated_output_notes() { upsert_output_note_tx(tx, output_note)?; } diff --git a/crates/rust-client/src/store/sqlite_store/sync.rs b/crates/rust-client/src/store/sqlite_store/sync.rs index 1086cfba3..804bc0412 100644 --- a/crates/rust-client/src/store/sqlite_store/sync.rs +++ b/crates/rust-client/src/store/sqlite_store/sync.rs @@ -91,19 +91,16 @@ impl SqliteStore { .expect("state sync block number exists") } - pub(super) fn apply_state_sync_step( + pub(super) fn apply_state_sync( conn: &mut Connection, state_sync_update: StateSyncUpdate, - block_has_relevant_notes: bool, ) -> Result<(), StoreError> { let StateSyncUpdate { - block_header, + block_num, + block_updates, note_updates, transaction_updates, - new_mmr_peaks, - new_authentication_nodes, account_updates, - tags_to_remove, } = state_sync_update; let mut locked_accounts = vec![]; @@ -125,21 +122,34 @@ impl SqliteStore { // Update state sync block number const BLOCK_NUMBER_QUERY: &str = "UPDATE state_sync SET block_num = ?"; - tx.execute(BLOCK_NUMBER_QUERY, params![block_header.block_num().as_u32() as i64])?; + tx.execute(BLOCK_NUMBER_QUERY, params![block_num.as_u64() as i64])?; + + for (block_header, block_has_relevant_notes, new_mmr_peaks) in block_updates.block_headers { + Self::insert_block_header_tx( + &tx, + block_header, + new_mmr_peaks, + block_has_relevant_notes, + )?; + } - Self::insert_block_header_tx(&tx, block_header, new_mmr_peaks, block_has_relevant_notes)?; + // Insert new authentication nodes (inner nodes of the PartialMmr) + Self::insert_chain_mmr_nodes_tx(&tx, &block_updates.new_authentication_nodes)?; // Update notes apply_note_updates_tx(&tx, ¬e_updates)?; // Remove tags + let tags_to_remove = note_updates.committed_input_notes().map(|note| { + NoteTagRecord::with_note_source( + note.metadata().expect("Committed notes should have metadata").tag(), + note.id(), + ) + }); for tag in tags_to_remove { remove_note_tag_tx(&tx, tag)?; } - // Insert new authentication nodes (inner nodes of the PartialMmr) - Self::insert_chain_mmr_nodes_tx(&tx, &new_authentication_nodes)?; - // Mark transactions as committed Self::mark_transactions_as_committed(&tx, transaction_updates.committed_transactions())?; diff --git a/crates/rust-client/src/store/web_store/js/sync.js b/crates/rust-client/src/store/web_store/js/sync.js index 2a633ac61..ed7cafd03 100644 --- a/crates/rust-client/src/store/web_store/js/sync.js +++ b/crates/rust-client/src/store/web_store/js/sync.js @@ -80,8 +80,9 @@ export async function removeNoteTag(tag, source_note_id, source_account_id) { export async function applyStateSync( blockNum, - blockHeader, - chainMmrPeaks, + newBlockHeadersAsFlattenedVec, + newBlockNums, + chainMmrPeaksAsFlattenedVec, hasClientNotes, nodeIndexes, nodes, @@ -89,6 +90,11 @@ export async function applyStateSync( committedTransactionIds, transactionBlockNums ) { + const newBlockHeaders = reconstructFlattenedVec( + newBlockHeadersAsFlattenedVec + ); + const chainMmrPeaks = reconstructFlattenedVec(chainMmrPeaksAsFlattenedVec); + return db.transaction( "rw", stateSync, @@ -100,13 +106,15 @@ export async function applyStateSync( tags, async (tx) => { await updateSyncHeight(tx, blockNum); - await updateBlockHeader( - tx, - blockNum, - blockHeader, - chainMmrPeaks, - hasClientNotes - ); + for (let i = 0; i < newBlockHeaders.length; i++) { + await updateBlockHeader( + tx, + newBlockNums[i], + newBlockHeaders[i], + chainMmrPeaks[i], + hasClientNotes[i] + ); + } await updateChainMmrNodes(tx, nodeIndexes, nodes); await updateCommittedNoteTags(tx, inputNoteIds); await updateCommittedTransactions( @@ -232,3 +240,17 @@ function uint8ArrayToBase64(bytes) { ); return btoa(binary); } + +// Helper function to reconstruct arrays from flattened data +function reconstructFlattenedVec(flattenedVec) { + const data = flattenedVec.data(); + const lengths = flattenedVec.lengths(); + + let index = 0; + const result = []; + lengths.forEach((length) => { + result.push(data.slice(index, index + length)); + index += length; + }); + return result; +} diff --git a/crates/rust-client/src/store/web_store/mod.rs b/crates/rust-client/src/store/web_store/mod.rs index 0d8d8d5dc..0e8968cbb 100644 --- a/crates/rust-client/src/store/web_store/mod.rs +++ b/crates/rust-client/src/store/web_store/mod.rs @@ -79,12 +79,8 @@ impl Store for WebStore { self.get_sync_height().await } - async fn apply_state_sync_step( - &self, - state_sync_update: StateSyncUpdate, - block_has_relevant_notes: bool, - ) -> Result<(), StoreError> { - self.apply_state_sync_step(state_sync_update, block_has_relevant_notes).await + async fn apply_state_sync(&self, state_sync_update: StateSyncUpdate) -> Result<(), StoreError> { + self.apply_state_sync(state_sync_update).await } // TRANSACTIONS diff --git a/crates/rust-client/src/store/web_store/note/utils.rs b/crates/rust-client/src/store/web_store/note/utils.rs index 0d1c79e0b..4dc4c103f 100644 --- a/crates/rust-client/src/store/web_store/note/utils.rs +++ b/crates/rust-client/src/store/web_store/note/utils.rs @@ -193,17 +193,11 @@ pub fn parse_output_note_idxdb_object( } pub(crate) async fn apply_note_updates_tx(note_updates: &NoteUpdates) -> Result<(), StoreError> { - for input_note in - note_updates.new_input_notes().iter().chain(note_updates.updated_input_notes()) - { + for input_note in note_updates.updated_input_notes() { upsert_input_note_tx(input_note).await?; } - for output_note in note_updates - .new_output_notes() - .iter() - .chain(note_updates.updated_output_notes()) - { + for output_note in note_updates.updated_output_notes() { upsert_output_note_tx(output_note).await?; } diff --git a/crates/rust-client/src/store/web_store/sync/js_bindings.rs b/crates/rust-client/src/store/web_store/sync/js_bindings.rs index 4c8fc7341..31249e78f 100644 --- a/crates/rust-client/src/store/web_store/sync/js_bindings.rs +++ b/crates/rust-client/src/store/web_store/sync/js_bindings.rs @@ -3,6 +3,8 @@ use alloc::{string::String, vec::Vec}; use wasm_bindgen::prelude::*; use wasm_bindgen_futures::*; +use super::flattened_vec::FlattenedU8Vec; + // Sync IndexedDB Operations #[wasm_bindgen(module = "/src/store/web_store/js/sync.js")] @@ -29,9 +31,10 @@ extern "C" { #[wasm_bindgen(js_name = applyStateSync)] pub fn idxdb_apply_state_sync( block_num: String, - block_header: Vec, - chain_mmr_peaks: Vec, - has_client_notes: bool, + flattened_new_block_headers: FlattenedU8Vec, + new_block_nums: Vec, + flattened_chain_mmr_peaks: FlattenedU8Vec, + has_client_notes: Vec, serialized_node_ids: Vec, serialized_nodes: Vec, note_tags_to_remove_as_str: Vec, diff --git a/crates/rust-client/src/store/web_store/sync/mod.rs b/crates/rust-client/src/store/web_store/sync/mod.rs index feba70f27..f847d8754 100644 --- a/crates/rust-client/src/store/web_store/sync/mod.rs +++ b/crates/rust-client/src/store/web_store/sync/mod.rs @@ -29,6 +29,9 @@ use js_bindings::*; mod models; use models::*; +mod flattened_vec; +use flattened_vec::*; + impl WebStore { pub(crate) async fn get_note_tags(&self) -> Result, StoreError> { let promise = idxdb_get_note_tags(); @@ -98,32 +101,38 @@ impl WebStore { Ok(removed_tags) } - pub(super) async fn apply_state_sync_step( + pub(super) async fn apply_state_sync( &self, state_sync_update: StateSyncUpdate, - block_has_relevant_notes: bool, ) -> Result<(), StoreError> { let StateSyncUpdate { - block_header, + block_num, + block_updates, note_updates, transaction_updates, //TODO: Add support for discarded transactions in web store - new_mmr_peaks, - new_authentication_nodes, account_updates, - tags_to_remove, } = state_sync_update; // Serialize data for updating state sync and block header - let block_num_as_str = block_header.block_num().to_string(); + let block_num_as_str = block_num.to_string(); // Serialize data for updating block header - let block_header_as_bytes = block_header.to_bytes(); - let new_mmr_peaks_as_bytes = new_mmr_peaks.peaks().to_vec().to_bytes(); + let mut block_headers_as_bytes = vec![]; + let mut new_mmr_peaks_as_bytes = vec![]; + let mut block_nums_as_str = vec![]; + let mut block_has_relevant_notes = vec![]; + + for (block_header, has_client_notes, mmr_peaks) in block_updates.block_headers.iter() { + block_headers_as_bytes.push(block_header.to_bytes()); + new_mmr_peaks_as_bytes.push(mmr_peaks.peaks().to_vec().to_bytes()); + block_nums_as_str.push(block_header.block_num().to_string()); + block_has_relevant_notes.push(*has_client_notes as u8); + } // Serialize data for updating chain MMR nodes let mut serialized_node_ids = Vec::new(); let mut serialized_nodes = Vec::new(); - for (id, node) in new_authentication_nodes.iter() { + for (id, node) in block_updates.new_authentication_nodes.iter() { let serialized_data = serialize_chain_mmr_node(*id, *node)?; serialized_node_ids.push(serialized_data.id); serialized_nodes.push(serialized_data.node); @@ -134,16 +143,8 @@ impl WebStore { apply_note_updates_tx(¬e_updates).await?; // Tags to remove - let note_tags_to_remove_as_str: Vec = tags_to_remove - .iter() - .filter_map(|tag_record| { - if let NoteTagSource::Note(note_id) = tag_record.source { - Some(note_id.to_hex()) - } else { - None - } - }) - .collect(); + let note_tags_to_remove_as_str: Vec = + note_updates.committed_input_notes().map(|note| note.id().to_hex()).collect(); // Serialize data for updating committed transactions let transactions_to_commit_block_nums_as_str = transaction_updates @@ -178,8 +179,9 @@ impl WebStore { let promise = idxdb_apply_state_sync( block_num_as_str, - block_header_as_bytes, - new_mmr_peaks_as_bytes, + flatten_nested_u8_vec(block_headers_as_bytes), + block_nums_as_str, + flatten_nested_u8_vec(new_mmr_peaks_as_bytes), block_has_relevant_notes, serialized_node_ids, serialized_nodes, diff --git a/crates/rust-client/src/sync/block_header.rs b/crates/rust-client/src/sync/block_header.rs index 7d63224a7..e0a2d44b4 100644 --- a/crates/rust-client/src/sync/block_header.rs +++ b/crates/rust-client/src/sync/block_header.rs @@ -8,19 +8,36 @@ use miden_objects::{ }; use tracing::warn; -use super::NoteUpdates; use crate::{ - note::NoteScreener, store::{ChainMmrNodeFilter, NoteFilter, StoreError}, Client, ClientError, }; +/// Contains all the block information that needs to be added in the client's store after a sync. + +#[derive(Debug, Clone, Default)] +pub struct BlockUpdates { + /// New block headers to be stored, along with a flag indicating whether the block contains + /// notes that are relevant to the client and the MMR peaks for the block. + pub block_headers: Vec<(BlockHeader, bool, MmrPeaks)>, + /// New authentication nodes that are meant to be stored in order to authenticate block + /// headers. + pub new_authentication_nodes: Vec<(InOrderIndex, Digest)>, +} + +impl BlockUpdates { + pub fn extend(&mut self, other: BlockUpdates) { + self.block_headers.extend(other.block_headers); + self.new_authentication_nodes.extend(other.new_authentication_nodes); + } +} + /// Network information management methods. impl Client { /// Updates committed notes with no MMR data. These could be notes that were /// imported with an inclusion proof, but its block header isn't tracked. pub(crate) async fn update_mmr_data(&self) -> Result<(), ClientError> { - let mut current_partial_mmr = self.build_current_partial_mmr(true).await?; + let mut current_partial_mmr = self.build_current_partial_mmr().await?; let mut changed_notes = vec![]; for mut note in self.store.get_input_notes(NoteFilter::Unverified).await? { @@ -73,43 +90,13 @@ impl Client { // HELPERS // -------------------------------------------------------------------------------------------- - /// Checks the relevance of the block by verifying if any of the input notes in the block are - /// relevant to the client. If any of the notes are relevant, the function returns `true`. - pub(crate) async fn check_block_relevance( - &self, - note_updates: &NoteUpdates, - ) -> Result { - // We'll only do the check for either incoming public notes or expected input notes as - // output notes are not really candidates to be consumed here. - - let note_screener = NoteScreener::new(self.store.clone()); - - // Find all relevant Input Notes using the note checker - for input_note in note_updates.committed_input_notes() { - // TODO: Map the below error into a better representation (ie, we expected to be able - // to convert here) - if !note_screener - .check_relevance(&input_note.try_into().map_err(ClientError::NoteRecordError)?) - .await? - .is_empty() - { - return Ok(true); - } - } - - Ok(false) - } - /// Builds the current store view of the chain's [PartialMmr]. Because we want to add all new /// authentication nodes that could come from applying the MMR updates, we need to track all /// known leaves thus far. /// /// As part of the syncing process, we add the current block number so we don't need to /// track it here. - pub(crate) async fn build_current_partial_mmr( - &self, - include_current_block: bool, - ) -> Result { + pub(crate) async fn build_current_partial_mmr(&self) -> Result { let current_block_num = self.store.get_sync_height().await?; let tracked_nodes = self.store.get_chain_mmr_nodes(ChainMmrNodeFilter::All).await?; @@ -131,15 +118,13 @@ impl Client { let mut current_partial_mmr = PartialMmr::from_parts(current_peaks, tracked_nodes, track_latest); - if include_current_block { - let (current_block, has_client_notes) = self - .store - .get_block_header_by_num(current_block_num) - .await? - .expect("Current block should be in the store"); + let (current_block, has_client_notes) = self + .store + .get_block_header_by_num(current_block_num) + .await? + .expect("Current block should be in the store"); - current_partial_mmr.add(current_block.hash(), has_client_notes); - } + current_partial_mmr.add(current_block.hash(), has_client_notes); Ok(current_partial_mmr) } @@ -207,7 +192,7 @@ impl Client { return self.ensure_genesis_in_place().await; } - let mut current_partial_mmr = self.build_current_partial_mmr(true).await?; + let mut current_partial_mmr = self.build_current_partial_mmr().await?; let anchor_block = self .get_and_store_authenticated_block(epoch_block_number, &mut current_partial_mmr) .await?; diff --git a/crates/rust-client/src/sync/mod.rs b/crates/rust-client/src/sync/mod.rs index 383d158a1..644202c25 100644 --- a/crates/rust-client/src/sync/mod.rs +++ b/crates/rust-client/src/sync/mod.rs @@ -38,7 +38,6 @@ //! let sync_summary: SyncSummary = client.sync_state().await?; //! //! println!("Synced up to block number: {}", sync_summary.block_num); -//! println!("Received notes: {}", sync_summary.received_notes.len()); //! println!("Committed notes: {}", sync_summary.committed_notes.len()); //! println!("Consumed notes: {}", sync_summary.consumed_notes.len()); //! println!("Updated accounts: {}", sync_summary.updated_accounts.len()); @@ -65,8 +64,9 @@ use miden_objects::{ note::{NoteId, NoteTag, Nullifier}, transaction::TransactionId, }; +use state_sync::on_note_received; -use crate::{note::NoteUpdates, Client, ClientError}; +use crate::{store::NoteFilter, Client, ClientError}; mod block_header; @@ -74,18 +74,13 @@ mod tag; pub use tag::{NoteTagRecord, NoteTagSource}; mod state_sync; -pub use state_sync::{ - on_note_received, on_nullifier_received, on_transaction_committed, OnNoteReceived, - OnNullifierReceived, OnTransactionCommitted, StateSync, StateSyncUpdate, SyncStatus, -}; +pub use state_sync::{OnNoteReceived, OnNullifierReceived, StateSync, StateSyncUpdate}; /// Contains stats about the sync operation. pub struct SyncSummary { /// Block number up to which the client has been synced. pub block_num: BlockNumber, - /// IDs of new notes received. - pub received_notes: Vec, - /// IDs of tracked notes that received inclusion proofs. + /// IDs of notes that have been committed. pub committed_notes: Vec, /// IDs of notes that have been consumed. pub consumed_notes: Vec, @@ -100,7 +95,6 @@ pub struct SyncSummary { impl SyncSummary { pub fn new( block_num: BlockNumber, - received_notes: Vec, committed_notes: Vec, consumed_notes: Vec, updated_accounts: Vec, @@ -109,7 +103,6 @@ impl SyncSummary { ) -> Self { Self { block_num, - received_notes, committed_notes, consumed_notes, updated_accounts, @@ -121,7 +114,6 @@ impl SyncSummary { pub fn new_empty(block_num: BlockNumber) -> Self { Self { block_num, - received_notes: vec![], committed_notes: vec![], consumed_notes: vec![], updated_accounts: vec![], @@ -131,8 +123,7 @@ impl SyncSummary { } pub fn is_empty(&self) -> bool { - self.received_notes.is_empty() - && self.committed_notes.is_empty() + self.committed_notes.is_empty() && self.consumed_notes.is_empty() && self.updated_accounts.is_empty() && self.locked_accounts.is_empty() @@ -140,7 +131,6 @@ impl SyncSummary { pub fn combine_with(&mut self, mut other: Self) { self.block_num = max(self.block_num, other.block_num); - self.received_notes.append(&mut other.received_notes); self.committed_notes.append(&mut other.committed_notes); self.consumed_notes.append(&mut other.consumed_notes); self.updated_accounts.append(&mut other.updated_accounts); @@ -190,86 +180,50 @@ impl Client { self.rpc_api.clone(), Box::new({ let store_clone = self.store.clone(); - move |committed_notes, block_header| { - Box::pin(on_note_received(store_clone.clone(), committed_notes, block_header)) - } - }), - Box::new({ - let store_clone = self.store.clone(); - move |transaction_update| { - Box::pin(on_transaction_committed(store_clone.clone(), transaction_update)) - } - }), - Box::new({ - let store_clone = self.store.clone(); - move |nullifier_update| { - Box::pin(on_nullifier_received(store_clone.clone(), nullifier_update)) + move |committed_note, public_note| { + Box::pin(on_note_received(store_clone.clone(), committed_note, public_note)) } }), + Box::new(move |_committed_note| Box::pin(async { Ok(true) })), ); - let current_block_num = self.store.get_sync_height().await?; - let mut total_sync_summary = SyncSummary::new_empty(current_block_num); - - loop { - // Get current state of the client - let current_block_num = self.store.get_sync_height().await?; - let (current_block, has_relevant_notes) = self - .store - .get_block_header_by_num(current_block_num) - .await? - .expect("Current block should be in the store"); + // Get current state of the client + let accounts = self + .store + .get_account_headers() + .await? + .into_iter() + .map(|(acc_header, _)| acc_header) + .collect(); + + let note_tags: Vec = + self.store.get_unique_note_tags().await?.into_iter().collect(); + + let unspent_input_notes = self.store.get_input_notes(NoteFilter::Unspent).await?; + let unspent_output_notes = self.store.get_output_notes(NoteFilter::Unspent).await?; + + // Get the sync update from the network + let state_sync_update = state_sync + .sync_state( + self.build_current_partial_mmr().await?, + accounts, + note_tags, + unspent_input_notes, + unspent_output_notes, + ) + .await?; + + let sync_summary: SyncSummary = (&state_sync_update).into(); + + // Apply received and computed updates to the store + self.store + .apply_state_sync(state_sync_update) + .await + .map_err(ClientError::StoreError)?; - let accounts = self - .store - .get_account_headers() - .await? - .into_iter() - .map(|(acc_header, _)| acc_header) - .collect(); - - let note_tags: Vec = - self.store.get_unique_note_tags().await?.into_iter().collect(); - - let unspent_nullifiers = self.store.get_unspent_input_note_nullifiers().await?; - - // Get the sync update from the network - let status = state_sync - .sync_state_step( - current_block, - has_relevant_notes, - self.build_current_partial_mmr(false).await?, - accounts, - note_tags, - unspent_nullifiers, - ) - .await?; - - let (is_last_block, state_sync_update): (bool, StateSyncUpdate) = match status { - Some(s) => (s.is_last_block(), s.into()), - None => break, - }; - - let sync_summary: SyncSummary = (&state_sync_update).into(); - - let has_relevant_notes = - self.check_block_relevance(&state_sync_update.note_updates).await?; - - // Apply received and computed updates to the store - self.store - .apply_state_sync_step(state_sync_update, has_relevant_notes) - .await - .map_err(ClientError::StoreError)?; - - total_sync_summary.combine_with(sync_summary); - - if is_last_block { - break; - } - } self.update_mmr_data().await?; - Ok(total_sync_summary) + Ok(sync_summary) } } diff --git a/crates/rust-client/src/sync/state_sync.rs b/crates/rust-client/src/sync/state_sync.rs index 1b6336bbe..20beb2510 100644 --- a/crates/rust-client/src/sync/state_sync.rs +++ b/crates/rust-client/src/sync/state_sync.rs @@ -3,22 +3,23 @@ use core::{future::Future, pin::Pin}; use miden_objects::{ account::{Account, AccountHeader, AccountId}, - block::BlockHeader, + block::{BlockHeader, BlockNumber}, crypto::merkle::{InOrderIndex, MmrDelta, MmrPeaks, PartialMmr}, note::{NoteId, NoteInclusionProof, NoteTag, Nullifier}, + transaction::TransactionId, Digest, }; use tracing::info; -use super::{get_nullifier_prefix, NoteTagRecord, SyncSummary}; +use super::{block_header::BlockUpdates, get_nullifier_prefix, SyncSummary}; use crate::{ account::AccountUpdates, - note::NoteUpdates, + note::{NoteScreener, NoteUpdates}, rpc::{ domain::{note::CommittedNote, nullifier::NullifierUpdate, transaction::TransactionUpdate}, NodeRpcClient, }, - store::{InputNoteRecord, NoteFilter, Store, StoreError}, + store::{InputNoteRecord, NoteFilter, OutputNoteRecord, Store, StoreError}, transaction::TransactionUpdates, ClientError, }; @@ -26,31 +27,25 @@ use crate::{ // STATE SYNC UPDATE // ================================================================================================ +#[derive(Default)] /// Contains all information needed to apply the update in the store after syncing with the node. pub struct StateSyncUpdate { - /// The new block header, returned as part of the - /// [StateSyncInfo](crate::rpc::domain::sync::StateSyncInfo) - pub block_header: BlockHeader, - /// Information about note changes after the sync. + /// The block number of the last block that was synced. + pub block_num: BlockNumber, + /// New blocks and authentication nodes. + pub block_updates: BlockUpdates, + /// New and updated notes to be upserted in the store. pub note_updates: NoteUpdates, - /// Information about transaction changes after the sync. + /// Committed and discarded transactions after the sync. pub transaction_updates: TransactionUpdates, - /// New MMR peaks for the locally tracked MMR of the blockchain. - pub new_mmr_peaks: MmrPeaks, - /// New authentications nodes that are meant to be stored in order to authenticate block - /// headers. - pub new_authentication_nodes: Vec<(InOrderIndex, Digest)>, - /// Information abount account changes after the sync. + /// Public account updates and mismatched private accounts after the sync. pub account_updates: AccountUpdates, - /// Tag records that are no longer relevant. - pub tags_to_remove: Vec, } impl From<&StateSyncUpdate> for SyncSummary { fn from(value: &StateSyncUpdate) -> Self { SyncSummary::new( - value.block_header.block_num(), - value.note_updates.new_input_notes().iter().map(|n| n.id()).collect(), + value.block_num, value.note_updates.committed_note_ids().into_iter().collect(), value.note_updates.consumed_note_ids().into_iter().collect(), value @@ -75,62 +70,30 @@ impl From<&StateSyncUpdate> for SyncSummary { } } -/// Gives information about the status of the sync process after a step. -pub enum SyncStatus { - SyncedToLastBlock(StateSyncUpdate), - SyncedToBlock(StateSyncUpdate), -} - -impl SyncStatus { - pub fn is_last_block(&self) -> bool { - matches!(self, SyncStatus::SyncedToLastBlock(_)) - } -} - -impl From for StateSyncUpdate { - fn from(value: SyncStatus) -> StateSyncUpdate { - match value { - SyncStatus::SyncedToLastBlock(update) => update, - SyncStatus::SyncedToBlock(update) => update, - } - } -} - // SYNC CALLBACKS // ================================================================================================ /// Callback to be executed when a new note inclusion is received in the sync response. It receives -/// the committed note received from the node and the block header in which the note was included. -/// It returns the note updates that should be applied to the store and a list of public note IDs -/// that should be queried from the node and start being tracked. +/// the committed note received from the node, the block header in which the note was included and +/// the list of public notes that were included in the block. +/// +/// It returns two optional notes (one input and one output) that should be updated in the store and +/// a flag indicating if the block is relevant to the client. pub type OnNoteReceived = Box< dyn Fn( CommittedNote, - BlockHeader, - ) -> Pin), ClientError>>>>, + Option, + ) -> Pin>>>, >; -/// Callback to be executed when a transaction is marked committed in the sync response. It receives -/// the transaction update received from the node. It returns the note updates and transaction -/// updates that should be applied to the store as a result of the transaction being committed. -pub type OnTransactionCommitted = Box< - dyn Fn( - TransactionUpdate, - ) - -> Pin>>>, ->; - -/// Callback to be executed when a nullifier is received in the sync response. If a note was -/// consumed by a committed transaction provided in the [OnTransactionCommitted] callback, its -/// nullifier will not be passed to this callback. It receives the nullifier update received from -/// the node. It returns the note updates and transaction updates that should be applied to the -/// store as a result of the nullifier being received. -pub type OnNullifierReceived = Box< - dyn Fn( - NullifierUpdate, - ) - -> Pin>>>, ->; +/// Callback to be executed when a nullifier is received in the sync response. It receives the +/// nullifier update received from the node and the list of transaction updates that were committed +/// in the block. +/// +/// It returns two optional notes (one input and one output) that should be updated in the store and +/// an optional transaction ID if a transaction should be discarded. +pub type OnNullifierReceived = + Box Pin>>>>; // STATE SYNC // ================================================================================================ @@ -145,10 +108,11 @@ pub struct StateSync { rpc_api: Arc, /// Callback to be executed when a new note inclusion is received. on_note_received: OnNoteReceived, - /// Callback to be executed when a transaction is committed. - on_transaction_committed: OnTransactionCommitted, /// Callback to be executed when a nullifier is received. on_nullifier_received: OnNullifierReceived, + /// The state sync update that will be returned after the sync process is completed. It + /// agregates all the updates that come from each sync step. + state_sync_update: StateSyncUpdate, } impl StateSync { @@ -158,51 +122,37 @@ impl StateSync { /// /// * `rpc_api` - The RPC client used to communicate with the node. /// * `on_note_received` - A callback to be executed when a new note inclusion is received. - /// * `on_committed_transaction` - A callback to be executed when a transaction is committed. /// * `on_nullifier_received` - A callback to be executed when a nullifier is received. pub fn new( rpc_api: Arc, on_note_received: OnNoteReceived, - on_transaction_committed: OnTransactionCommitted, on_nullifier_received: OnNullifierReceived, ) -> Self { Self { rpc_api, on_note_received, - on_transaction_committed, on_nullifier_received, + state_sync_update: StateSyncUpdate::default(), } } - /// Executes a single step of the state sync process, returning the changes that should be - /// applied to the store. + /// Executes a single step of the state sync process, returning `true` if the client should + /// continue syncing and `false` if the client has reached the chain tip. /// /// A step in this context means a single request to the node to get the next relevant block and /// the changes that happened in it. This block may not be the last one in the chain and /// the client may need to call this method multiple times until it reaches the chain tip. - /// Wheter or not the client has reached the chain tip is indicated by the returned - /// [SyncStatus] variant. `None` is returned if the client is already synced with the chain tip - /// and there are no new changes. /// - /// # Arguments - /// * `current_block` - The latest tracked block header. - /// * `current_block_has_relevant_notes` - A flag indicating if the current block has notes that - /// are relevant to the client. This is used to determine whether new MMR authentication nodes - /// are stored for this block. - /// * `current_partial_mmr` - The current partial MMR. - /// * `accounts` - The headers of tracked accounts. - /// * `note_tags` - The note tags to be used in the sync state request. - /// * `unspent_nullifiers` - The nullifiers of tracked notes that haven't been consumed. - pub async fn sync_state_step( - &self, - current_block: BlockHeader, - current_block_has_relevant_notes: bool, - current_partial_mmr: PartialMmr, - accounts: Vec, - note_tags: Vec, - unspent_nullifiers: Vec, - ) -> Result, ClientError> { - let current_block_num = current_block.block_num(); + /// The `sync_state_update` field of the struct will be updated with the new changes from this + /// step. + async fn sync_state_step( + &mut self, + current_partial_mmr: &mut PartialMmr, + accounts: &[AccountHeader], + note_tags: &[NoteTag], + unspent_nullifiers: &[Nullifier], + ) -> Result { + let current_block_num = (current_partial_mmr.num_leaves() as u32 - 1).into(); let account_ids: Vec = accounts.iter().map(|acc| acc.id()).collect(); // To receive information about added nullifiers, we reduce them to the higher 16 bits @@ -214,18 +164,19 @@ impl StateSync { let response = self .rpc_api - .sync_state(current_block_num, &account_ids, ¬e_tags, &nullifiers_tags) + .sync_state(current_block_num, &account_ids, note_tags, &nullifiers_tags) .await?; + self.state_sync_update.block_num = response.block_header.block_num(); + // We don't need to continue if the chain has not advanced, there are no new changes if response.block_header.block_num() == current_block_num { - return Ok(None); + return Ok(false); } - let account_updates = - self.account_state_sync(&accounts, &response.account_hash_updates).await?; + self.account_state_sync(accounts, &response.account_hash_updates).await?; - let (note_updates, transaction_updates, tags_to_remove) = self + let found_relevant_note = self .note_state_sync( response.note_inclusions, response.transactions, @@ -235,34 +186,81 @@ impl StateSync { .await?; let (new_mmr_peaks, new_authentication_nodes) = apply_mmr_changes( - current_block, - current_block_has_relevant_notes, + response.block_header, + found_relevant_note, current_partial_mmr, response.mmr_delta, ) .await?; - let update = StateSyncUpdate { - block_header: response.block_header, - note_updates, - transaction_updates, - new_mmr_peaks, + self.state_sync_update.block_updates.extend(BlockUpdates { + block_headers: vec![(response.block_header, found_relevant_note, new_mmr_peaks)], new_authentication_nodes, - account_updates, - tags_to_remove, - }; + }); if response.chain_tip == response.block_header.block_num() { - Ok(Some(SyncStatus::SyncedToLastBlock(update))) + Ok(false) } else { - Ok(Some(SyncStatus::SyncedToBlock(update))) + Ok(true) } } + /// Syncs the state of the client with the chain tip of the node, returning the updates that + /// should be applied to the store. + /// + /// # Arguments + /// * `current_block` - The latest tracked block header. + /// * `current_block_has_relevant_notes` - A flag indicating if the current block has notes that + /// are relevant to the client. This is used to determine whether new MMR authentication nodes + /// are stored for this block. + /// * `current_partial_mmr` - The current partial MMR. + /// * `accounts` - The headers of tracked accounts. + /// * `note_tags` - The note tags to be used in the sync state request. + /// * `unspent_nullifiers` - The nullifiers of tracked notes that haven't been consumed. + pub async fn sync_state( + mut self, + mut current_partial_mmr: PartialMmr, + accounts: Vec, + note_tags: Vec, + unspent_input_notes: Vec, + unspent_output_notes: Vec, + ) -> Result { + let mut unspent_nullifiers: Vec = unspent_input_notes + .iter() + .map(|note| note.nullifier()) + .chain(unspent_output_notes.iter().filter_map(|note| note.nullifier())) + .collect(); + + self.state_sync_update.note_updates = + NoteUpdates::new(unspent_input_notes, unspent_output_notes); + + while self + .sync_state_step(&mut current_partial_mmr, &accounts, ¬e_tags, &unspent_nullifiers) + .await? + { + // New nullfiers should be added for new untracked notes that were added in previous + // steps + unspent_nullifiers.append( + &mut self + .state_sync_update + .note_updates + .updated_input_notes() + .filter(|note| { + note.is_committed() && !unspent_nullifiers.contains(¬e.nullifier()) + }) + .map(|note| note.nullifier()) + .collect::>(), + ); + } + + Ok(self.state_sync_update) + } + // HELPERS // -------------------------------------------------------------------------------------------- - /// Compares the state of tracked accounts with the updates received from the node and returns + /// Compares the state of tracked accounts with the updates received from the node and updates + /// the `state_sync_update` with the details of /// the accounts that need to be updated. /// /// When a mismatch is detected, two scenarios are possible: @@ -271,10 +269,10 @@ impl StateSync { /// * If the account is private it will be marked as mismatched and the client will need to /// handle it (it could be a stale account state or a reason to lock the account). async fn account_state_sync( - &self, + &mut self, accounts: &[AccountHeader], account_hash_updates: &[(AccountId, Digest)], - ) -> Result { + ) -> Result<(), ClientError> { let (public_accounts, private_accounts): (Vec<_>, Vec<_>) = accounts.iter().partition(|account_header| account_header.id().is_public()); @@ -291,7 +289,11 @@ impl StateSync { .cloned() .collect::>(); - Ok(AccountUpdates::new(updated_public_accounts, mismatched_private_accounts)) + self.state_sync_update + .account_updates + .extend(AccountUpdates::new(updated_public_accounts, mismatched_private_accounts)); + + Ok(()) } /// Queries the node for the latest state of the public accounts that don't match the current @@ -320,7 +322,8 @@ impl StateSync { } /// Applies the changes received from the sync response to the notes and transactions tracked - /// by the client. It returns the updates that should be applied to the store. + /// by the client and updates the + /// `state_sync_update` accordingly. /// /// This method uses the callbacks provided to the [StateSync] component to apply the changes. /// @@ -336,66 +339,66 @@ impl StateSync { /// * Local tracked transactions that were discarded because the notes that they were processing /// were nullified by an another transaction. async fn note_state_sync( - &self, + &mut self, note_inclusions: Vec, transactions: Vec, - mut nullifiers: Vec, + nullifiers: Vec, block_header: BlockHeader, - ) -> Result<(NoteUpdates, TransactionUpdates, Vec), ClientError> { - let mut note_updates = NoteUpdates::default(); - let mut public_note_ids = vec![]; - - for committed_note in note_inclusions { - let (new_note_update, new_note_ids) = - (self.on_note_received)(committed_note, block_header).await?; - note_updates.extend(new_note_update); - public_note_ids.extend(new_note_ids); - } - - let new_public_notes = - self.fetch_public_note_details(&public_note_ids, &block_header).await?; - - note_updates.extend(NoteUpdates::new(new_public_notes, vec![], vec![], vec![])); - - // We can remove tags from notes that got committed - let tags_to_remove = note_updates - .updated_input_notes() + ) -> Result { + let public_note_ids: Vec = note_inclusions .iter() - .filter(|note| note.is_committed()) - .map(|note| { - NoteTagRecord::with_note_source( - note.metadata().expect("Committed note should have metadata").tag(), - note.id(), - ) - }) + .filter_map(|note| (!note.metadata().is_private()).then_some(*note.note_id())) .collect(); - let mut transaction_updates = TransactionUpdates::default(); - - for transaction_update in transactions { - let (new_note_update, new_transaction_update) = - (self.on_transaction_committed)(transaction_update).await?; + let mut found_relevant_note = false; - // Remove nullifiers if they were consumed by the transaction - nullifiers.retain(|nullifier| { - !new_note_update - .updated_input_notes() - .iter() - .any(|note| note.nullifier() == nullifier.nullifier) - }); - - note_updates.extend(new_note_update); - transaction_updates.extend(new_transaction_update); + // Process note inclusions + let new_public_notes = + Arc::new(self.fetch_public_note_details(&public_note_ids, &block_header).await?); + for committed_note in note_inclusions { + let public_note = new_public_notes + .iter() + .find(|note| ¬e.id() == committed_note.note_id()) + .cloned(); + if (self.on_note_received)(committed_note.clone(), public_note.clone()).await? { + found_relevant_note = true; + + if let Some(public_note) = public_note { + self.state_sync_update.note_updates.insert_updates(Some(public_note), None); + } + + committed_state_transions( + &mut self.state_sync_update.note_updates, + committed_note, + block_header, + ) + .await?; + } } + // Process nullifiers for nullifier_update in nullifiers { - let (new_note_update, new_transaction_update) = - (self.on_nullifier_received)(nullifier_update).await?; - note_updates.extend(new_note_update); - transaction_updates.extend(new_transaction_update); + if (self.on_nullifier_received)(nullifier_update.clone()).await? { + let discarded_transaction = nullfier_state_transitions( + &mut self.state_sync_update.note_updates, + nullifier_update, + &transactions, + ) + .await?; + + if let Some(transaction_id) = discarded_transaction { + self.state_sync_update + .transaction_updates + .discarded_transaction(transaction_id); + } + } } - Ok((note_updates, transaction_updates, tags_to_remove)) + self.state_sync_update + .transaction_updates + .extend(TransactionUpdates::new(transactions, vec![])); + + Ok(found_relevant_note) } /// Queries the node for all received notes that aren't being locally tracked in the client. @@ -427,180 +430,132 @@ impl StateSync { /// Applies changes to the current MMR structure, returns the updated [MmrPeaks] and the /// authentication nodes for leaves we track. -pub(crate) async fn apply_mmr_changes( - current_block: BlockHeader, - current_block_has_relevant_notes: bool, - mut current_partial_mmr: PartialMmr, +async fn apply_mmr_changes( + new_block: BlockHeader, + new_block_has_relevant_notes: bool, + current_partial_mmr: &mut PartialMmr, mmr_delta: MmrDelta, ) -> Result<(MmrPeaks, Vec<(InOrderIndex, Digest)>), ClientError> { // First, apply curent_block to the MMR. This is needed as the MMR delta received from the // node doesn't contain the request block itself. - let new_authentication_nodes = current_partial_mmr - .add(current_block.hash(), current_block_has_relevant_notes) - .into_iter(); + // let new_authentication_nodes = current_partial_mmr + // .add(current_block.hash(), current_block_has_relevant_notes) + // .into_iter(); // Apply the MMR delta to bring MMR to forest equal to chain tip - let new_authentication_nodes: Vec<(InOrderIndex, Digest)> = current_partial_mmr - .apply(mmr_delta) - .map_err(StoreError::MmrError)? - .into_iter() - .chain(new_authentication_nodes) - .collect(); - - Ok((current_partial_mmr.peaks(), new_authentication_nodes)) + let mut new_authentication_nodes: Vec<(InOrderIndex, Digest)> = + current_partial_mmr.apply(mmr_delta).map_err(StoreError::MmrError)?; + + let new_peaks = current_partial_mmr.peaks(); + + new_authentication_nodes + .append(&mut current_partial_mmr.add(new_block.hash(), new_block_has_relevant_notes)); + + Ok((new_peaks, new_authentication_nodes)) } // DEFAULT CALLBACK IMPLEMENTATIONS // ================================================================================================ /// Default implementation of the [OnNoteReceived] callback. It queries the store for the committed -/// note and updates the note records accordingly. If the note is not being tracked, it returns the -/// note ID to be queried from the node so it can be queried from the node and tracked. -pub async fn on_note_received( - store: Arc, +/// note and updates it accordingly. If the note wasn't being tracked but it came in the sync +/// response, it is also returned so it can be inserted in the store. The method also returns a +/// flag indicating if the block is relevant to the client. +async fn committed_state_transions( + note_updates: &mut NoteUpdates, committed_note: CommittedNote, block_header: BlockHeader, -) -> Result<(NoteUpdates, Vec), ClientError> { +) -> Result<(), ClientError> { let inclusion_proof = NoteInclusionProof::new( block_header.block_num(), committed_note.note_index(), committed_note.merkle_path().clone(), )?; - let mut updated_input_notes = vec![]; - let mut updated_output_notes = vec![]; - let mut new_note_ids = vec![]; - - if let Some(mut input_note_record) = store - .get_input_notes(NoteFilter::List(vec![*committed_note.note_id()])) - .await? - .pop() - { + if let Some(input_note_record) = note_updates.get_input_note_by_id(*committed_note.note_id()) { // The note belongs to our locally tracked set of input notes - let inclusion_proof_received = input_note_record + input_note_record .inclusion_proof_received(inclusion_proof.clone(), committed_note.metadata())?; - let block_header_received = input_note_record.block_header_received(block_header)?; - - if inclusion_proof_received || block_header_received { - updated_input_notes.push(input_note_record); - } + input_note_record.block_header_received(block_header)?; } - if let Some(mut output_note_record) = store - .get_output_notes(NoteFilter::List(vec![*committed_note.note_id()])) - .await? - .pop() + if let Some(output_note_record) = note_updates.get_output_note_by_id(*committed_note.note_id()) { // The note belongs to our locally tracked set of output notes - if output_note_record.inclusion_proof_received(inclusion_proof.clone())? { - updated_output_notes.push(output_note_record); - } - } - - if updated_input_notes.is_empty() && updated_output_notes.is_empty() { - // The note is public and we are not tracking it, push to the list of IDs to query - new_note_ids.push(*committed_note.note_id()); + output_note_record.inclusion_proof_received(inclusion_proof.clone())?; } - Ok(( - NoteUpdates::new(vec![], vec![], updated_input_notes, updated_output_notes), - new_note_ids, - )) + Ok(()) } -/// Default implementation of the [OnTransactionCommitted] callback. It queries the store for the -/// input notes that were consumed by the transaction and updates the note records accordingly. It -/// also returns the committed transaction update to be applied to the store. -pub async fn on_transaction_committed( - store: Arc, - transaction_update: TransactionUpdate, -) -> Result<(NoteUpdates, TransactionUpdates), ClientError> { - // TODO: This could be improved if we add a filter to get only notes that are being processed by - // a specific transaction - let processing_notes = store.get_input_notes(NoteFilter::Processing).await?; - let consumed_input_notes: Vec = processing_notes - .into_iter() - .filter(|note_record| { - note_record.consumer_transaction_id() == Some(&transaction_update.transaction_id) - }) - .collect(); - - let consumed_output_notes = store - .get_output_notes(NoteFilter::Nullifiers( - consumed_input_notes.iter().map(|n| n.nullifier()).collect(), - )) - .await?; +/// Default implementation of the [OnNullifierReceived] callback. It queries the store for the notes +/// that match the nullifier and updates the note records accordingly. It also returns an optional +/// transaction ID that should be discarded. +async fn nullfier_state_transitions( + note_updates: &mut NoteUpdates, + nullifier_update: NullifierUpdate, + transaction_updates: &[TransactionUpdate], +) -> Result, ClientError> { + let mut discarded_transaction = None; - let mut updated_input_notes = vec![]; - let mut updated_output_notes = vec![]; - for mut input_note_record in consumed_input_notes { - if input_note_record.transaction_committed( - transaction_update.transaction_id, - transaction_update.block_num, - )? { - updated_input_notes.push(input_note_record); + if let Some(input_note_record) = + note_updates.get_input_note_by_nullifier(nullifier_update.nullifier) + { + if let Some(consumer_transaction) = transaction_updates.iter().find(|t| { + input_note_record + .consumer_transaction_id() + .map_or(false, |id| id == &t.transaction_id) + }) { + // The note was being processed by a local transaction that just got committed + input_note_record.transaction_committed( + consumer_transaction.transaction_id, + consumer_transaction.block_num, + )?; + } else { + // The note was consumed by an external transaction + if let Some(id) = input_note_record.consumer_transaction_id() { + // The note was being processed by a local transaction that didn't end up being + // committed so it should be discarded + discarded_transaction.replace(*id); + } + input_note_record + .consumed_externally(nullifier_update.nullifier, nullifier_update.block_num)?; } } - for mut output_note_record in consumed_output_notes { - // SAFETY: Output notes were queried from a nullifier list and should have a nullifier - let nullifier = output_note_record.nullifier().unwrap(); - if output_note_record.nullifier_received(nullifier, transaction_update.block_num)? { - updated_output_notes.push(output_note_record); - } + if let Some(output_note_record) = + note_updates.get_output_note_by_nullifier(nullifier_update.nullifier) + { + output_note_record + .nullifier_received(nullifier_update.nullifier, nullifier_update.block_num)?; } - Ok(( - NoteUpdates::new(vec![], vec![], updated_input_notes, updated_output_notes), - TransactionUpdates::new(vec![transaction_update], vec![]), - )) + Ok(discarded_transaction) } -/// Default implementation of the [OnNullifierReceived] callback. It queries the store for the notes -/// that match the nullifier and updates the note records accordingly. It also returns the -/// transactions that should be discarded as they weren't committed when the nullifier was received. -pub async fn on_nullifier_received( +pub async fn on_note_received( store: Arc, - nullifier_update: NullifierUpdate, -) -> Result<(NoteUpdates, TransactionUpdates), ClientError> { - let mut discarded_transactions = vec![]; - let mut updated_input_notes = vec![]; - let mut updated_output_notes = vec![]; - - if let Some(mut input_note_record) = store - .get_input_notes(NoteFilter::Nullifiers(vec![nullifier_update.nullifier])) - .await? - .pop() - { - if input_note_record.is_processing() { - discarded_transactions.push( - *input_note_record - .consumer_transaction_id() - .expect("Processing note should have consumer transaction id"), - ); - } - - if input_note_record - .consumed_externally(nullifier_update.nullifier, nullifier_update.block_num)? - { - updated_input_notes.push(input_note_record); - } - } + committed_note: CommittedNote, + public_note: Option, +) -> Result { + let note_id = *committed_note.note_id(); + let note_screener = NoteScreener::new(store.clone()); - if let Some(mut output_note_record) = store - .get_output_notes(NoteFilter::Nullifiers(vec![nullifier_update.nullifier])) - .await? - .pop() + if !store.get_input_notes(NoteFilter::Unique(note_id)).await?.is_empty() + || !store.get_output_notes(NoteFilter::Unique(note_id)).await?.is_empty() { - if output_note_record - .nullifier_received(nullifier_update.nullifier, nullifier_update.block_num)? - { - updated_output_notes.push(output_note_record); - } - } + // The note is being tracked by the client so it is relevant + Ok(true) + } else if let Some(public_note) = public_note { + // The note is not being tracked by the client and is public so we can screen it + let new_note_relevance = note_screener + .check_relevance(&public_note.try_into().expect("Public notes should contain metadata")) + .await?; - Ok(( - NoteUpdates::new(vec![], vec![], updated_input_notes, updated_output_notes), - TransactionUpdates::new(vec![], discarded_transactions), - )) + Ok(!new_note_relevance.is_empty()) + } else { + // The note is not being tracked by the client and is private so we can't determine if it + // is relevant + Ok(false) + } } diff --git a/crates/rust-client/src/tests.rs b/crates/rust-client/src/tests.rs index babad8f49..2239bfc9d 100644 --- a/crates/rust-client/src/tests.rs +++ b/crates/rust-client/src/tests.rs @@ -374,7 +374,7 @@ async fn test_sync_state_mmr() { ); // Try reconstructing the chain_mmr from what's in the database - let partial_mmr = client.build_current_partial_mmr(true).await.unwrap(); + let partial_mmr = client.build_current_partial_mmr().await.unwrap(); assert_eq!(partial_mmr.forest(), 6); assert!(partial_mmr.open(0).unwrap().is_some()); // Account anchor block assert!(partial_mmr.open(1).unwrap().is_some()); diff --git a/crates/rust-client/src/transaction/mod.rs b/crates/rust-client/src/transaction/mod.rs index 6ecf951eb..e9ab67438 100644 --- a/crates/rust-client/src/transaction/mod.rs +++ b/crates/rust-client/src/transaction/mod.rs @@ -155,6 +155,11 @@ impl TransactionUpdates { pub fn discarded_transactions(&self) -> &[TransactionId] { &self.discarded_transactions } + + /// Inserts a committed transaction into the transaction updates. + pub fn discarded_transaction(&mut self, transaction_id: TransactionId) { + self.discarded_transactions.push(transaction_id); + } } // TRANSACTION RESULT @@ -363,10 +368,8 @@ impl TransactionStoreUpdate { executed_transaction, updated_account, note_updates: NoteUpdates::new( - created_input_notes, + [created_input_notes, updated_input_notes].concat(), created_output_notes, - updated_input_notes, - vec![], ), new_tags, } @@ -910,7 +913,7 @@ impl Client { let summary = self.sync_state().await?; if summary.block_num != block_num { - let mut current_partial_mmr = self.build_current_partial_mmr(true).await?; + let mut current_partial_mmr = self.build_current_partial_mmr().await?; self.get_and_store_authenticated_block(block_num, &mut current_partial_mmr) .await?; } diff --git a/crates/web-client/package-lock.json b/crates/web-client/package-lock.json index 9c504c5b3..e8b9125bc 100644 --- a/crates/web-client/package-lock.json +++ b/crates/web-client/package-lock.json @@ -1,12 +1,12 @@ { "name": "@demox-labs/miden-sdk", - "version": "0.6.1-next.3", + "version": "0.6.1-next.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@demox-labs/miden-sdk", - "version": "0.6.1-next.3", + "version": "0.6.1-next.4", "dependencies": { "chai-as-promised": "^8.0.0", "dexie": "^4.0.1", diff --git a/crates/web-client/src/models/sync_summary.rs b/crates/web-client/src/models/sync_summary.rs index 34d1bf4e1..67ac51fda 100644 --- a/crates/web-client/src/models/sync_summary.rs +++ b/crates/web-client/src/models/sync_summary.rs @@ -12,10 +12,6 @@ impl SyncSummary { self.0.block_num.as_u32() } - pub fn received_notes(&self) -> Vec { - self.0.received_notes.iter().map(|note_id| note_id.into()).collect() - } - pub fn committed_notes(&self) -> Vec { self.0.committed_notes.iter().map(|note_id| note_id.into()).collect() } diff --git a/tests/integration/main.rs b/tests/integration/main.rs index 21c49ed83..1d8e80e8b 100644 --- a/tests/integration/main.rs +++ b/tests/integration/main.rs @@ -850,8 +850,7 @@ async fn test_sync_detail_values() { // Second client sync should have new note let new_details = client2.sync_state().await.unwrap(); - assert_eq!(new_details.received_notes.len(), 1); - assert_eq!(new_details.committed_notes.len(), 0); + assert_eq!(new_details.committed_notes.len(), 1); assert_eq!(new_details.consumed_notes.len(), 0); assert_eq!(new_details.updated_accounts.len(), 0); @@ -861,7 +860,6 @@ async fn test_sync_detail_values() { // First client sync should have a new nullifier as the note was consumed let new_details = client1.sync_state().await.unwrap(); - assert_eq!(new_details.received_notes.len(), 0); assert_eq!(new_details.committed_notes.len(), 0); assert_eq!(new_details.consumed_notes.len(), 1); } diff --git a/tests/integration/onchain_tests.rs b/tests/integration/onchain_tests.rs index 2761bc3a3..3e2dc5cef 100644 --- a/tests/integration/onchain_tests.rs +++ b/tests/integration/onchain_tests.rs @@ -83,9 +83,33 @@ async fn test_onchain_notes_flow() { .build(); execute_tx_and_sync(&mut client_2, basic_wallet_1.id(), tx_request).await; + // Create a note for client 3 that is already consumed before syncing + let tx_request = TransactionRequestBuilder::pay_to_id( + PaymentTransactionData::new( + vec![p2id_asset.into()], + basic_wallet_1.id(), + basic_wallet_2.id(), + ), + Some(1.into()), + NoteType::Public, + client_2.rng(), + ) + .unwrap() + .build(); + let note = tx_request.expected_output_notes().next().unwrap().clone(); + execute_tx_and_sync(&mut client_2, basic_wallet_1.id(), tx_request).await; + + let tx_request = TransactionRequestBuilder::consume_notes(vec![note.id()]).build(); + execute_tx_and_sync(&mut client_2, basic_wallet_1.id(), tx_request).await; + // sync client 3 (basic account 2) client_3.sync_state().await.unwrap(); - // client 3 should only have one note + + // client 3 should have two notes, the one directed to them and the one consumed by client 2 + // (which should come from the tag added) + assert_eq!(client_3.get_input_notes(NoteFilter::Committed).await.unwrap().len(), 1); + assert_eq!(client_3.get_input_notes(NoteFilter::Consumed).await.unwrap().len(), 1); + let note = client_3 .get_input_notes(NoteFilter::Committed) .await From c1281f0162fc5c34c2a471f12627449dce82cfdf Mon Sep 17 00:00:00 2001 From: tomyrd Date: Mon, 10 Feb 2025 11:09:57 -0300 Subject: [PATCH 22/34] fix: move and update note tag test --- .../integration/custom_transactions_tests.rs | 76 ++++++++++++++++++- tests/integration/onchain_tests.rs | 63 +-------------- 2 files changed, 77 insertions(+), 62 deletions(-) diff --git a/tests/integration/custom_transactions_tests.rs b/tests/integration/custom_transactions_tests.rs index 9c02a6b96..a9e70b61c 100644 --- a/tests/integration/custom_transactions_tests.rs +++ b/tests/integration/custom_transactions_tests.rs @@ -1,6 +1,7 @@ use miden_client::{ note::NoteExecutionHint, - transaction::{TransactionRequest, TransactionRequestBuilder}, + store::NoteFilter, + transaction::{InputNote, TransactionRequest, TransactionRequestBuilder}, utils::{Deserializable, Serializable}, ZERO, }; @@ -220,6 +221,79 @@ async fn test_merkle_store() { client.sync_state().await.unwrap(); } +#[tokio::test] +async fn test_onchain_notes_sync_with_tag() { + // Client 1 has an private faucet which will mint an onchain note for client 2 + let mut client_1 = create_test_client().await; + // Client 2 will be used to sync and check that by adding the tag we can still fetch notes + // whose tag doesn't necessarily match any of its accounts + let mut client_2 = create_test_client().await; + // Client 3 will be the control client. We won't add any tags and expect the note not to be + // fetched + let mut client_3 = create_test_client().await; + wait_for_node(&mut client_3).await; + + // Create accounts + let (basic_account_1, _) = + insert_new_wallet(&mut client_1, AccountStorageMode::Private).await.unwrap(); + + insert_new_wallet(&mut client_2, AccountStorageMode::Private).await.unwrap(); + + client_1.sync_state().await.unwrap(); + client_2.sync_state().await.unwrap(); + client_3.sync_state().await.unwrap(); + + // Create the custom note + let note_script = " + begin + push.1 push.1 + assert_eq + end + "; + let note_script = client_1.compile_note_script(note_script).unwrap(); + let inputs = NoteInputs::new(vec![]).unwrap(); + let serial_num = client_1.rng().draw_word(); + let note_metadata = NoteMetadata::new( + basic_account_1.id(), + NoteType::Public, + NoteTag::from_account_id(basic_account_1.id(), NoteExecutionMode::Local).unwrap(), + NoteExecutionHint::None, + Default::default(), + ) + .unwrap(); + let note_assets = NoteAssets::new(vec![]).unwrap(); + let note_recipient = NoteRecipient::new(serial_num, note_script, inputs); + let note = Note::new(note_assets, note_metadata, note_recipient); + + // Send transaction and wait for it to be committed + let tx_request = TransactionRequestBuilder::new() + .with_own_output_notes(vec![OutputNote::Full(note.clone())]) + .unwrap() + .build(); + + let note = tx_request.expected_output_notes().next().unwrap().clone(); + execute_tx_and_sync(&mut client_1, basic_account_1.id(), tx_request).await; + + // Load tag into client 2 + client_2 + .add_note_tag( + NoteTag::from_account_id(basic_account_1.id(), NoteExecutionMode::Local).unwrap(), + ) + .await + .unwrap(); + + // Client 2's account should receive the note here: + client_2.sync_state().await.unwrap(); + client_3.sync_state().await.unwrap(); + + // Assert that the note is the same + let received_note: InputNote = + client_2.get_input_note(note.id()).await.unwrap().unwrap().try_into().unwrap(); + assert_eq!(received_note.note().hash(), note.hash()); + assert_eq!(received_note.note(), ¬e); + assert!(client_3.get_input_notes(NoteFilter::All).await.unwrap().is_empty()); +} + async fn mint_custom_note( client: &mut TestClient, faucet_account_id: AccountId, diff --git a/tests/integration/onchain_tests.rs b/tests/integration/onchain_tests.rs index 3e2dc5cef..908225507 100644 --- a/tests/integration/onchain_tests.rs +++ b/tests/integration/onchain_tests.rs @@ -3,9 +3,9 @@ use miden_client::{ transaction::{PaymentTransactionData, TransactionRequestBuilder}, }; use miden_objects::{ - account::{AccountId, AccountStorageMode}, + account::AccountStorageMode, asset::{Asset, FungibleAsset}, - note::{NoteFile, NoteTag, NoteType}, + note::{NoteFile, NoteType}, transaction::InputNote, }; @@ -299,62 +299,3 @@ async fn test_onchain_accounts() { assert_eq!(new_from_account_balance, from_account_balance - TRANSFER_AMOUNT); assert_eq!(new_to_account_balance, to_account_balance + TRANSFER_AMOUNT); } - -#[tokio::test] -async fn test_onchain_notes_sync_with_tag() { - // Client 1 has an private faucet which will mint an onchain note for client 2 - let mut client_1 = create_test_client().await; - // Client 2 will be used to sync and check that by adding the tag we can still fetch notes - // whose tag doesn't necessarily match any of its accounts - let mut client_2 = create_test_client().await; - // Client 3 will be the control client. We won't add any tags and expect the note not to be - // fetched - let mut client_3 = create_test_client().await; - wait_for_node(&mut client_3).await; - - // Create faucet account - let (faucet_account, _) = - insert_new_fungible_faucet(&mut client_1, AccountStorageMode::Private) - .await - .unwrap(); - - client_1.sync_state().await.unwrap(); - client_2.sync_state().await.unwrap(); - client_3.sync_state().await.unwrap(); - - let target_account_id = AccountId::try_from(ACCOUNT_ID_REGULAR).unwrap(); - - let tx_request = TransactionRequestBuilder::mint_fungible_asset( - FungibleAsset::new(faucet_account.id(), MINT_AMOUNT).unwrap(), - target_account_id, - NoteType::Public, - client_1.rng(), - ) - .unwrap() - .build(); - let note = tx_request.expected_output_notes().next().unwrap().clone(); - execute_tx_and_sync(&mut client_1, faucet_account.id(), tx_request).await; - - // Load tag into client 2 - client_2 - .add_note_tag( - NoteTag::from_account_id( - target_account_id, - miden_objects::note::NoteExecutionMode::Local, - ) - .unwrap(), - ) - .await - .unwrap(); - - // Client 2's account should receive the note here: - client_2.sync_state().await.unwrap(); - client_3.sync_state().await.unwrap(); - - // Assert that the note is the same - let received_note: InputNote = - client_2.get_input_note(note.id()).await.unwrap().unwrap().try_into().unwrap(); - assert_eq!(received_note.note().hash(), note.hash()); - assert_eq!(received_note.note(), ¬e); - assert!(client_3.get_input_notes(NoteFilter::All).await.unwrap().is_empty()); -} From 12e47060685ee72a2055f6faabb12d7351059990 Mon Sep 17 00:00:00 2001 From: tomyrd Date: Tue, 11 Feb 2025 15:43:17 -0300 Subject: [PATCH 23/34] review: improve `StateSync` comments --- .../rust-client/src/rpc/domain/transaction.rs | 2 +- crates/rust-client/src/sync/mod.rs | 130 ++++++------ crates/rust-client/src/sync/state_sync.rs | 187 ++++++++---------- .../rust-client/src/sync/state_sync_update.rs | 50 +++++ 4 files changed, 197 insertions(+), 172 deletions(-) create mode 100644 crates/rust-client/src/sync/state_sync_update.rs diff --git a/crates/rust-client/src/rpc/domain/transaction.rs b/crates/rust-client/src/rpc/domain/transaction.rs index 0496e9ce9..288863b48 100644 --- a/crates/rust-client/src/rpc/domain/transaction.rs +++ b/crates/rust-client/src/rpc/domain/transaction.rs @@ -34,8 +34,8 @@ impl TryFrom for TransactionId { // TRANSACTION UPDATE // ================================================================================================ -#[derive(Debug, Clone)] /// Represents a transaction that was included in the node at a certain block. +#[derive(Debug, Clone)] pub struct TransactionUpdate { /// The transaction identifier. pub transaction_id: TransactionId, diff --git a/crates/rust-client/src/sync/mod.rs b/crates/rust-client/src/sync/mod.rs index 644202c25..199c04e78 100644 --- a/crates/rust-client/src/sync/mod.rs +++ b/crates/rust-client/src/sync/mod.rs @@ -74,69 +74,10 @@ mod tag; pub use tag::{NoteTagRecord, NoteTagSource}; mod state_sync; -pub use state_sync::{OnNoteReceived, OnNullifierReceived, StateSync, StateSyncUpdate}; +pub use state_sync::{OnNoteReceived, OnNullifierReceived, StateSync}; -/// Contains stats about the sync operation. -pub struct SyncSummary { - /// Block number up to which the client has been synced. - pub block_num: BlockNumber, - /// IDs of notes that have been committed. - pub committed_notes: Vec, - /// IDs of notes that have been consumed. - pub consumed_notes: Vec, - /// IDs of on-chain accounts that have been updated. - pub updated_accounts: Vec, - /// IDs of private accounts that have been locked. - pub locked_accounts: Vec, - /// IDs of committed transactions. - pub committed_transactions: Vec, -} - -impl SyncSummary { - pub fn new( - block_num: BlockNumber, - committed_notes: Vec, - consumed_notes: Vec, - updated_accounts: Vec, - locked_accounts: Vec, - committed_transactions: Vec, - ) -> Self { - Self { - block_num, - committed_notes, - consumed_notes, - updated_accounts, - locked_accounts, - committed_transactions, - } - } - - pub fn new_empty(block_num: BlockNumber) -> Self { - Self { - block_num, - committed_notes: vec![], - consumed_notes: vec![], - updated_accounts: vec![], - locked_accounts: vec![], - committed_transactions: vec![], - } - } - - pub fn is_empty(&self) -> bool { - self.committed_notes.is_empty() - && self.consumed_notes.is_empty() - && self.updated_accounts.is_empty() - && self.locked_accounts.is_empty() - } - - pub fn combine_with(&mut self, mut other: Self) { - self.block_num = max(self.block_num, other.block_num); - self.committed_notes.append(&mut other.committed_notes); - self.consumed_notes.append(&mut other.consumed_notes); - self.updated_accounts.append(&mut other.updated_accounts); - self.locked_accounts.append(&mut other.locked_accounts); - } -} +mod state_sync_update; +pub use state_sync_update::StateSyncUpdate; // CONSTANTS // ================================================================================================ @@ -230,3 +171,68 @@ impl Client { pub(crate) fn get_nullifier_prefix(nullifier: &Nullifier) -> u16 { (nullifier.inner()[3].as_int() >> FILTER_ID_SHIFT) as u16 } + +// SYNC SUMMARY +// ================================================================================================ + +/// Contains stats about the sync operation. +pub struct SyncSummary { + /// Block number up to which the client has been synced. + pub block_num: BlockNumber, + /// IDs of notes that have been committed. + pub committed_notes: Vec, + /// IDs of notes that have been consumed. + pub consumed_notes: Vec, + /// IDs of on-chain accounts that have been updated. + pub updated_accounts: Vec, + /// IDs of private accounts that have been locked. + pub locked_accounts: Vec, + /// IDs of committed transactions. + pub committed_transactions: Vec, +} + +impl SyncSummary { + pub fn new( + block_num: BlockNumber, + committed_notes: Vec, + consumed_notes: Vec, + updated_accounts: Vec, + locked_accounts: Vec, + committed_transactions: Vec, + ) -> Self { + Self { + block_num, + committed_notes, + consumed_notes, + updated_accounts, + locked_accounts, + committed_transactions, + } + } + + pub fn new_empty(block_num: BlockNumber) -> Self { + Self { + block_num, + committed_notes: vec![], + consumed_notes: vec![], + updated_accounts: vec![], + locked_accounts: vec![], + committed_transactions: vec![], + } + } + + pub fn is_empty(&self) -> bool { + self.committed_notes.is_empty() + && self.consumed_notes.is_empty() + && self.updated_accounts.is_empty() + && self.locked_accounts.is_empty() + } + + pub fn combine_with(&mut self, mut other: Self) { + self.block_num = max(self.block_num, other.block_num); + self.committed_notes.append(&mut other.committed_notes); + self.consumed_notes.append(&mut other.consumed_notes); + self.updated_accounts.append(&mut other.updated_accounts); + self.locked_accounts.append(&mut other.locked_accounts); + } +} diff --git a/crates/rust-client/src/sync/state_sync.rs b/crates/rust-client/src/sync/state_sync.rs index 20beb2510..884758247 100644 --- a/crates/rust-client/src/sync/state_sync.rs +++ b/crates/rust-client/src/sync/state_sync.rs @@ -3,15 +3,15 @@ use core::{future::Future, pin::Pin}; use miden_objects::{ account::{Account, AccountHeader, AccountId}, - block::{BlockHeader, BlockNumber}, + block::BlockHeader, crypto::merkle::{InOrderIndex, MmrDelta, MmrPeaks, PartialMmr}, - note::{NoteId, NoteInclusionProof, NoteTag, Nullifier}, + note::{NoteId, NoteInclusionProof, NoteTag}, transaction::TransactionId, Digest, }; use tracing::info; -use super::{block_header::BlockUpdates, get_nullifier_prefix, SyncSummary}; +use super::{block_header::BlockUpdates, get_nullifier_prefix, StateSyncUpdate}; use crate::{ account::AccountUpdates, note::{NoteScreener, NoteUpdates}, @@ -24,61 +24,15 @@ use crate::{ ClientError, }; -// STATE SYNC UPDATE -// ================================================================================================ - -#[derive(Default)] -/// Contains all information needed to apply the update in the store after syncing with the node. -pub struct StateSyncUpdate { - /// The block number of the last block that was synced. - pub block_num: BlockNumber, - /// New blocks and authentication nodes. - pub block_updates: BlockUpdates, - /// New and updated notes to be upserted in the store. - pub note_updates: NoteUpdates, - /// Committed and discarded transactions after the sync. - pub transaction_updates: TransactionUpdates, - /// Public account updates and mismatched private accounts after the sync. - pub account_updates: AccountUpdates, -} - -impl From<&StateSyncUpdate> for SyncSummary { - fn from(value: &StateSyncUpdate) -> Self { - SyncSummary::new( - value.block_num, - value.note_updates.committed_note_ids().into_iter().collect(), - value.note_updates.consumed_note_ids().into_iter().collect(), - value - .account_updates - .updated_public_accounts() - .iter() - .map(|acc| acc.id()) - .collect(), - value - .account_updates - .mismatched_private_accounts() - .iter() - .map(|(id, _)| *id) - .collect(), - value - .transaction_updates - .committed_transactions() - .iter() - .map(|t| t.transaction_id) - .collect(), - ) - } -} - // SYNC CALLBACKS // ================================================================================================ /// Callback to be executed when a new note inclusion is received in the sync response. It receives -/// the committed note received from the node, the block header in which the note was included and -/// the list of public notes that were included in the block. +/// the committed note received from the node and the input note state and an optional note record +/// that corresponds to the state of the note in the node (only if the note is public). /// -/// It returns two optional notes (one input and one output) that should be updated in the store and -/// a flag indicating if the block is relevant to the client. +/// It returns a boolean indicating if received note update is relevant and the client should be +/// updated. pub type OnNoteReceived = Box< dyn Fn( CommittedNote, @@ -87,11 +41,10 @@ pub type OnNoteReceived = Box< >; /// Callback to be executed when a nullifier is received in the sync response. It receives the -/// nullifier update received from the node and the list of transaction updates that were committed -/// in the block. +/// nullifier update received from the node. /// -/// It returns two optional notes (one input and one output) that should be updated in the store and -/// an optional transaction ID if a transaction should be discarded. +/// It returns a boolean indicating if the received note update is relevant and the client should be +/// updated. pub type OnNullifierReceived = Box Pin>>>>; @@ -100,7 +53,11 @@ pub type OnNullifierReceived = /// The state sync components encompasses the client's sync logic. /// -/// When created it receives the current state of the client's relevant elements (block, accounts, +/// When created it receives callbacks that will be executed when a new note inclusion or a +/// nullifier is received in the sync response. +/// +/// +/// current state of the client's relevant elements (block, accounts, /// notes, etc). It is then used to requset updates from the node and apply them to the relevant /// elements. The updates are then returned and can be applied to the store to persist the changes. pub struct StateSync { @@ -150,21 +107,14 @@ impl StateSync { current_partial_mmr: &mut PartialMmr, accounts: &[AccountHeader], note_tags: &[NoteTag], - unspent_nullifiers: &[Nullifier], + nullifiers_tags: &[u16], ) -> Result { let current_block_num = (current_partial_mmr.num_leaves() as u32 - 1).into(); let account_ids: Vec = accounts.iter().map(|acc| acc.id()).collect(); - // To receive information about added nullifiers, we reduce them to the higher 16 bits - // Note that besides filtering by nullifier prefixes, the node also filters by block number - // (it only returns nullifiers from current_block_num until - // response.block_header.block_num()) - let nullifiers_tags: Vec = - unspent_nullifiers.iter().map(get_nullifier_prefix).collect(); - let response = self .rpc_api - .sync_state(current_block_num, &account_ids, note_tags, &nullifiers_tags) + .sync_state(current_block_num, &account_ids, note_tags, nullifiers_tags) .await?; self.state_sync_update.block_num = response.block_header.block_num(); @@ -208,15 +158,29 @@ impl StateSync { /// Syncs the state of the client with the chain tip of the node, returning the updates that /// should be applied to the store. /// + /// During the sync process, the client will go through the following steps: + /// 1. A request is sent to the node to get the state updates. This request includes tracked + /// account IDs and the tags of notes that might have changed or that might be of interest to + /// the client. + /// 2. A response is received with the current state of the network. The response includes + /// information about new/committed/consumed notes, updated accounts, and committed + /// transactions. + /// 3. Tracked public accounts are updated and private accounts are validated against the node + /// state. + /// 4. Tracked notes are updated with their new states. Notes might be committed or nullified + /// during the sync processing. + /// 5. New notes are checked, and only relevant ones are stored. Relevance is determined by the + /// [OnNoteReceived] callback. + /// 6. Transactions are updated with their new states. Transactions might be committed or + /// discarded. + /// 7. The MMR is updated with the new peaks and authentication nodes. + /// /// # Arguments - /// * `current_block` - The latest tracked block header. - /// * `current_block_has_relevant_notes` - A flag indicating if the current block has notes that - /// are relevant to the client. This is used to determine whether new MMR authentication nodes - /// are stored for this block. /// * `current_partial_mmr` - The current partial MMR. - /// * `accounts` - The headers of tracked accounts. + /// * `accounts` - All the headers of tracked accounts. /// * `note_tags` - The note tags to be used in the sync state request. - /// * `unspent_nullifiers` - The nullifiers of tracked notes that haven't been consumed. + /// * `unspent_input_notes` - The current state of unspent input notes tracked by the client. + /// * `unspent_output_notes` - The current state of unspent output notes tracked by the client. pub async fn sync_state( mut self, mut current_partial_mmr: PartialMmr, @@ -225,30 +189,37 @@ impl StateSync { unspent_input_notes: Vec, unspent_output_notes: Vec, ) -> Result { - let mut unspent_nullifiers: Vec = unspent_input_notes + let unspent_nullifiers = unspent_input_notes .iter() .map(|note| note.nullifier()) - .chain(unspent_output_notes.iter().filter_map(|note| note.nullifier())) - .collect(); + .chain(unspent_output_notes.iter().filter_map(|note| note.nullifier())); + + // To receive information about added nullifiers, we reduce them to the higher 16 bits + // Note that besides filtering by nullifier prefixes, the node also filters by block number + // (it only returns nullifiers from current_block_num until + // response.block_header.block_num()) + let mut nullifiers_tags: Vec = + unspent_nullifiers.map(|nullifier| get_nullifier_prefix(&nullifier)).collect(); self.state_sync_update.note_updates = NoteUpdates::new(unspent_input_notes, unspent_output_notes); while self - .sync_state_step(&mut current_partial_mmr, &accounts, ¬e_tags, &unspent_nullifiers) + .sync_state_step(&mut current_partial_mmr, &accounts, ¬e_tags, &nullifiers_tags) .await? { // New nullfiers should be added for new untracked notes that were added in previous // steps - unspent_nullifiers.append( + nullifiers_tags.append( &mut self .state_sync_update .note_updates .updated_input_notes() .filter(|note| { - note.is_committed() && !unspent_nullifiers.contains(¬e.nullifier()) + note.is_committed() + && !nullifiers_tags.contains(&get_nullifier_prefix(¬e.nullifier())) }) - .map(|note| note.nullifier()) + .map(|note| get_nullifier_prefix(¬e.nullifier())) .collect::>(), ); } @@ -259,15 +230,15 @@ impl StateSync { // HELPERS // -------------------------------------------------------------------------------------------- - /// Compares the state of tracked accounts with the updates received from the node and updates - /// the `state_sync_update` with the details of - /// the accounts that need to be updated. + /// Compares the state of tracked accounts with the updates received from the node. The method + /// updates the `state_sync_update` field with the details of the accounts that need to be + /// updated. /// - /// When a mismatch is detected, two scenarios are possible: - /// * If the account is public, the component will request the node for the updated account - /// details. - /// * If the account is private it will be marked as mismatched and the client will need to - /// handle it (it could be a stale account state or a reason to lock the account). + /// The account updates might include: + /// * Public accounts that have been updated in the node. + /// * Private accounts that have been marked as mismatched because the current hash doesn't + /// match the one received from the node. The client will need to handle these cases as they + /// could be a stale account state or a reason to lock the account. async fn account_state_sync( &mut self, accounts: &[AccountHeader], @@ -322,18 +293,18 @@ impl StateSync { } /// Applies the changes received from the sync response to the notes and transactions tracked - /// by the client and updates the - /// `state_sync_update` accordingly. + /// by the client and updates the `state_sync_update` accordingly. /// - /// This method uses the callbacks provided to the [StateSync] component to apply the changes. + /// This method uses the callbacks provided to the [StateSync] component to check if the updates + /// received are relevant to the client. /// - /// The note changes might include: + /// The note updates might include: /// * New notes that we received from the node and might be relevant to the client. /// * Tracked expected notes that were committed in the block. /// * Tracked notes that were being processed by a transaction that got committed. /// * Tracked notes that were nullified by an external transaction. /// - /// The transaction changes might include: + /// The transaction updates might include: /// * Transactions that were committed in the block. Some of these might me tracked by the /// client and need to be marked as committed. /// * Local tracked transactions that were discarded because the notes that they were processing @@ -436,12 +407,6 @@ async fn apply_mmr_changes( current_partial_mmr: &mut PartialMmr, mmr_delta: MmrDelta, ) -> Result<(MmrPeaks, Vec<(InOrderIndex, Digest)>), ClientError> { - // First, apply curent_block to the MMR. This is needed as the MMR delta received from the - // node doesn't contain the request block itself. - // let new_authentication_nodes = current_partial_mmr - // .add(current_block.hash(), current_block_has_relevant_notes) - // .into_iter(); - // Apply the MMR delta to bring MMR to forest equal to chain tip let mut new_authentication_nodes: Vec<(InOrderIndex, Digest)> = current_partial_mmr.apply(mmr_delta).map_err(StoreError::MmrError)?; @@ -454,13 +419,8 @@ async fn apply_mmr_changes( Ok((new_peaks, new_authentication_nodes)) } -// DEFAULT CALLBACK IMPLEMENTATIONS -// ================================================================================================ - -/// Default implementation of the [OnNoteReceived] callback. It queries the store for the committed -/// note and updates it accordingly. If the note wasn't being tracked but it came in the sync -/// response, it is also returned so it can be inserted in the store. The method also returns a -/// flag indicating if the block is relevant to the client. +/// Applies the necessary state transitions to the [NoteUpdates] when a note is committed in a +/// block. async fn committed_state_transions( note_updates: &mut NoteUpdates, committed_note: CommittedNote, @@ -488,9 +448,11 @@ async fn committed_state_transions( Ok(()) } -/// Default implementation of the [OnNullifierReceived] callback. It queries the store for the notes -/// that match the nullifier and updates the note records accordingly. It also returns an optional -/// transaction ID that should be discarded. +/// Applies the necessary state transitions to the [NoteUpdates] when a note is nullified in a +/// block. For input note records two possible scenarios are considered: +/// 1. The note was being processed by a local transaction that just got committed. +/// 2. The note was consumed by an external transaction. If a local transaction was processing the +/// note and it didn't get committed, the transaction should be discarded. async fn nullfier_state_transitions( note_updates: &mut NoteUpdates, nullifier_update: NullifierUpdate, @@ -533,6 +495,13 @@ async fn nullfier_state_transitions( Ok(discarded_transaction) } +// DEFAULT CALLBACK IMPLEMENTATIONS +// ================================================================================================ + +/// Default implementation of the [OnNoteReceived] callback. It queries the store for the committed +/// note to check if it's relevant. If the note wasn't being tracked but it came in the sync +/// response it may be a new public note, in that case we use the [NoteScreener] to check its +/// relevance. pub async fn on_note_received( store: Arc, committed_note: CommittedNote, diff --git a/crates/rust-client/src/sync/state_sync_update.rs b/crates/rust-client/src/sync/state_sync_update.rs new file mode 100644 index 000000000..51bee7b6a --- /dev/null +++ b/crates/rust-client/src/sync/state_sync_update.rs @@ -0,0 +1,50 @@ +use miden_objects::block::BlockNumber; + +use super::{block_header::BlockUpdates, SyncSummary}; +use crate::{account::AccountUpdates, note::NoteUpdates, transaction::TransactionUpdates}; + +// STATE SYNC UPDATE +// ================================================================================================ + +/// Contains all information needed to apply the update in the store after syncing with the node. +#[derive(Default)] +pub struct StateSyncUpdate { + /// The block number of the last block that was synced. + pub block_num: BlockNumber, + /// New blocks and authentication nodes. + pub block_updates: BlockUpdates, + /// New and updated notes to be upserted in the store. + pub note_updates: NoteUpdates, + /// Committed and discarded transactions after the sync. + pub transaction_updates: TransactionUpdates, + /// Public account updates and mismatched private accounts after the sync. + pub account_updates: AccountUpdates, +} + +impl From<&StateSyncUpdate> for SyncSummary { + fn from(value: &StateSyncUpdate) -> Self { + SyncSummary::new( + value.block_num, + value.note_updates.committed_note_ids().into_iter().collect(), + value.note_updates.consumed_note_ids().into_iter().collect(), + value + .account_updates + .updated_public_accounts() + .iter() + .map(|acc| acc.id()) + .collect(), + value + .account_updates + .mismatched_private_accounts() + .iter() + .map(|(id, _)| *id) + .collect(), + value + .transaction_updates + .committed_transactions() + .iter() + .map(|t| t.transaction_id) + .collect(), + ) + } +} From 8ea1a156db3d8783c5993b90a930a01cc0377f6e Mon Sep 17 00:00:00 2001 From: tomyrd Date: Fri, 14 Feb 2025 10:43:17 -0300 Subject: [PATCH 24/34] review: refactor `BlockUpdates` --- .../src/store/sqlite_store/sync.rs | 19 +++++-------- .../src/store/web_store/sync/mod.rs | 4 +-- crates/rust-client/src/sync/block_header.rs | 27 ++++++++++++++++--- 3 files changed, 33 insertions(+), 17 deletions(-) diff --git a/crates/rust-client/src/store/sqlite_store/sync.rs b/crates/rust-client/src/store/sqlite_store/sync.rs index 933bf79cb..cc1f5c674 100644 --- a/crates/rust-client/src/store/sqlite_store/sync.rs +++ b/crates/rust-client/src/store/sqlite_store/sync.rs @@ -130,29 +130,24 @@ impl SqliteStore { const BLOCK_NUMBER_QUERY: &str = "UPDATE state_sync SET block_num = ?"; tx.execute(BLOCK_NUMBER_QUERY, params![i64::from(block_num.as_u32())])?; - for (block_header, block_has_relevant_notes, new_mmr_peaks) in block_updates.block_headers { + for (block_header, block_has_relevant_notes, new_mmr_peaks) in block_updates.block_headers() + { Self::insert_block_header_tx( &tx, - &block_header, - &new_mmr_peaks, - block_has_relevant_notes, + block_header, + new_mmr_peaks, + *block_has_relevant_notes, )?; } // Insert new authentication nodes (inner nodes of the PartialMmr) - Self::insert_chain_mmr_nodes_tx(&tx, &block_updates.new_authentication_nodes)?; + Self::insert_chain_mmr_nodes_tx(&tx, block_updates.new_authentication_nodes())?; // Update notes apply_note_updates_tx(&tx, ¬e_updates)?; // Remove tags - let tags_to_remove = note_updates.committed_input_notes().map(|note| { - NoteTagRecord::with_note_source( - note.metadata().expect("Committed notes should have metadata").tag(), - note.id(), - ) - }); - for tag in tags_to_remove { + for tag in note_updates.tags_to_remove() { remove_note_tag_tx(&tx, tag)?; } diff --git a/crates/rust-client/src/store/web_store/sync/mod.rs b/crates/rust-client/src/store/web_store/sync/mod.rs index 1f2a2ce98..12c7764c5 100644 --- a/crates/rust-client/src/store/web_store/sync/mod.rs +++ b/crates/rust-client/src/store/web_store/sync/mod.rs @@ -122,7 +122,7 @@ impl WebStore { let mut block_nums_as_str = vec![]; let mut block_has_relevant_notes = vec![]; - for (block_header, has_client_notes, mmr_peaks) in &block_updates.block_headers { + for (block_header, has_client_notes, mmr_peaks) in block_updates.block_headers() { block_headers_as_bytes.push(block_header.to_bytes()); new_mmr_peaks_as_bytes.push(mmr_peaks.peaks().to_vec().to_bytes()); block_nums_as_str.push(block_header.block_num().to_string()); @@ -132,7 +132,7 @@ impl WebStore { // Serialize data for updating chain MMR nodes let mut serialized_node_ids = Vec::new(); let mut serialized_nodes = Vec::new(); - for (id, node) in &block_updates.new_authentication_nodes { + for (id, node) in block_updates.new_authentication_nodes() { let serialized_data = serialize_chain_mmr_node(*id, *node)?; serialized_node_ids.push(serialized_data.id); serialized_nodes.push(serialized_data.node); diff --git a/crates/rust-client/src/sync/block_header.rs b/crates/rust-client/src/sync/block_header.rs index 06e7f91ee..f8c822f3f 100644 --- a/crates/rust-client/src/sync/block_header.rs +++ b/crates/rust-client/src/sync/block_header.rs @@ -19,14 +19,35 @@ use crate::{ pub struct BlockUpdates { /// New block headers to be stored, along with a flag indicating whether the block contains /// notes that are relevant to the client and the MMR peaks for the block. - pub block_headers: Vec<(BlockHeader, bool, MmrPeaks)>, + block_headers: Vec<(BlockHeader, bool, MmrPeaks)>, /// New authentication nodes that are meant to be stored in order to authenticate block /// headers. - pub new_authentication_nodes: Vec<(InOrderIndex, Digest)>, + new_authentication_nodes: Vec<(InOrderIndex, Digest)>, } impl BlockUpdates { - pub fn extend(&mut self, other: BlockUpdates) { + /// Creates a new instance of [`BlockUpdates`]. + pub fn new( + block_headers: Vec<(BlockHeader, bool, MmrPeaks)>, + new_authentication_nodes: Vec<(InOrderIndex, Digest)>, + ) -> Self { + Self { block_headers, new_authentication_nodes } + } + + /// Returns the new block headers to be stored, along with a flag indicating whether the block + /// contains notes that are relevant to the client and the MMR peaks for the block. + pub fn block_headers(&self) -> &[(BlockHeader, bool, MmrPeaks)] { + &self.block_headers + } + + /// Returns the new authentication nodes that are meant to be stored in order to authenticate + /// block headers. + pub fn new_authentication_nodes(&self) -> &[(InOrderIndex, Digest)] { + &self.new_authentication_nodes + } + + /// Extends the current [`BlockUpdates`] with the provided one. + pub(crate) fn extend(&mut self, other: BlockUpdates) { self.block_headers.extend(other.block_headers); self.new_authentication_nodes.extend(other.new_authentication_nodes); } From a6ac9489c144b8a63684a550b61017a998832208 Mon Sep 17 00:00:00 2001 From: tomyrd Date: Fri, 14 Feb 2025 10:44:10 -0300 Subject: [PATCH 25/34] review: improve docs --- crates/rust-client/src/account.rs | 9 +- crates/rust-client/src/sync/state_sync.rs | 147 +++++++++++----------- 2 files changed, 80 insertions(+), 76 deletions(-) diff --git a/crates/rust-client/src/account.rs b/crates/rust-client/src/account.rs index 66ed94d12..f73d27da8 100644 --- a/crates/rust-client/src/account.rs +++ b/crates/rust-client/src/account.rs @@ -257,11 +257,16 @@ impl Client { // ================================================================================================ #[derive(Debug, Clone, Default)] -/// Contains account changes to apply to the store. +/// Contains account changes to apply to the store after a sync request. pub struct AccountUpdates { /// Updated public accounts. updated_public_accounts: Vec, - /// Node account hashes that don't match the tracked information. + /// Account hashes received from the network that don't match the currently locally-tracked + /// state of the private accounts. + /// + /// These updates may represent a stale account hash (meaning that the latest local state + /// hasn't been committed). If this is not the case, the account may be locked until the state + /// is restored manually. mismatched_private_accounts: Vec<(AccountId, Digest)>, } diff --git a/crates/rust-client/src/sync/state_sync.rs b/crates/rust-client/src/sync/state_sync.rs index 5522510cf..7928961a7 100644 --- a/crates/rust-client/src/sync/state_sync.rs +++ b/crates/rust-client/src/sync/state_sync.rs @@ -27,12 +27,12 @@ use crate::{ // SYNC CALLBACKS // ================================================================================================ -/// Callback to be executed when a new note inclusion is received in the sync response. It receives -/// the committed note received from the node and the input note state and an optional note record -/// that corresponds to the state of the note in the node (only if the note is public). +/// Callback that gets executed when a new note inclusion is received as part of the sync response. +/// It receives the committed note received from the network and the input note state and an optional +/// note record that corresponds to the state of the note in the network (only if the note is public). /// -/// It returns a boolean indicating if received note update is relevant and the client should be -/// updated. +/// It returns a boolean indicating if the received note update is relevant. +/// If the return value is `false`, it gets discarded. If it is `true`, the update gets committed to the client's store. pub type OnNoteReceived = Box< dyn Fn( CommittedNote, @@ -40,11 +40,11 @@ pub type OnNoteReceived = Box< ) -> Pin>>>, >; -/// Callback to be executed when a nullifier is received in the sync response. It receives the -/// nullifier update received from the node. +/// Callback to be executed when a nullifier is received as part of the the sync response. It receives the +/// nullifier update received from the network. /// -/// It returns a boolean indicating if the received note update is relevant and the client should be -/// updated. +/// It returns a boolean indicating if the received note update is relevant +/// If the return value is `false`, it gets discarded. If it is `true`, the update gets committed to the client's store. pub type OnNullifierReceived = Box Pin>>>>; @@ -93,70 +93,6 @@ impl StateSync { } } - /// Executes a single step of the state sync process, returning `true` if the client should - /// continue syncing and `false` if the client has reached the chain tip. - /// - /// A step in this context means a single request to the node to get the next relevant block and - /// the changes that happened in it. This block may not be the last one in the chain and - /// the client may need to call this method multiple times until it reaches the chain tip. - /// - /// The `sync_state_update` field of the struct will be updated with the new changes from this - /// step. - async fn sync_state_step( - &mut self, - current_partial_mmr: &mut PartialMmr, - accounts: &[AccountHeader], - note_tags: &[NoteTag], - nullifiers_tags: &[u16], - ) -> Result { - let current_block_num = (u32::try_from(current_partial_mmr.num_leaves()) - .expect("The number of leaves in the MMR should be less than 2^32") - - 1) - .into(); - let account_ids: Vec = accounts.iter().map(AccountHeader::id).collect(); - - let response = self - .rpc_api - .sync_state(current_block_num, &account_ids, note_tags, nullifiers_tags) - .await?; - - self.state_sync_update.block_num = response.block_header.block_num(); - - // We don't need to continue if the chain has not advanced, there are no new changes - if response.block_header.block_num() == current_block_num { - return Ok(false); - } - - self.account_state_sync(accounts, &response.account_hash_updates).await?; - - let found_relevant_note = self - .note_state_sync( - response.note_inclusions, - response.transactions, - response.nullifiers, - &response.block_header, - ) - .await?; - - let (new_mmr_peaks, new_authentication_nodes) = apply_mmr_changes( - &response.block_header, - found_relevant_note, - current_partial_mmr, - response.mmr_delta, - )?; - - self.state_sync_update.block_updates.extend(BlockUpdates { - block_headers: vec![(response.block_header, found_relevant_note, new_mmr_peaks)], - new_authentication_nodes, - }); - - if response.chain_tip == response.block_header.block_num() { - Ok(false) - } else { - Ok(true) - } - } - /// Syncs the state of the client with the chain tip of the node, returning the updates that /// should be applied to the store. /// @@ -229,6 +165,69 @@ impl StateSync { Ok(self.state_sync_update) } + /// Executes a single step of the state sync process, returning `true` if the client should + /// continue syncing and `false` if the client has reached the chain tip. + /// + /// A step in this context means a single request to the node to get the next relevant block and + /// the changes that happened in it. This block may not be the last one in the chain and + /// the client may need to call this method multiple times until it reaches the chain tip. + /// + /// The `sync_state_update` field of the struct will be updated with the new changes from this + /// step. + async fn sync_state_step( + &mut self, + current_partial_mmr: &mut PartialMmr, + accounts: &[AccountHeader], + note_tags: &[NoteTag], + nullifiers_tags: &[u16], + ) -> Result { + let current_block_num = (u32::try_from(current_partial_mmr.num_leaves() - 1) + .expect("The number of leaves in the MMR should be greater than 0 and less than 2^32")) + .into(); + let account_ids: Vec = accounts.iter().map(AccountHeader::id).collect(); + + let response = self + .rpc_api + .sync_state(current_block_num, &account_ids, note_tags, nullifiers_tags) + .await?; + + self.state_sync_update.block_num = response.block_header.block_num(); + + // We don't need to continue if the chain has not advanced, there are no new changes + if response.block_header.block_num() == current_block_num { + return Ok(false); + } + + self.account_state_sync(accounts, &response.account_hash_updates).await?; + + let found_relevant_note = self + .note_state_sync( + response.note_inclusions, + response.transactions, + response.nullifiers, + &response.block_header, + ) + .await?; + + let (new_mmr_peaks, new_authentication_nodes) = apply_mmr_changes( + &response.block_header, + found_relevant_note, + current_partial_mmr, + response.mmr_delta, + )?; + + self.state_sync_update.block_updates.extend(BlockUpdates::new( + vec![(response.block_header, found_relevant_note, new_mmr_peaks)], + new_authentication_nodes, + )); + + if response.chain_tip == response.block_header.block_num() { + Ok(false) + } else { + Ok(true) + } + } + // HELPERS // -------------------------------------------------------------------------------------------- @@ -279,7 +278,7 @@ impl StateSync { let mut mismatched_public_accounts = vec![]; for (id, hash) in account_updates { - // check if this updated account is tracked by the client + // check if this updated account state is tracked by the client if let Some(account) = current_public_accounts .iter() .find(|acc| *id == acc.id() && *hash != acc.hash()) From b1c0110a22259e1fe5cba8510b9104da0092bfbe Mon Sep 17 00:00:00 2001 From: tomyrd Date: Fri, 14 Feb 2025 11:08:45 -0300 Subject: [PATCH 26/34] review: improve `NoteUpdates` --- crates/rust-client/src/note/mod.rs | 148 +----------- crates/rust-client/src/note/note_updates.rs | 247 ++++++++++++++++++++ crates/rust-client/src/sync/state_sync.rs | 110 ++------- 3 files changed, 267 insertions(+), 238 deletions(-) create mode 100644 crates/rust-client/src/note/note_updates.rs diff --git a/crates/rust-client/src/note/mod.rs b/crates/rust-client/src/note/mod.rs index 4a370731b..7154e0cf7 100644 --- a/crates/rust-client/src/note/mod.rs +++ b/crates/rust-client/src/note/mod.rs @@ -56,11 +56,7 @@ //! For more details on the API and error handling, see the documentation for the specific functions //! and types in this module. -use alloc::{ - collections::{BTreeMap, BTreeSet}, - string::ToString, - vec::Vec, -}; +use alloc::{string::ToString, vec::Vec}; use miden_lib::transaction::TransactionKernel; use miden_objects::{account::AccountId, crypto::rand::FeltRng}; @@ -74,6 +70,7 @@ pub mod script_roots; mod import; mod note_screener; +mod note_updates; // RE-EXPORTS // ================================================================================================ @@ -92,6 +89,7 @@ pub use miden_objects::{ NoteError, }; pub use note_screener::{NoteConsumability, NoteRelevance, NoteScreener, NoteScreenerError}; +pub use note_updates::NoteUpdates; /// Contains functions to simplify standard note scripts creation. pub mod scripts { @@ -239,143 +237,3 @@ pub async fn get_input_note_with_id_prefix( .pop() .expect("input_note_records should always have one element")) } - -// NOTE UPDATES -// ------------------------------------------------------------------------------------------------ - -/// Contains note changes to apply to the store. -#[derive(Clone, Debug, Default)] -pub struct NoteUpdates { - /// A map of new and updated input note records to be upserted in the store. - updated_input_notes: BTreeMap, - /// A map of updated output note records to be upserted in the store. - updated_output_notes: BTreeMap, -} - -impl NoteUpdates { - /// Creates a [`NoteUpdates`]. - pub fn new( - updated_input_notes: impl IntoIterator, - updated_output_notes: impl IntoIterator, - ) -> Self { - Self { - updated_input_notes: updated_input_notes - .into_iter() - .map(|note| (note.id(), note)) - .collect(), - updated_output_notes: updated_output_notes - .into_iter() - .map(|note| (note.id(), note)) - .collect(), - } - } - - /// Returns all input note records that have been updated. - /// This may include: - /// - New notes that have been created that should be inserted. - /// - Existing tracked notes that should be updated. - pub fn updated_input_notes(&self) -> impl Iterator { - self.updated_input_notes.values() - } - - /// Returns all output note records that have been updated. - /// This may include: - /// - New notes that have been created that should be inserted. - /// - Existing tracked notes that should be updated. - pub fn updated_output_notes(&self) -> impl Iterator { - self.updated_output_notes.values() - } - - /// Returns whether no new note-related information has been retrieved. - pub fn is_empty(&self) -> bool { - self.updated_input_notes.is_empty() && self.updated_output_notes.is_empty() - } - - /// Returns any note that has been committed into the chain in this update (either new or - /// already locally tracked) - pub fn committed_input_notes(&self) -> impl Iterator { - self.updated_input_notes.values().filter(|note| note.is_committed()) - } - - /// Returns the IDs of all notes that have been committed in this update. - /// This includes both new notes and tracked expected notes that were committed in this update. - pub fn committed_note_ids(&self) -> BTreeSet { - let committed_output_note_ids = self - .updated_output_notes - .values() - .filter_map(|note_record| note_record.is_committed().then_some(note_record.id())); - - let committed_input_note_ids = self - .updated_input_notes - .values() - .filter_map(|note_record| note_record.is_committed().then_some(note_record.id())); - - committed_input_note_ids - .chain(committed_output_note_ids) - .collect::>() - } - - /// Returns the IDs of all notes that have been consumed. - /// This includes both notes that have been consumed locally or externally in this update. - pub fn consumed_note_ids(&self) -> BTreeSet { - let consumed_output_note_ids = self - .updated_output_notes - .values() - .filter_map(|note_record| note_record.is_consumed().then_some(note_record.id())); - - let consumed_input_note_ids = self - .updated_input_notes - .values() - .filter_map(|note_record| note_record.is_consumed().then_some(note_record.id())); - - consumed_input_note_ids.chain(consumed_output_note_ids).collect::>() - } - - /// Inserts new or updated input and output notes. If an update with the same note ID already - /// exists, it will be replaced. - pub(crate) fn insert_updates( - &mut self, - input_note: Option, - output_note: Option, - ) { - if let Some(input_note) = input_note { - self.updated_input_notes.insert(input_note.id(), input_note); - } - if let Some(output_note) = output_note { - self.updated_output_notes.insert(output_note.id(), output_note); - } - } - - /// Returns a mutable reference to the input note record with the provided ID if it exists. - pub(crate) fn get_input_note_by_id(&mut self, note_id: NoteId) -> Option<&mut InputNoteRecord> { - self.updated_input_notes.get_mut(¬e_id) - } - - /// Returns a mutable reference to the output note record with the provided ID if it exists. - pub(crate) fn get_output_note_by_id( - &mut self, - note_id: NoteId, - ) -> Option<&mut OutputNoteRecord> { - self.updated_output_notes.get_mut(¬e_id) - } - - /// Returns a mutable reference to the input note record with the provided nullifier if it - /// exists. - pub(crate) fn get_input_note_by_nullifier( - &mut self, - nullifier: Nullifier, - ) -> Option<&mut InputNoteRecord> { - self.updated_input_notes.values_mut().find(|note| note.nullifier() == nullifier) - } - - /// Returns a mutable reference to the output note record with the provided nullifier if it - /// exists. - pub(crate) fn get_output_note_by_nullifier( - &mut self, - nullifier: Nullifier, - ) -> Option<&mut OutputNoteRecord> { - self.updated_output_notes - .values_mut() - .find(|note| note.nullifier() == Some(nullifier)) - } -} diff --git a/crates/rust-client/src/note/note_updates.rs b/crates/rust-client/src/note/note_updates.rs new file mode 100644 index 000000000..867e543f6 --- /dev/null +++ b/crates/rust-client/src/note/note_updates.rs @@ -0,0 +1,247 @@ +use alloc::collections::{BTreeMap, BTreeSet}; + +use miden_objects::{ + block::BlockHeader, + note::{NoteId, NoteInclusionProof, Nullifier}, + transaction::TransactionId, +}; + +use crate::{ + rpc::domain::{ + note::CommittedNote, nullifier::NullifierUpdate, transaction::TransactionUpdate, + }, + store::{InputNoteRecord, OutputNoteRecord}, + sync::NoteTagRecord, + ClientError, +}; + +// NOTE UPDATES +// ================================================================================================ + +/// Contains note changes to apply to the store. +#[derive(Clone, Debug, Default)] +pub struct NoteUpdates { + /// A map of new and updated input note records to be upserted in the store. + updated_input_notes: BTreeMap, + /// A map of updated output note records to be upserted in the store. + updated_output_notes: BTreeMap, +} + +impl NoteUpdates { + /// Creates a [`NoteUpdates`]. + pub fn new( + updated_input_notes: impl IntoIterator, + updated_output_notes: impl IntoIterator, + ) -> Self { + Self { + updated_input_notes: updated_input_notes + .into_iter() + .map(|note| (note.id(), note)) + .collect(), + updated_output_notes: updated_output_notes + .into_iter() + .map(|note| (note.id(), note)) + .collect(), + } + } + + // GETTERS + // -------------------------------------------------------------------------------------------- + + /// Returns all input note records that have been updated. + /// This may include: + /// - New notes that have been created that should be inserted. + /// - Existing tracked notes that should be updated. + pub fn updated_input_notes(&self) -> impl Iterator { + self.updated_input_notes.values() + } + + /// Returns all output note records that have been updated. + /// This may include: + /// - New notes that have been created that should be inserted. + /// - Existing tracked notes that should be updated. + pub fn updated_output_notes(&self) -> impl Iterator { + self.updated_output_notes.values() + } + + /// Returns whether no new note-related information has been retrieved. + pub fn is_empty(&self) -> bool { + self.updated_input_notes.is_empty() && self.updated_output_notes.is_empty() + } + + /// Returns any note that has been committed into the chain in this update (either new or + /// already locally tracked) + pub fn committed_input_notes(&self) -> impl Iterator { + self.updated_input_notes.values().filter(|note| note.is_committed()) + } + + /// Returns the tags of all notes that need to be removed from the store after the state sync. + /// These are the tags of notes that have been committed and no longer need to be tracked. + pub fn tags_to_remove(&self) -> impl Iterator + '_ { + self.committed_input_notes().map(|note| { + NoteTagRecord::with_note_source( + note.metadata().expect("Committed notes should have metadata").tag(), + note.id(), + ) + }) + } + + /// Returns the IDs of all notes that have been committed in this update. + /// This includes both new notes and tracked expected notes that were committed in this update. + pub fn committed_note_ids(&self) -> BTreeSet { + let committed_output_note_ids = self + .updated_output_notes + .values() + .filter_map(|note_record| note_record.is_committed().then_some(note_record.id())); + + let committed_input_note_ids = self + .updated_input_notes + .values() + .filter_map(|note_record| note_record.is_committed().then_some(note_record.id())); + + committed_input_note_ids + .chain(committed_output_note_ids) + .collect::>() + } + + /// Returns the IDs of all notes that have been consumed. + /// This includes both notes that have been consumed locally or externally in this update. + pub fn consumed_note_ids(&self) -> BTreeSet { + let consumed_output_note_ids = self + .updated_output_notes + .values() + .filter_map(|note_record| note_record.is_consumed().then_some(note_record.id())); + + let consumed_input_note_ids = self + .updated_input_notes + .values() + .filter_map(|note_record| note_record.is_consumed().then_some(note_record.id())); + + consumed_input_note_ids.chain(consumed_output_note_ids).collect::>() + } + + // UPDATE METHODS + // -------------------------------------------------------------------------------------------- + + /// Inserts new or updated input and output notes. If an update with the same note ID already + /// exists, it will be replaced. + pub(crate) fn insert_updates( + &mut self, + input_note: Option, + output_note: Option, + ) { + if let Some(input_note) = input_note { + self.updated_input_notes.insert(input_note.id(), input_note); + } + if let Some(output_note) = output_note { + self.updated_output_notes.insert(output_note.id(), output_note); + } + } + + /// Applies the necessary state transitions to the [`NoteUpdates`] when a note is committed in a + /// block. + pub(crate) fn apply_committed_note_state_transitions( + &mut self, + committed_note: &CommittedNote, + block_header: &BlockHeader, + ) -> Result<(), ClientError> { + let inclusion_proof = NoteInclusionProof::new( + block_header.block_num(), + committed_note.note_index(), + committed_note.merkle_path().clone(), + )?; + + if let Some(input_note_record) = self.get_input_note_by_id(*committed_note.note_id()) { + // The note belongs to our locally tracked set of input notes + input_note_record + .inclusion_proof_received(inclusion_proof.clone(), committed_note.metadata())?; + input_note_record.block_header_received(block_header)?; + } + + if let Some(output_note_record) = self.get_output_note_by_id(*committed_note.note_id()) { + // The note belongs to our locally tracked set of output notes + output_note_record.inclusion_proof_received(inclusion_proof.clone())?; + } + + Ok(()) + } + + /// Applies the necessary state transitions to the [`NoteUpdates`] when a note is nullified in a + /// block. For input note records two possible scenarios are considered: + /// 1. The note was being processed by a local transaction that just got committed. + /// 2. The note was consumed by an external transaction. If a local transaction was processing + /// the note and it didn't get committed, the transaction should be discarded. + pub(crate) fn apply_nullifiers_state_transitions( + &mut self, + nullifier_update: &NullifierUpdate, + transaction_updates: &[TransactionUpdate], + ) -> Result, ClientError> { + let mut discarded_transaction = None; + + if let Some(input_note_record) = + self.get_input_note_by_nullifier(nullifier_update.nullifier) + { + if let Some(consumer_transaction) = transaction_updates + .iter() + .find(|t| input_note_record.consumer_transaction_id() == Some(&t.transaction_id)) + { + // The note was being processed by a local transaction that just got committed + input_note_record.transaction_committed( + consumer_transaction.transaction_id, + consumer_transaction.block_num, + )?; + } else { + // The note was consumed by an external transaction + if let Some(id) = input_note_record.consumer_transaction_id() { + // The note was being processed by a local transaction that didn't end up being + // committed so it should be discarded + discarded_transaction.replace(*id); + } + input_note_record + .consumed_externally(nullifier_update.nullifier, nullifier_update.block_num)?; + } + } + + if let Some(output_note_record) = + self.get_output_note_by_nullifier(nullifier_update.nullifier) + { + output_note_record + .nullifier_received(nullifier_update.nullifier, nullifier_update.block_num)?; + } + + Ok(discarded_transaction) + } + + // PRIVATE HELPERS + // -------------------------------------------------------------------------------------------- + + /// Returns a mutable reference to the input note record with the provided ID if it exists. + fn get_input_note_by_id(&mut self, note_id: NoteId) -> Option<&mut InputNoteRecord> { + self.updated_input_notes.get_mut(¬e_id) + } + + /// Returns a mutable reference to the output note record with the provided ID if it exists. + fn get_output_note_by_id(&mut self, note_id: NoteId) -> Option<&mut OutputNoteRecord> { + self.updated_output_notes.get_mut(¬e_id) + } + + /// Returns a mutable reference to the input note record with the provided nullifier if it + /// exists. + fn get_input_note_by_nullifier( + &mut self, + nullifier: Nullifier, + ) -> Option<&mut InputNoteRecord> { + self.updated_input_notes.values_mut().find(|note| note.nullifier() == nullifier) + } + + /// Returns a mutable reference to the output note record with the provided nullifier if it + /// exists. + fn get_output_note_by_nullifier( + &mut self, + nullifier: Nullifier, + ) -> Option<&mut OutputNoteRecord> { + self.updated_output_notes + .values_mut() + .find(|note| note.nullifier() == Some(nullifier)) + } +} diff --git a/crates/rust-client/src/sync/state_sync.rs b/crates/rust-client/src/sync/state_sync.rs index 7928961a7..f321c3525 100644 --- a/crates/rust-client/src/sync/state_sync.rs +++ b/crates/rust-client/src/sync/state_sync.rs @@ -5,8 +5,7 @@ use miden_objects::{ account::{Account, AccountHeader, AccountId}, block::BlockHeader, crypto::merkle::{InOrderIndex, MmrDelta, MmrPeaks, PartialMmr}, - note::{NoteId, NoteInclusionProof, NoteTag}, - transaction::TransactionId, + note::{NoteId, NoteTag}, Digest, }; use tracing::info; @@ -28,11 +27,13 @@ use crate::{ // ================================================================================================ /// Callback that gets executed when a new note inclusion is received as part of the sync response. -/// It receives the committed note received from the network and the input note state and an optional -/// note record that corresponds to the state of the note in the network (only if the note is public). +/// It receives the committed note received from the network and the input note state and an +/// optional note record that corresponds to the state of the note in the network (only if the note +/// is public). /// /// It returns a boolean indicating if the received note update is relevant. -/// If the return value is `false`, it gets discarded. If it is `true`, the update gets committed to the client's store. +/// If the return value is `false`, it gets discarded. If it is `true`, the update gets committed to +/// the client's store. pub type OnNoteReceived = Box< dyn Fn( CommittedNote, @@ -40,11 +41,12 @@ pub type OnNoteReceived = Box< ) -> Pin>>>, >; -/// Callback to be executed when a nullifier is received as part of the the sync response. It receives the -/// nullifier update received from the network. +/// Callback to be executed when a nullifier is received as part of the the sync response. It +/// receives the nullifier update received from the network. /// /// It returns a boolean indicating if the received note update is relevant -/// If the return value is `false`, it gets discarded. If it is `true`, the update gets committed to the client's store. +/// If the return value is `false`, it gets discarded. If it is `true`, the update gets committed to +/// the client's store. pub type OnNullifierReceived = Box Pin>>>>; @@ -339,22 +341,19 @@ impl StateSync { self.state_sync_update.note_updates.insert_updates(Some(public_note), None); } - committed_state_transions( - &mut self.state_sync_update.note_updates, - &committed_note, - block_header, - )?; + self.state_sync_update + .note_updates + .apply_committed_note_state_transitions(&committed_note, block_header)?; } } // Process nullifiers for nullifier_update in nullifiers { if (self.on_nullifier_received)(nullifier_update.clone()).await? { - let discarded_transaction = nullfier_state_transitions( - &mut self.state_sync_update.note_updates, - &nullifier_update, - &transactions, - )?; + let discarded_transaction = self + .state_sync_update + .note_updates + .apply_nullifiers_state_transitions(&nullifier_update, &transactions)?; if let Some(transaction_id) = discarded_transaction { self.state_sync_update @@ -418,81 +417,6 @@ fn apply_mmr_changes( Ok((new_peaks, new_authentication_nodes)) } -/// Applies the necessary state transitions to the [`NoteUpdates`] when a note is committed in a -/// block. -fn committed_state_transions( - note_updates: &mut NoteUpdates, - committed_note: &CommittedNote, - block_header: &BlockHeader, -) -> Result<(), ClientError> { - let inclusion_proof = NoteInclusionProof::new( - block_header.block_num(), - committed_note.note_index(), - committed_note.merkle_path().clone(), - )?; - - if let Some(input_note_record) = note_updates.get_input_note_by_id(*committed_note.note_id()) { - // The note belongs to our locally tracked set of input notes - input_note_record - .inclusion_proof_received(inclusion_proof.clone(), committed_note.metadata())?; - input_note_record.block_header_received(block_header)?; - } - - if let Some(output_note_record) = note_updates.get_output_note_by_id(*committed_note.note_id()) - { - // The note belongs to our locally tracked set of output notes - output_note_record.inclusion_proof_received(inclusion_proof.clone())?; - } - - Ok(()) -} - -/// Applies the necessary state transitions to the [`NoteUpdates`] when a note is nullified in a -/// block. For input note records two possible scenarios are considered: -/// 1. The note was being processed by a local transaction that just got committed. -/// 2. The note was consumed by an external transaction. If a local transaction was processing the -/// note and it didn't get committed, the transaction should be discarded. -fn nullfier_state_transitions( - note_updates: &mut NoteUpdates, - nullifier_update: &NullifierUpdate, - transaction_updates: &[TransactionUpdate], -) -> Result, ClientError> { - let mut discarded_transaction = None; - - if let Some(input_note_record) = - note_updates.get_input_note_by_nullifier(nullifier_update.nullifier) - { - if let Some(consumer_transaction) = transaction_updates - .iter() - .find(|t| input_note_record.consumer_transaction_id() == Some(&t.transaction_id)) - { - // The note was being processed by a local transaction that just got committed - input_note_record.transaction_committed( - consumer_transaction.transaction_id, - consumer_transaction.block_num, - )?; - } else { - // The note was consumed by an external transaction - if let Some(id) = input_note_record.consumer_transaction_id() { - // The note was being processed by a local transaction that didn't end up being - // committed so it should be discarded - discarded_transaction.replace(*id); - } - input_note_record - .consumed_externally(nullifier_update.nullifier, nullifier_update.block_num)?; - } - } - - if let Some(output_note_record) = - note_updates.get_output_note_by_nullifier(nullifier_update.nullifier) - { - output_note_record - .nullifier_received(nullifier_update.nullifier, nullifier_update.block_num)?; - } - - Ok(discarded_transaction) -} - // DEFAULT CALLBACK IMPLEMENTATIONS // ================================================================================================ From 52c04abeec105a4557e894810daf1b00ce6d6910 Mon Sep 17 00:00:00 2001 From: tomyrd Date: Fri, 14 Feb 2025 12:35:14 -0300 Subject: [PATCH 27/34] remove `state_sync_update` from component --- crates/rust-client/src/sync/state_sync.rs | 72 ++++++++++++----------- crates/rust-client/src/transaction/mod.rs | 5 -- 2 files changed, 37 insertions(+), 40 deletions(-) diff --git a/crates/rust-client/src/sync/state_sync.rs b/crates/rust-client/src/sync/state_sync.rs index f321c3525..5d04554a9 100644 --- a/crates/rust-client/src/sync/state_sync.rs +++ b/crates/rust-client/src/sync/state_sync.rs @@ -69,9 +69,6 @@ pub struct StateSync { on_note_received: OnNoteReceived, /// Callback to be executed when a nullifier is received. on_nullifier_received: OnNullifierReceived, - /// The state sync update that will be returned after the sync process is completed. It - /// agregates all the updates that come from each sync step. - state_sync_update: StateSyncUpdate, } impl StateSync { @@ -91,7 +88,6 @@ impl StateSync { rpc_api, on_note_received, on_nullifier_received, - state_sync_update: StateSyncUpdate::default(), } } @@ -141,18 +137,25 @@ impl StateSync { let mut nullifiers_tags: Vec = unspent_nullifiers.map(|nullifier| get_nullifier_prefix(&nullifier)).collect(); - self.state_sync_update.note_updates = - NoteUpdates::new(unspent_input_notes, unspent_output_notes); + let mut state_sync_update = StateSyncUpdate { + note_updates: NoteUpdates::new(unspent_input_notes, unspent_output_notes), + ..Default::default() + }; while self - .sync_state_step(&mut current_partial_mmr, &accounts, ¬e_tags, &nullifiers_tags) + .sync_state_step( + &mut state_sync_update, + &mut current_partial_mmr, + &accounts, + ¬e_tags, + &nullifiers_tags, + ) .await? { // New nullfiers should be added for new untracked notes that were added in previous // steps nullifiers_tags.append( - &mut self - .state_sync_update + &mut state_sync_update .note_updates .updated_input_notes() .filter(|note| { @@ -164,7 +167,7 @@ impl StateSync { ); } - Ok(self.state_sync_update) + Ok(state_sync_update) } /// Executes a single step of the state sync process, returning `true` if the client should @@ -178,6 +181,7 @@ impl StateSync { /// step. async fn sync_state_step( &mut self, + state_sync_update: &mut StateSyncUpdate, current_partial_mmr: &mut PartialMmr, accounts: &[AccountHeader], note_tags: &[NoteTag], @@ -193,17 +197,21 @@ impl StateSync { .sync_state(current_block_num, &account_ids, note_tags, nullifiers_tags) .await?; - self.state_sync_update.block_num = response.block_header.block_num(); + state_sync_update.block_num = response.block_header.block_num(); // We don't need to continue if the chain has not advanced, there are no new changes if response.block_header.block_num() == current_block_num { return Ok(false); } - self.account_state_sync(accounts, &response.account_hash_updates).await?; + let account_updates = + self.account_state_sync(accounts, &response.account_hash_updates).await?; + + state_sync_update.account_updates = account_updates; - let found_relevant_note = self + let (found_relevant_note, transaction_updates) = self .note_state_sync( + &mut state_sync_update.note_updates, response.note_inclusions, response.transactions, response.nullifiers, @@ -211,6 +219,8 @@ impl StateSync { ) .await?; + state_sync_update.transaction_updates.extend(transaction_updates); + let (new_mmr_peaks, new_authentication_nodes) = apply_mmr_changes( &response.block_header, found_relevant_note, @@ -218,7 +228,7 @@ impl StateSync { response.mmr_delta, )?; - self.state_sync_update.block_updates.extend(BlockUpdates::new( + state_sync_update.block_updates.extend(BlockUpdates::new( vec![(response.block_header, found_relevant_note, new_mmr_peaks)], new_authentication_nodes, )); @@ -246,7 +256,7 @@ impl StateSync { &mut self, accounts: &[AccountHeader], account_hash_updates: &[(AccountId, Digest)], - ) -> Result<(), ClientError> { + ) -> Result { let (public_accounts, private_accounts): (Vec<_>, Vec<_>) = accounts.iter().partition(|account_header| account_header.id().is_public()); @@ -263,11 +273,7 @@ impl StateSync { .copied() .collect::>(); - self.state_sync_update - .account_updates - .extend(AccountUpdates::new(updated_public_accounts, mismatched_private_accounts)); - - Ok(()) + Ok(AccountUpdates::new(updated_public_accounts, mismatched_private_accounts)) } /// Queries the node for the latest state of the public accounts that don't match the current @@ -314,17 +320,19 @@ impl StateSync { /// were nullified by an another transaction. async fn note_state_sync( &mut self, + note_updates: &mut NoteUpdates, note_inclusions: Vec, transactions: Vec, nullifiers: Vec, block_header: &BlockHeader, - ) -> Result { + ) -> Result<(bool, TransactionUpdates), ClientError> { let public_note_ids: Vec = note_inclusions .iter() .filter_map(|note| (!note.metadata().is_private()).then_some(*note.note_id())) .collect(); let mut found_relevant_note = false; + let mut discarded_transactions = vec![]; // Process note inclusions let new_public_notes = @@ -338,11 +346,10 @@ impl StateSync { found_relevant_note = true; if let Some(public_note) = public_note { - self.state_sync_update.note_updates.insert_updates(Some(public_note), None); + note_updates.insert_updates(Some(public_note), None); } - self.state_sync_update - .note_updates + note_updates .apply_committed_note_state_transitions(&committed_note, block_header)?; } } @@ -350,24 +357,19 @@ impl StateSync { // Process nullifiers for nullifier_update in nullifiers { if (self.on_nullifier_received)(nullifier_update.clone()).await? { - let discarded_transaction = self - .state_sync_update - .note_updates + let discarded_transaction = note_updates .apply_nullifiers_state_transitions(&nullifier_update, &transactions)?; if let Some(transaction_id) = discarded_transaction { - self.state_sync_update - .transaction_updates - .insert_discarded_transaction(transaction_id); + discarded_transactions.push(transaction_id); } } } - self.state_sync_update - .transaction_updates - .extend(TransactionUpdates::new(transactions, vec![])); - - Ok(found_relevant_note) + Ok(( + found_relevant_note, + TransactionUpdates::new(transactions, discarded_transactions), + )) } /// Queries the node for all received notes that aren't being locally tracked in the client. diff --git a/crates/rust-client/src/transaction/mod.rs b/crates/rust-client/src/transaction/mod.rs index 029ebc5c9..8b0193331 100644 --- a/crates/rust-client/src/transaction/mod.rs +++ b/crates/rust-client/src/transaction/mod.rs @@ -405,11 +405,6 @@ impl TransactionUpdates { pub fn discarded_transactions(&self) -> &[TransactionId] { &self.discarded_transactions } - - /// Inserts a discarded transaction into the transaction updates. - pub fn insert_discarded_transaction(&mut self, transaction_id: TransactionId) { - self.discarded_transactions.push(transaction_id); - } } /// Transaction management methods From efb95899d61c5eaf9de485ada4320b55a0c7cd3c Mon Sep 17 00:00:00 2001 From: tomyrd Date: Fri, 21 Feb 2025 16:10:27 -0300 Subject: [PATCH 28/34] feat: add state sync component to client constructor --- bin/miden-cli/src/lib.rs | 23 ++++++++++++++--- bin/miden-cli/src/tests.rs | 24 ++++++++++++------ crates/rust-client/src/lib.rs | 8 ++++++ crates/rust-client/src/mock.rs | 17 +++++++++++-- crates/rust-client/src/sync/mod.rs | 19 +++----------- crates/rust-client/src/sync/state_sync.rs | 8 +++--- crates/web-client/src/lib.rs | 23 +++++++++++++++-- tests/integration/common.rs | 25 ++++++++++++------- .../integration/custom_transactions_tests.rs | 1 - tests/integration/onchain_tests.rs | 1 - 10 files changed, 103 insertions(+), 46 deletions(-) diff --git a/bin/miden-cli/src/lib.rs b/bin/miden-cli/src/lib.rs index d9f1deebf..eaad48c91 100644 --- a/bin/miden-cli/src/lib.rs +++ b/bin/miden-cli/src/lib.rs @@ -9,6 +9,7 @@ use miden_client::{ crypto::{FeltRng, RpoRandomCoin}, rpc::TonicRpcClient, store::{sqlite_store::SqliteStore, NoteFilter as ClientNoteFilter, OutputNoteRecord, Store}, + sync::{on_note_received, StateSync}, Client, ClientError, Felt, IdPrefixFetchError, }; use rand::Rng; @@ -111,14 +112,28 @@ impl Cli { .map_err(CliError::KeyStore)?; let authenticator = ClientAuthenticator::new(rng, keystore.clone()); + let rpc_api = Arc::new(TonicRpcClient::new( + &cli_config.rpc.endpoint.clone().into(), + cli_config.rpc.timeout_ms, + )); + + let state_sync_component = StateSync::new( + rpc_api.clone(), + Box::new({ + let store_clone = store.clone(); + move |committed_note, public_note| { + Box::pin(on_note_received(store_clone.clone(), committed_note, public_note)) + } + }), + Box::new(move |_committed_note| Box::pin(async { Ok(true) })), + ); + let client = Client::new( - Arc::new(TonicRpcClient::new( - &cli_config.rpc.endpoint.clone().into(), - cli_config.rpc.timeout_ms, - )), + rpc_api, rng, store as Arc, Arc::new(authenticator), + state_sync_component, in_debug_mode, ); diff --git a/bin/miden-cli/src/tests.rs b/bin/miden-cli/src/tests.rs index d7111e800..21ea835d3 100644 --- a/bin/miden-cli/src/tests.rs +++ b/bin/miden-cli/src/tests.rs @@ -22,6 +22,7 @@ use miden_client::{ }, rpc::{Endpoint, TonicRpcClient}, store::{sqlite_store::SqliteStore, NoteFilter}, + sync::{on_note_received, StateSync}, testing::account_id::ACCOUNT_ID_OFF_CHAIN_SENDER, transaction::{OutputNote, TransactionRequestBuilder}, utils::Serializable, @@ -748,16 +749,23 @@ async fn create_test_client_with_store_path(store_path: &Path) -> (TestClient, F let rng = RpoRandomCoin::new(coin_seed.map(Felt::new)); let keystore = FilesystemKeyStore::new(temp_dir()).unwrap(); - let authenticator = ClientAuthenticator::new(rng, keystore.clone()); + + let rpc_api = Arc::new(TonicRpcClient::new(&rpc_config.endpoint.into(), rpc_config.timeout_ms)); + + let state_sync_component = StateSync::new( + rpc_api.clone(), + Box::new({ + let store_clone = store.clone(); + move |committed_note, public_note| { + Box::pin(on_note_received(store_clone.clone(), committed_note, public_note)) + } + }), + Box::new(move |_committed_note| Box::pin(async { Ok(true) })), + ); + ( - TestClient::new( - Arc::new(TonicRpcClient::new(&rpc_config.endpoint.into(), rpc_config.timeout_ms)), - rng, - store, - Arc::new(authenticator), - true, - ), + TestClient::new(rpc_api, rng, store, Arc::new(authenticator), state_sync_component, true), keystore, ) } diff --git a/crates/rust-client/src/lib.rs b/crates/rust-client/src/lib.rs index 67bc48179..ba4a907e6 100644 --- a/crates/rust-client/src/lib.rs +++ b/crates/rust-client/src/lib.rs @@ -192,6 +192,7 @@ use miden_tx::{ }; use rpc::NodeRpcClient; use store::{data_store::ClientDataStore, Store}; +use sync::StateSync; use tracing::info; // MIDEN CLIENT @@ -218,6 +219,9 @@ pub struct Client { tx_prover: Arc, /// An instance of a [`TransactionExecutor`] that will be used to execute transactions. tx_executor: TransactionExecutor, + /// An instance of a [`StateSync`] component that will be used to synchronize the client's + /// state with the state of the Miden network. + state_sync_component: StateSync, /// Flag to enable the debug mode for scripts compilation and execution. in_debug_mode: bool, } @@ -240,6 +244,8 @@ impl Client { /// store as the one for `store`, but it doesn't have to be the **same instance**. /// - `authenticator`: Defines the transaction authenticator that will be used by the /// transaction executor whenever a signature is requested from within the VM. + /// - `state_sync_component`: An instance of [`StateSync`] that will be used to synchronize the + /// client's state with the state of the Miden network. /// - `in_debug_mode`: Instantiates the transaction executor (and in turn, its compiler) in /// debug mode, which will enable debug logs for scripts compiled with this mode for easier /// MASM debugging. @@ -252,6 +258,7 @@ impl Client { rng: R, store: Arc, authenticator: Arc, + state_sync_component: StateSync, in_debug_mode: bool, ) -> Self { let data_store = Arc::new(ClientDataStore::new(store.clone())) as Arc; @@ -270,6 +277,7 @@ impl Client { rpc_api, tx_prover, tx_executor, + state_sync_component, in_debug_mode, } } diff --git a/crates/rust-client/src/mock.rs b/crates/rust-client/src/mock.rs index 6d2c1a339..43c043e69 100644 --- a/crates/rust-client/src/mock.rs +++ b/crates/rust-client/src/mock.rs @@ -44,6 +44,7 @@ use crate::{ NodeRpcClient, RpcError, }, store::sqlite_store::SqliteStore, + sync::{on_note_received, StateSync}, transaction::ForeignAccount, Client, }; @@ -320,11 +321,23 @@ pub async fn create_test_client() -> (MockClient, MockRpcApi, FilesystemKeyStore let keystore = FilesystemKeyStore::new(temp_dir()).unwrap(); - let authenticator = ClientAuthenticator::new(rng, keystore.clone()); + let authenticator = Arc::new(ClientAuthenticator::new(rng, keystore.clone())); let rpc_api = MockRpcApi::new(); let arc_rpc_api = Arc::new(rpc_api.clone()); - let client = MockClient::new(arc_rpc_api, rng, store, Arc::new(authenticator.clone()), true); + let state_sync_component = StateSync::new( + arc_rpc_api.clone(), + Box::new({ + let store_clone = store.clone(); + move |committed_note, public_note| { + Box::pin(on_note_received(store_clone.clone(), committed_note, public_note)) + } + }), + Box::new(move |_committed_note| Box::pin(async { Ok(true) })), + ); + + let client = + MockClient::new(arc_rpc_api, rng, store, authenticator, state_sync_component, true); (client, rpc_api, keystore) } diff --git a/crates/rust-client/src/sync/mod.rs b/crates/rust-client/src/sync/mod.rs index c32bd02e6..f0f03ba4a 100644 --- a/crates/rust-client/src/sync/mod.rs +++ b/crates/rust-client/src/sync/mod.rs @@ -54,7 +54,7 @@ //! `committed_note_updates` and `consumed_note_updates`) to understand how the sync data is //! processed and applied to the local store. -use alloc::{boxed::Box, vec::Vec}; +use alloc::vec::Vec; use core::cmp::max; use miden_objects::{ @@ -64,7 +64,6 @@ use miden_objects::{ note::{NoteId, NoteTag, Nullifier}, transaction::TransactionId, }; -use state_sync::on_note_received; use crate::{store::NoteFilter, Client, ClientError}; @@ -74,7 +73,7 @@ mod tag; pub use tag::{NoteTagRecord, NoteTagSource}; mod state_sync; -pub use state_sync::{OnNoteReceived, OnNullifierReceived, StateSync}; +pub use state_sync::{on_note_received, OnNoteReceived, OnNullifierReceived, StateSync}; mod state_sync_update; pub use state_sync_update::StateSyncUpdate; @@ -117,17 +116,6 @@ impl Client { pub async fn sync_state(&mut self) -> Result { _ = self.ensure_genesis_in_place().await?; - let state_sync = StateSync::new( - self.rpc_api.clone(), - Box::new({ - let store_clone = self.store.clone(); - move |committed_note, public_note| { - Box::pin(on_note_received(store_clone.clone(), committed_note, public_note)) - } - }), - Box::new(move |_committed_note| Box::pin(async { Ok(true) })), - ); - // Get current state of the client let accounts = self .store @@ -144,7 +132,8 @@ impl Client { let unspent_output_notes = self.store.get_output_notes(NoteFilter::Unspent).await?; // Get the sync update from the network - let state_sync_update = state_sync + let state_sync_update = self + .state_sync_component .sync_state( self.build_current_partial_mmr().await?, accounts, diff --git a/crates/rust-client/src/sync/state_sync.rs b/crates/rust-client/src/sync/state_sync.rs index 5d04554a9..7f41be219 100644 --- a/crates/rust-client/src/sync/state_sync.rs +++ b/crates/rust-client/src/sync/state_sync.rs @@ -118,7 +118,7 @@ impl StateSync { /// * `unspent_input_notes` - The current state of unspent input notes tracked by the client. /// * `unspent_output_notes` - The current state of unspent output notes tracked by the client. pub async fn sync_state( - mut self, + &self, mut current_partial_mmr: PartialMmr, accounts: Vec, note_tags: Vec, @@ -180,7 +180,7 @@ impl StateSync { /// The `sync_state_update` field of the struct will be updated with the new changes from this /// step. async fn sync_state_step( - &mut self, + &self, state_sync_update: &mut StateSyncUpdate, current_partial_mmr: &mut PartialMmr, accounts: &[AccountHeader], @@ -253,7 +253,7 @@ impl StateSync { /// match the one received from the node. The client will need to handle these cases as they /// could be a stale account state or a reason to lock the account. async fn account_state_sync( - &mut self, + &self, accounts: &[AccountHeader], account_hash_updates: &[(AccountId, Digest)], ) -> Result { @@ -319,7 +319,7 @@ impl StateSync { /// * Local tracked transactions that were discarded because the notes that they were processing /// were nullified by an another transaction. async fn note_state_sync( - &mut self, + &self, note_updates: &mut NoteUpdates, note_inclusions: Vec, transactions: Vec, diff --git a/crates/web-client/src/lib.rs b/crates/web-client/src/lib.rs index 268305528..cda8879c8 100644 --- a/crates/web-client/src/lib.rs +++ b/crates/web-client/src/lib.rs @@ -6,6 +6,7 @@ use miden_client::{ authenticator::{keystore::WebKeyStore, ClientAuthenticator}, rpc::WebTonicRpcClient, store::web_store::WebStore, + sync::{on_note_received, StateSync}, Client, }; use miden_objects::{crypto::rand::RpoRandomCoin, Felt}; @@ -90,8 +91,26 @@ impl WebClient { self.remote_prover = prover_url.map(|prover_url| Arc::new(RemoteTransactionProver::new(prover_url))); - self.inner = - Some(Client::new(web_rpc_client, rng, web_store.clone(), authenticator, false)); + + let state_sync_component = StateSync::new( + web_rpc_client.clone(), + Box::new({ + let store_clone = web_store.clone(); + move |committed_note, public_note| { + Box::pin(on_note_received(store_clone.clone(), committed_note, public_note)) + } + }), + Box::new(move |_committed_note| Box::pin(async { Ok(true) })), + ); + + self.inner = Some(Client::new( + web_rpc_client, + rng, + web_store.clone(), + authenticator, + state_sync_component, + false, + )); self.store = Some(web_store); self.keystore = Some(keystore); diff --git a/tests/integration/common.rs b/tests/integration/common.rs index f08b1d5d4..55c9b9e31 100644 --- a/tests/integration/common.rs +++ b/tests/integration/common.rs @@ -14,7 +14,7 @@ use miden_client::{ note::create_p2id_note, rpc::{Endpoint, RpcError, TonicRpcClient}, store::{sqlite_store::SqliteStore, NoteFilter, TransactionFilter}, - sync::SyncSummary, + sync::{on_note_received, StateSync, SyncSummary}, testing::account_id::ACCOUNT_ID_REGULAR_ACCOUNT_UPDATABLE_CODE_OFF_CHAIN, transaction::{ DataStoreError, TransactionExecutorError, TransactionRequest, TransactionRequestBuilder, @@ -61,16 +61,23 @@ pub async fn create_test_client() -> (TestClient, FilesystemKeyStore) { let rng = RpoRandomCoin::new(coin_seed.map(Felt::new)); let keystore = FilesystemKeyStore::new(auth_path).unwrap(); - let authenticator = ClientAuthenticator::new(rng, keystore.clone()); + + let rpc_api = Arc::new(TonicRpcClient::new(&rpc_endpoint, rpc_timeout)); + + let state_sync_component = StateSync::new( + rpc_api.clone(), + Box::new({ + let store_clone = store.clone(); + move |committed_note, public_note| { + Box::pin(on_note_received(store_clone.clone(), committed_note, public_note)) + } + }), + Box::new(move |_committed_note| Box::pin(async { Ok(true) })), + ); + ( - TestClient::new( - Arc::new(TonicRpcClient::new(&rpc_endpoint, rpc_timeout)), - rng, - store, - Arc::new(authenticator), - true, - ), + TestClient::new(rpc_api, rng, store, Arc::new(authenticator), state_sync_component, true), keystore, ) } diff --git a/tests/integration/custom_transactions_tests.rs b/tests/integration/custom_transactions_tests.rs index 41e006add..9caee4e90 100644 --- a/tests/integration/custom_transactions_tests.rs +++ b/tests/integration/custom_transactions_tests.rs @@ -1,5 +1,4 @@ use miden_client::{ - authenticator::keystore, note::NoteExecutionHint, store::NoteFilter, transaction::{InputNote, TransactionRequest, TransactionRequestBuilder}, diff --git a/tests/integration/onchain_tests.rs b/tests/integration/onchain_tests.rs index 0092b4d85..859533ebe 100644 --- a/tests/integration/onchain_tests.rs +++ b/tests/integration/onchain_tests.rs @@ -1,5 +1,4 @@ use miden_client::{ - account::build_wallet_id, auth::AuthSecretKey, authenticator::keystore::KeyStore, store::{InputNoteState, NoteFilter}, From d1c8490cb4e8a02ba9bc8b030320b769f59fa0d0 Mon Sep 17 00:00:00 2001 From: tomyrd Date: Mon, 24 Feb 2025 11:07:35 -0300 Subject: [PATCH 29/34] fix: use `Nullifier::prefix` --- crates/rust-client/src/rpc/mod.rs | 4 +--- crates/rust-client/src/sync/mod.rs | 11 ++--------- crates/rust-client/src/sync/state_sync.rs | 9 ++++----- 3 files changed, 7 insertions(+), 17 deletions(-) diff --git a/crates/rust-client/src/rpc/mod.rs b/crates/rust-client/src/rpc/mod.rs index 27fc0b174..83fb11268 100644 --- a/crates/rust-client/src/rpc/mod.rs +++ b/crates/rust-client/src/rpc/mod.rs @@ -84,7 +84,6 @@ pub use web_tonic_client::WebTonicRpcClient; use crate::{ store::{input_note_states::UnverifiedNoteState, InputNoteRecord}, - sync::get_nullifier_prefix, transaction::ForeignAccount, }; @@ -183,8 +182,7 @@ pub trait NodeRpcClient { &self, nullifier: &Nullifier, ) -> Result, RpcError> { - let nullifiers = - self.check_nullifiers_by_prefix(&[get_nullifier_prefix(nullifier)]).await?; + let nullifiers = self.check_nullifiers_by_prefix(&[nullifier.prefix()]).await?; Ok(nullifiers.iter().find(|(n, _)| n == nullifier).map(|(_, block_num)| *block_num)) } diff --git a/crates/rust-client/src/sync/mod.rs b/crates/rust-client/src/sync/mod.rs index f0f03ba4a..794206a06 100644 --- a/crates/rust-client/src/sync/mod.rs +++ b/crates/rust-client/src/sync/mod.rs @@ -61,7 +61,7 @@ use miden_objects::{ account::AccountId, block::BlockNumber, crypto::rand::FeltRng, - note::{NoteId, NoteTag, Nullifier}, + note::{NoteId, NoteTag}, transaction::TransactionId, }; @@ -81,10 +81,7 @@ pub use state_sync_update::StateSyncUpdate; // CONSTANTS // ================================================================================================ -/// The number of bits to shift identifiers for in use of filters. -pub(crate) const FILTER_ID_SHIFT: u8 = 48; - -/// Client syncronization methods. +// Client syncronization methods. impl Client { // SYNC STATE // -------------------------------------------------------------------------------------------- @@ -157,10 +154,6 @@ impl Client { } } -pub(crate) fn get_nullifier_prefix(nullifier: &Nullifier) -> u16 { - (nullifier.inner()[3].as_int() >> FILTER_ID_SHIFT) as u16 -} - // SYNC SUMMARY // ================================================================================================ diff --git a/crates/rust-client/src/sync/state_sync.rs b/crates/rust-client/src/sync/state_sync.rs index 7f41be219..eb2214700 100644 --- a/crates/rust-client/src/sync/state_sync.rs +++ b/crates/rust-client/src/sync/state_sync.rs @@ -10,7 +10,7 @@ use miden_objects::{ }; use tracing::info; -use super::{block_header::BlockUpdates, get_nullifier_prefix, StateSyncUpdate}; +use super::{block_header::BlockUpdates, StateSyncUpdate}; use crate::{ account::AccountUpdates, note::{NoteScreener, NoteUpdates}, @@ -135,7 +135,7 @@ impl StateSync { // (it only returns nullifiers from current_block_num until // response.block_header.block_num()) let mut nullifiers_tags: Vec = - unspent_nullifiers.map(|nullifier| get_nullifier_prefix(&nullifier)).collect(); + unspent_nullifiers.map(|nullifier| nullifier.prefix()).collect(); let mut state_sync_update = StateSyncUpdate { note_updates: NoteUpdates::new(unspent_input_notes, unspent_output_notes), @@ -159,10 +159,9 @@ impl StateSync { .note_updates .updated_input_notes() .filter(|note| { - note.is_committed() - && !nullifiers_tags.contains(&get_nullifier_prefix(¬e.nullifier())) + note.is_committed() && !nullifiers_tags.contains(¬e.nullifier().prefix()) }) - .map(|note| get_nullifier_prefix(¬e.nullifier())) + .map(|note| note.nullifier().prefix()) .collect::>(), ); } From 728f3f652c62944831327c7c587ab59cd51d6a05 Mon Sep 17 00:00:00 2001 From: tomyrd Date: Mon, 24 Feb 2025 12:09:58 -0300 Subject: [PATCH 30/34] review: address suggestions --- crates/rust-client/src/lib.rs | 17 ++++++++++++++++- crates/rust-client/src/sync/state_sync.rs | 15 +++++++++------ 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/crates/rust-client/src/lib.rs b/crates/rust-client/src/lib.rs index ba4a907e6..0f3100278 100644 --- a/crates/rust-client/src/lib.rs +++ b/crates/rust-client/src/lib.rs @@ -58,6 +58,7 @@ //! crypto::RpoRandomCoin, //! rpc::{Endpoint, TonicRpcClient}, //! store::{sqlite_store::SqliteStore, Store}, +//! sync::{on_note_received, StateSync}, //! Client, Felt, //! }; //! use miden_objects::crypto::rand::FeltRng; @@ -82,11 +83,25 @@ //! //! // Instantiate the client using a Tonic RPC client //! let endpoint = Endpoint::new("https".into(), "localhost".into(), Some(57291)); +//! let rpc_api = Arc::new(TonicRpcClient::new(&endpoint, 10_000)); +//! +//! let state_sync_component = StateSync::new( +//! rpc_api.clone(), +//! Box::new({ +//! let store_clone = store.clone(); +//! move |committed_note, public_note| { +//! Box::pin(on_note_received(store_clone.clone(), committed_note, public_note)) +//! } +//! }), +//! Box::new(move |_committed_note| Box::pin(async { Ok(true) })), +//! ); +//! //! let client: Client = Client::new( -//! Arc::new(TonicRpcClient::new(&endpoint, 10_000)), +//! rpc_api, //! rng, //! store, //! Arc::new(authenticator), +//! state_sync_component, //! false, // Set to true for debug mode, if needed. //! ); //! diff --git a/crates/rust-client/src/sync/state_sync.rs b/crates/rust-client/src/sync/state_sync.rs index eb2214700..866360a80 100644 --- a/crates/rust-client/src/sync/state_sync.rs +++ b/crates/rust-client/src/sync/state_sync.rs @@ -138,6 +138,10 @@ impl StateSync { unspent_nullifiers.map(|nullifier| nullifier.prefix()).collect(); let mut state_sync_update = StateSyncUpdate { + block_num: (u32::try_from(current_partial_mmr.num_leaves() - 1).expect( + "The number of leaves in the MMR should be greater than 0 and less than 2^32", + )) + .into(), note_updates: NoteUpdates::new(unspent_input_notes, unspent_output_notes), ..Default::default() }; @@ -186,9 +190,8 @@ impl StateSync { note_tags: &[NoteTag], nullifiers_tags: &[u16], ) -> Result { - let current_block_num = (u32::try_from(current_partial_mmr.num_leaves() - 1) - .expect("The number of leaves in the MMR should be greater than 0 and less than 2^32")) - .into(); + let current_block_num = state_sync_update.block_num; + let account_ids: Vec = accounts.iter().map(AccountHeader::id).collect(); let response = self @@ -196,17 +199,17 @@ impl StateSync { .sync_state(current_block_num, &account_ids, note_tags, nullifiers_tags) .await?; - state_sync_update.block_num = response.block_header.block_num(); - // We don't need to continue if the chain has not advanced, there are no new changes if response.block_header.block_num() == current_block_num { return Ok(false); } + state_sync_update.block_num = response.block_header.block_num(); + let account_updates = self.account_state_sync(accounts, &response.account_hash_updates).await?; - state_sync_update.account_updates = account_updates; + state_sync_update.account_updates.extend(account_updates); let (found_relevant_note, transaction_updates) = self .note_state_sync( From 2090593097f4a23d74ff15b5477ca21470ccfb57 Mon Sep 17 00:00:00 2001 From: tomyrd Date: Mon, 24 Feb 2025 14:59:09 -0300 Subject: [PATCH 31/34] review: remove `OnNullifierReceived` callback --- bin/miden-cli/src/lib.rs | 1 - bin/miden-cli/src/tests.rs | 1 - crates/rust-client/src/lib.rs | 1 - crates/rust-client/src/mock.rs | 1 - crates/rust-client/src/sync/mod.rs | 2 +- crates/rust-client/src/sync/state_sync.rs | 33 +++++------------------ crates/web-client/src/lib.rs | 1 - tests/integration/common.rs | 1 - 8 files changed, 7 insertions(+), 34 deletions(-) diff --git a/bin/miden-cli/src/lib.rs b/bin/miden-cli/src/lib.rs index eaad48c91..29e4d7782 100644 --- a/bin/miden-cli/src/lib.rs +++ b/bin/miden-cli/src/lib.rs @@ -125,7 +125,6 @@ impl Cli { Box::pin(on_note_received(store_clone.clone(), committed_note, public_note)) } }), - Box::new(move |_committed_note| Box::pin(async { Ok(true) })), ); let client = Client::new( diff --git a/bin/miden-cli/src/tests.rs b/bin/miden-cli/src/tests.rs index 21ea835d3..437c43bfd 100644 --- a/bin/miden-cli/src/tests.rs +++ b/bin/miden-cli/src/tests.rs @@ -761,7 +761,6 @@ async fn create_test_client_with_store_path(store_path: &Path) -> (TestClient, F Box::pin(on_note_received(store_clone.clone(), committed_note, public_note)) } }), - Box::new(move |_committed_note| Box::pin(async { Ok(true) })), ); ( diff --git a/crates/rust-client/src/lib.rs b/crates/rust-client/src/lib.rs index 0f3100278..dafbb85ea 100644 --- a/crates/rust-client/src/lib.rs +++ b/crates/rust-client/src/lib.rs @@ -93,7 +93,6 @@ //! Box::pin(on_note_received(store_clone.clone(), committed_note, public_note)) //! } //! }), -//! Box::new(move |_committed_note| Box::pin(async { Ok(true) })), //! ); //! //! let client: Client = Client::new( diff --git a/crates/rust-client/src/mock.rs b/crates/rust-client/src/mock.rs index 43c043e69..4133dab27 100644 --- a/crates/rust-client/src/mock.rs +++ b/crates/rust-client/src/mock.rs @@ -333,7 +333,6 @@ pub async fn create_test_client() -> (MockClient, MockRpcApi, FilesystemKeyStore Box::pin(on_note_received(store_clone.clone(), committed_note, public_note)) } }), - Box::new(move |_committed_note| Box::pin(async { Ok(true) })), ); let client = diff --git a/crates/rust-client/src/sync/mod.rs b/crates/rust-client/src/sync/mod.rs index 794206a06..b26c56c36 100644 --- a/crates/rust-client/src/sync/mod.rs +++ b/crates/rust-client/src/sync/mod.rs @@ -73,7 +73,7 @@ mod tag; pub use tag::{NoteTagRecord, NoteTagSource}; mod state_sync; -pub use state_sync::{on_note_received, OnNoteReceived, OnNullifierReceived, StateSync}; +pub use state_sync::{on_note_received, OnNoteReceived, StateSync}; mod state_sync_update; pub use state_sync_update::StateSyncUpdate; diff --git a/crates/rust-client/src/sync/state_sync.rs b/crates/rust-client/src/sync/state_sync.rs index 866360a80..4b6573688 100644 --- a/crates/rust-client/src/sync/state_sync.rs +++ b/crates/rust-client/src/sync/state_sync.rs @@ -41,15 +41,6 @@ pub type OnNoteReceived = Box< ) -> Pin>>>, >; -/// Callback to be executed when a nullifier is received as part of the the sync response. It -/// receives the nullifier update received from the network. -/// -/// It returns a boolean indicating if the received note update is relevant -/// If the return value is `false`, it gets discarded. If it is `true`, the update gets committed to -/// the client's store. -pub type OnNullifierReceived = - Box Pin>>>>; - // STATE SYNC // ================================================================================================ @@ -67,8 +58,6 @@ pub struct StateSync { rpc_api: Arc, /// Callback to be executed when a new note inclusion is received. on_note_received: OnNoteReceived, - /// Callback to be executed when a nullifier is received. - on_nullifier_received: OnNullifierReceived, } impl StateSync { @@ -79,16 +68,8 @@ impl StateSync { /// * `rpc_api` - The RPC client used to communicate with the node. /// * `on_note_received` - A callback to be executed when a new note inclusion is received. /// * `on_nullifier_received` - A callback to be executed when a nullifier is received. - pub fn new( - rpc_api: Arc, - on_note_received: OnNoteReceived, - on_nullifier_received: OnNullifierReceived, - ) -> Self { - Self { - rpc_api, - on_note_received, - on_nullifier_received, - } + pub fn new(rpc_api: Arc, on_note_received: OnNoteReceived) -> Self { + Self { rpc_api, on_note_received } } /// Syncs the state of the client with the chain tip of the node, returning the updates that @@ -358,13 +339,11 @@ impl StateSync { // Process nullifiers for nullifier_update in nullifiers { - if (self.on_nullifier_received)(nullifier_update.clone()).await? { - let discarded_transaction = note_updates - .apply_nullifiers_state_transitions(&nullifier_update, &transactions)?; + let discarded_transaction = note_updates + .apply_nullifiers_state_transitions(&nullifier_update, &transactions)?; - if let Some(transaction_id) = discarded_transaction { - discarded_transactions.push(transaction_id); - } + if let Some(transaction_id) = discarded_transaction { + discarded_transactions.push(transaction_id); } } diff --git a/crates/web-client/src/lib.rs b/crates/web-client/src/lib.rs index cda8879c8..98709b071 100644 --- a/crates/web-client/src/lib.rs +++ b/crates/web-client/src/lib.rs @@ -100,7 +100,6 @@ impl WebClient { Box::pin(on_note_received(store_clone.clone(), committed_note, public_note)) } }), - Box::new(move |_committed_note| Box::pin(async { Ok(true) })), ); self.inner = Some(Client::new( diff --git a/tests/integration/common.rs b/tests/integration/common.rs index 55c9b9e31..ae904b5d1 100644 --- a/tests/integration/common.rs +++ b/tests/integration/common.rs @@ -73,7 +73,6 @@ pub async fn create_test_client() -> (TestClient, FilesystemKeyStore) { Box::pin(on_note_received(store_clone.clone(), committed_note, public_note)) } }), - Box::new(move |_committed_note| Box::pin(async { Ok(true) })), ); ( From 807a92942adfa68c0fbc236b538422e9519b07c3 Mon Sep 17 00:00:00 2001 From: tomyrd Date: Wed, 26 Feb 2025 19:01:18 -0300 Subject: [PATCH 32/34] revert efb9589 --- bin/miden-cli/src/lib.rs | 22 ++++----------------- bin/miden-cli/src/tests.rs | 23 ++++++++-------------- crates/rust-client/src/builder.rs | 13 ------------ crates/rust-client/src/lib.rs | 8 -------- crates/rust-client/src/mock.rs | 16 ++------------- crates/rust-client/src/sync/mod.rs | 15 +++++++++++--- crates/rust-client/src/sync/state_sync.rs | 8 ++++---- crates/web-client/src/lib.rs | 24 +++-------------------- tests/integration/common.rs | 24 +++++++++-------------- 9 files changed, 42 insertions(+), 111 deletions(-) diff --git a/bin/miden-cli/src/lib.rs b/bin/miden-cli/src/lib.rs index 29e4d7782..d9f1deebf 100644 --- a/bin/miden-cli/src/lib.rs +++ b/bin/miden-cli/src/lib.rs @@ -9,7 +9,6 @@ use miden_client::{ crypto::{FeltRng, RpoRandomCoin}, rpc::TonicRpcClient, store::{sqlite_store::SqliteStore, NoteFilter as ClientNoteFilter, OutputNoteRecord, Store}, - sync::{on_note_received, StateSync}, Client, ClientError, Felt, IdPrefixFetchError, }; use rand::Rng; @@ -112,27 +111,14 @@ impl Cli { .map_err(CliError::KeyStore)?; let authenticator = ClientAuthenticator::new(rng, keystore.clone()); - let rpc_api = Arc::new(TonicRpcClient::new( - &cli_config.rpc.endpoint.clone().into(), - cli_config.rpc.timeout_ms, - )); - - let state_sync_component = StateSync::new( - rpc_api.clone(), - Box::new({ - let store_clone = store.clone(); - move |committed_note, public_note| { - Box::pin(on_note_received(store_clone.clone(), committed_note, public_note)) - } - }), - ); - let client = Client::new( - rpc_api, + Arc::new(TonicRpcClient::new( + &cli_config.rpc.endpoint.clone().into(), + cli_config.rpc.timeout_ms, + )), rng, store as Arc, Arc::new(authenticator), - state_sync_component, in_debug_mode, ); diff --git a/bin/miden-cli/src/tests.rs b/bin/miden-cli/src/tests.rs index 50bb13dd8..edd22915d 100644 --- a/bin/miden-cli/src/tests.rs +++ b/bin/miden-cli/src/tests.rs @@ -22,7 +22,6 @@ use miden_client::{ }, rpc::{Endpoint, TonicRpcClient}, store::{sqlite_store::SqliteStore, NoteFilter}, - sync::{on_note_received, StateSync}, testing::account_id::ACCOUNT_ID_OFF_CHAIN_SENDER, transaction::{OutputNote, TransactionRequestBuilder}, utils::Serializable, @@ -749,22 +748,16 @@ async fn create_test_client_with_store_path(store_path: &Path) -> (TestClient, F let rng = RpoRandomCoin::new(coin_seed.map(Felt::new)); let keystore = FilesystemKeyStore::new(temp_dir()).unwrap(); - let authenticator = ClientAuthenticator::new(rng, keystore.clone()); - - let rpc_api = Arc::new(TonicRpcClient::new(&rpc_config.endpoint.into(), rpc_config.timeout_ms)); - - let state_sync_component = StateSync::new( - rpc_api.clone(), - Box::new({ - let store_clone = store.clone(); - move |committed_note, public_note| { - Box::pin(on_note_received(store_clone.clone(), committed_note, public_note)) - } - }), - ); + let authenticator = ClientAuthenticator::new(rng, keystore.clone()); ( - TestClient::new(rpc_api, rng, store, Arc::new(authenticator), state_sync_component, true), + TestClient::new( + Arc::new(TonicRpcClient::new(&rpc_config.endpoint.into(), rpc_config.timeout_ms)), + rng, + store, + Arc::new(authenticator), + true, + ), keystore, ) } diff --git a/crates/rust-client/src/builder.rs b/crates/rust-client/src/builder.rs index 2a4802b9f..5194c34de 100644 --- a/crates/rust-client/src/builder.rs +++ b/crates/rust-client/src/builder.rs @@ -1,5 +1,4 @@ use alloc::{ - boxed::Box, string::{String, ToString}, sync::Arc, }; @@ -11,7 +10,6 @@ use crate::{ authenticator::{keystore::FilesystemKeyStore, ClientAuthenticator}, rpc::{Endpoint, NodeRpcClient, TonicRpcClient}, store::{sqlite_store::SqliteStore, Store}, - sync::{on_note_received, StateSync}, Client, ClientError, Felt, }; @@ -214,22 +212,11 @@ impl ClientBuilder { let authenticator = ClientAuthenticator::new(rng, keystore); - let state_sync_component = StateSync::new( - rpc_api.clone(), - Box::new({ - let store_clone = arc_store.clone(); - move |committed_note, public_note| { - Box::pin(on_note_received(store_clone.clone(), committed_note, public_note)) - } - }), - ); - Ok(Client::new( rpc_api, rng, arc_store, Arc::new(authenticator), - state_sync_component, self.in_debug_mode, )) } diff --git a/crates/rust-client/src/lib.rs b/crates/rust-client/src/lib.rs index 00d518dd8..0446002f6 100644 --- a/crates/rust-client/src/lib.rs +++ b/crates/rust-client/src/lib.rs @@ -213,7 +213,6 @@ use miden_tx::{ }; use rpc::NodeRpcClient; use store::{data_store::ClientDataStore, Store}; -use sync::StateSync; use tracing::info; // MIDEN CLIENT @@ -240,9 +239,6 @@ pub struct Client { tx_prover: Arc, /// An instance of a [`TransactionExecutor`] that will be used to execute transactions. tx_executor: TransactionExecutor, - /// An instance of a [`StateSync`] component that will be used to synchronize the client's - /// state with the state of the Miden network. - state_sync_component: StateSync, /// Flag to enable the debug mode for scripts compilation and execution. in_debug_mode: bool, } @@ -265,8 +261,6 @@ impl Client { /// store as the one for `store`, but it doesn't have to be the **same instance**. /// - `authenticator`: Defines the transaction authenticator that will be used by the /// transaction executor whenever a signature is requested from within the VM. - /// - `state_sync_component`: An instance of [`StateSync`] that will be used to synchronize the - /// client's state with the state of the Miden network. /// - `in_debug_mode`: Instantiates the transaction executor (and in turn, its compiler) in /// debug mode, which will enable debug logs for scripts compiled with this mode for easier /// MASM debugging. @@ -279,7 +273,6 @@ impl Client { rng: R, store: Arc, authenticator: Arc, - state_sync_component: StateSync, in_debug_mode: bool, ) -> Self { let data_store = Arc::new(ClientDataStore::new(store.clone())) as Arc; @@ -298,7 +291,6 @@ impl Client { rpc_api, tx_prover, tx_executor, - state_sync_component, in_debug_mode, } } diff --git a/crates/rust-client/src/mock.rs b/crates/rust-client/src/mock.rs index d2eddec86..a384815fd 100644 --- a/crates/rust-client/src/mock.rs +++ b/crates/rust-client/src/mock.rs @@ -45,7 +45,6 @@ use crate::{ NodeRpcClient, RpcError, }, store::sqlite_store::SqliteStore, - sync::{on_note_received, StateSync}, transaction::ForeignAccount, Client, }; @@ -311,22 +310,11 @@ pub async fn create_test_client() -> (MockClient, MockRpcApi, FilesystemKeyStore let keystore = FilesystemKeyStore::new(temp_dir()).unwrap(); - let authenticator = Arc::new(ClientAuthenticator::new(rng, keystore.clone())); + let authenticator = ClientAuthenticator::new(rng, keystore.clone()); let rpc_api = MockRpcApi::new(); let arc_rpc_api = Arc::new(rpc_api.clone()); - let state_sync_component = StateSync::new( - arc_rpc_api.clone(), - Box::new({ - let store_clone = store.clone(); - move |committed_note, public_note| { - Box::pin(on_note_received(store_clone.clone(), committed_note, public_note)) - } - }), - ); - - let client = - MockClient::new(arc_rpc_api, rng, store, authenticator, state_sync_component, true); + let client = MockClient::new(arc_rpc_api, rng, store, Arc::new(authenticator.clone()), true); (client, rpc_api, keystore) } diff --git a/crates/rust-client/src/sync/mod.rs b/crates/rust-client/src/sync/mod.rs index b26c56c36..411465076 100644 --- a/crates/rust-client/src/sync/mod.rs +++ b/crates/rust-client/src/sync/mod.rs @@ -54,7 +54,7 @@ //! `committed_note_updates` and `consumed_note_updates`) to understand how the sync data is //! processed and applied to the local store. -use alloc::vec::Vec; +use alloc::{boxed::Box, vec::Vec}; use core::cmp::max; use miden_objects::{ @@ -113,6 +113,16 @@ impl Client { pub async fn sync_state(&mut self) -> Result { _ = self.ensure_genesis_in_place().await?; + let state_sync = StateSync::new( + self.rpc_api.clone(), + Box::new({ + let store_clone = self.store.clone(); + move |committed_note, public_note| { + Box::pin(on_note_received(store_clone.clone(), committed_note, public_note)) + } + }), + ); + // Get current state of the client let accounts = self .store @@ -129,8 +139,7 @@ impl Client { let unspent_output_notes = self.store.get_output_notes(NoteFilter::Unspent).await?; // Get the sync update from the network - let state_sync_update = self - .state_sync_component + let state_sync_update = state_sync .sync_state( self.build_current_partial_mmr().await?, accounts, diff --git a/crates/rust-client/src/sync/state_sync.rs b/crates/rust-client/src/sync/state_sync.rs index e82b2562f..295c399fc 100644 --- a/crates/rust-client/src/sync/state_sync.rs +++ b/crates/rust-client/src/sync/state_sync.rs @@ -99,7 +99,7 @@ impl StateSync { /// * `unspent_input_notes` - The current state of unspent input notes tracked by the client. /// * `unspent_output_notes` - The current state of unspent output notes tracked by the client. pub async fn sync_state( - &self, + mut self, mut current_partial_mmr: PartialMmr, accounts: Vec, note_tags: Vec, @@ -164,7 +164,7 @@ impl StateSync { /// The `sync_state_update` field of the struct will be updated with the new changes from this /// step. async fn sync_state_step( - &self, + &mut self, state_sync_update: &mut StateSyncUpdate, current_partial_mmr: &mut PartialMmr, accounts: &[AccountHeader], @@ -233,7 +233,7 @@ impl StateSync { /// match the one received from the node. The client will need to handle these cases as they /// could be a stale account state or a reason to lock the account. async fn account_state_sync( - &self, + &mut self, accounts: &[AccountHeader], account_hash_updates: &[(AccountId, Digest)], ) -> Result { @@ -299,7 +299,7 @@ impl StateSync { /// * Local tracked transactions that were discarded because the notes that they were processing /// were nullified by an another transaction. async fn note_state_sync( - &self, + &mut self, note_updates: &mut NoteUpdates, note_inclusions: Vec, transactions: Vec, diff --git a/crates/web-client/src/lib.rs b/crates/web-client/src/lib.rs index 7893d0091..6dd96c9aa 100644 --- a/crates/web-client/src/lib.rs +++ b/crates/web-client/src/lib.rs @@ -6,8 +6,7 @@ use miden_client::{ authenticator::{keystore::WebKeyStore, ClientAuthenticator}, rpc::WebTonicRpcClient, store::web_store::WebStore, - sync::{on_note_received, StateSync}, - Client, + Client, RemoteTransactionProver, }; use miden_objects::{crypto::rand::RpoRandomCoin, Felt}; use rand::{rngs::StdRng, Rng, SeedableRng}; @@ -90,25 +89,8 @@ impl WebClient { self.remote_prover = prover_url.map(|prover_url| Arc::new(RemoteTransactionProver::new(prover_url))); - - let state_sync_component = StateSync::new( - web_rpc_client.clone(), - Box::new({ - let store_clone = web_store.clone(); - move |committed_note, public_note| { - Box::pin(on_note_received(store_clone.clone(), committed_note, public_note)) - } - }), - ); - - self.inner = Some(Client::new( - web_rpc_client, - rng, - web_store.clone(), - authenticator, - state_sync_component, - false, - )); + self.inner = + Some(Client::new(web_rpc_client, rng, web_store.clone(), authenticator, false)); self.store = Some(web_store); self.keystore = Some(keystore); diff --git a/tests/integration/common.rs b/tests/integration/common.rs index d7bc47e37..cef8343e4 100644 --- a/tests/integration/common.rs +++ b/tests/integration/common.rs @@ -14,7 +14,7 @@ use miden_client::{ note::create_p2id_note, rpc::{Endpoint, RpcError, TonicRpcClient}, store::{sqlite_store::SqliteStore, NoteFilter, TransactionFilter}, - sync::{on_note_received, StateSync, SyncSummary}, + sync::SyncSummary, testing::account_id::ACCOUNT_ID_REGULAR_ACCOUNT_UPDATABLE_CODE_OFF_CHAIN, transaction::{ DataStoreError, TransactionExecutorError, TransactionRequest, TransactionRequestBuilder, @@ -61,22 +61,16 @@ pub async fn create_test_client() -> (TestClient, FilesystemKeyStore) { let rng = RpoRandomCoin::new(coin_seed.map(Felt::new)); let keystore = FilesystemKeyStore::new(auth_path).unwrap(); - let authenticator = ClientAuthenticator::new(rng, keystore.clone()); - - let rpc_api = Arc::new(TonicRpcClient::new(&rpc_endpoint, rpc_timeout)); - - let state_sync_component = StateSync::new( - rpc_api.clone(), - Box::new({ - let store_clone = store.clone(); - move |committed_note, public_note| { - Box::pin(on_note_received(store_clone.clone(), committed_note, public_note)) - } - }), - ); + let authenticator = ClientAuthenticator::new(rng, keystore.clone()); ( - TestClient::new(rpc_api, rng, store, Arc::new(authenticator), state_sync_component, true), + TestClient::new( + Arc::new(TonicRpcClient::new(&rpc_endpoint, rpc_timeout)), + rng, + store, + Arc::new(authenticator), + true, + ), keystore, ) } From 54b277f70b559f94aee7ee712616f0d76719968d Mon Sep 17 00:00:00 2001 From: tomyrd Date: Wed, 5 Mar 2025 10:49:40 -0300 Subject: [PATCH 33/34] review:address suggestions --- crates/rust-client/src/account/mod.rs | 45 ------ crates/rust-client/src/sync/block_header.rs | 40 ----- crates/rust-client/src/sync/state_sync.rs | 9 +- .../rust-client/src/sync/state_sync_update.rs | 139 +++++++++++++++++- crates/rust-client/src/transaction/mod.rs | 39 ----- 5 files changed, 137 insertions(+), 135 deletions(-) diff --git a/crates/rust-client/src/account/mod.rs b/crates/rust-client/src/account/mod.rs index 2f36ca0b0..bfd66977f 100644 --- a/crates/rust-client/src/account/mod.rs +++ b/crates/rust-client/src/account/mod.rs @@ -300,51 +300,6 @@ pub fn build_wallet_id( Ok(account.id()) } -// ACCOUNT UPDATES -// ================================================================================================ - -#[derive(Debug, Clone, Default)] -/// Contains account changes to apply to the store after a sync request. -pub struct AccountUpdates { - /// Updated public accounts. - updated_public_accounts: Vec, - /// Account hashes received from the network that don't match the currently locally-tracked - /// state of the private accounts. - /// - /// These updates may represent a stale account hash (meaning that the latest local state - /// hasn't been committed). If this is not the case, the account may be locked until the state - /// is restored manually. - mismatched_private_accounts: Vec<(AccountId, Digest)>, -} - -impl AccountUpdates { - /// Creates a new instance of `AccountUpdates`. - pub fn new( - updated_public_accounts: Vec, - mismatched_private_accounts: Vec<(AccountId, Digest)>, - ) -> Self { - Self { - updated_public_accounts, - mismatched_private_accounts, - } - } - - /// Returns the updated public accounts. - pub fn updated_public_accounts(&self) -> &[Account] { - &self.updated_public_accounts - } - - /// Returns the mismatched private accounts. - pub fn mismatched_private_accounts(&self) -> &[(AccountId, Digest)] { - &self.mismatched_private_accounts - } - - pub fn extend(&mut self, other: AccountUpdates) { - self.updated_public_accounts.extend(other.updated_public_accounts); - self.mismatched_private_accounts.extend(other.mismatched_private_accounts); - } -} - // TESTS // ================================================================================================ diff --git a/crates/rust-client/src/sync/block_header.rs b/crates/rust-client/src/sync/block_header.rs index f8c822f3f..614d01b87 100644 --- a/crates/rust-client/src/sync/block_header.rs +++ b/crates/rust-client/src/sync/block_header.rs @@ -13,46 +13,6 @@ use crate::{ Client, ClientError, }; -/// Contains all the block information that needs to be added in the client's store after a sync. - -#[derive(Debug, Clone, Default)] -pub struct BlockUpdates { - /// New block headers to be stored, along with a flag indicating whether the block contains - /// notes that are relevant to the client and the MMR peaks for the block. - block_headers: Vec<(BlockHeader, bool, MmrPeaks)>, - /// New authentication nodes that are meant to be stored in order to authenticate block - /// headers. - new_authentication_nodes: Vec<(InOrderIndex, Digest)>, -} - -impl BlockUpdates { - /// Creates a new instance of [`BlockUpdates`]. - pub fn new( - block_headers: Vec<(BlockHeader, bool, MmrPeaks)>, - new_authentication_nodes: Vec<(InOrderIndex, Digest)>, - ) -> Self { - Self { block_headers, new_authentication_nodes } - } - - /// Returns the new block headers to be stored, along with a flag indicating whether the block - /// contains notes that are relevant to the client and the MMR peaks for the block. - pub fn block_headers(&self) -> &[(BlockHeader, bool, MmrPeaks)] { - &self.block_headers - } - - /// Returns the new authentication nodes that are meant to be stored in order to authenticate - /// block headers. - pub fn new_authentication_nodes(&self) -> &[(InOrderIndex, Digest)] { - &self.new_authentication_nodes - } - - /// Extends the current [`BlockUpdates`] with the provided one. - pub(crate) fn extend(&mut self, other: BlockUpdates) { - self.block_headers.extend(other.block_headers); - self.new_authentication_nodes.extend(other.new_authentication_nodes); - } -} - /// Network information management methods. impl Client { /// Updates committed notes with no MMR data. These could be notes that were diff --git a/crates/rust-client/src/sync/state_sync.rs b/crates/rust-client/src/sync/state_sync.rs index 295c399fc..df3f180af 100644 --- a/crates/rust-client/src/sync/state_sync.rs +++ b/crates/rust-client/src/sync/state_sync.rs @@ -171,14 +171,15 @@ impl StateSync { note_tags: &[NoteTag], _nullifiers_tags: &[u16], ) -> Result { - let current_block_num = state_sync_update.block_num; - let account_ids: Vec = accounts.iter().map(AccountHeader::id).collect(); - let response = self.rpc_api.sync_state(current_block_num, &account_ids, note_tags).await?; + let response = self + .rpc_api + .sync_state(state_sync_update.block_num, &account_ids, note_tags) + .await?; // We don't need to continue if the chain has not advanced, there are no new changes - if response.block_header.block_num() == current_block_num { + if response.block_header.block_num() == state_sync_update.block_num { return Ok(false); } diff --git a/crates/rust-client/src/sync/state_sync_update.rs b/crates/rust-client/src/sync/state_sync_update.rs index 58fdc6029..e00cf5009 100644 --- a/crates/rust-client/src/sync/state_sync_update.rs +++ b/crates/rust-client/src/sync/state_sync_update.rs @@ -1,12 +1,14 @@ -use miden_objects::block::BlockNumber; - -use super::{block_header::BlockUpdates, SyncSummary}; -use crate::{ - account::{Account, AccountUpdates}, - note::NoteUpdates, - transaction::TransactionUpdates, +use miden_objects::{ + account::AccountId, + block::{BlockHeader, BlockNumber}, + crypto::merkle::{InOrderIndex, MmrPeaks}, + transaction::TransactionId, + Digest, }; +use super::SyncSummary; +use crate::{account::Account, note::NoteUpdates, rpc::domain::transaction::TransactionUpdate}; + // STATE SYNC UPDATE // ================================================================================================ @@ -52,3 +54,126 @@ impl From<&StateSyncUpdate> for SyncSummary { ) } } + +/// Contains all the block information that needs to be added in the client's store after a sync. +#[derive(Debug, Clone, Default)] +pub struct BlockUpdates { + /// New block headers to be stored, along with a flag indicating whether the block contains + /// notes that are relevant to the client and the MMR peaks for the block. + block_headers: Vec<(BlockHeader, bool, MmrPeaks)>, + /// New authentication nodes that are meant to be stored in order to authenticate block + /// headers. + new_authentication_nodes: Vec<(InOrderIndex, Digest)>, +} + +impl BlockUpdates { + /// Creates a new instance of [`BlockUpdates`]. + pub fn new( + block_headers: Vec<(BlockHeader, bool, MmrPeaks)>, + new_authentication_nodes: Vec<(InOrderIndex, Digest)>, + ) -> Self { + Self { block_headers, new_authentication_nodes } + } + + /// Returns the new block headers to be stored, along with a flag indicating whether the block + /// contains notes that are relevant to the client and the MMR peaks for the block. + pub fn block_headers(&self) -> &[(BlockHeader, bool, MmrPeaks)] { + &self.block_headers + } + + /// Returns the new authentication nodes that are meant to be stored in order to authenticate + /// block headers. + pub fn new_authentication_nodes(&self) -> &[(InOrderIndex, Digest)] { + &self.new_authentication_nodes + } + + /// Extends the current [`BlockUpdates`] with the provided one. + pub(crate) fn extend(&mut self, other: BlockUpdates) { + self.block_headers.extend(other.block_headers); + self.new_authentication_nodes.extend(other.new_authentication_nodes); + } +} + +/// Contains transaction changes to apply to the store. +#[derive(Default)] +pub struct TransactionUpdates { + /// Transaction updates for any transaction that was committed between the sync request's block + /// number and the response's block number. + committed_transactions: Vec, + /// Transaction IDs for any transactions that were discarded in the sync. + discarded_transactions: Vec, +} + +impl TransactionUpdates { + /// Creates a new [`TransactionUpdate`] + pub fn new( + committed_transactions: Vec, + discarded_transactions: Vec, + ) -> Self { + Self { + committed_transactions, + discarded_transactions, + } + } + + /// Extends the transaction update information with `other`. + pub fn extend(&mut self, other: Self) { + self.committed_transactions.extend(other.committed_transactions); + self.discarded_transactions.extend(other.discarded_transactions); + } + + /// Returns a reference to committed transactions. + pub fn committed_transactions(&self) -> &[TransactionUpdate] { + &self.committed_transactions + } + + /// Returns a reference to discarded transactions. + pub fn discarded_transactions(&self) -> &[TransactionId] { + &self.discarded_transactions + } +} + +// ACCOUNT UPDATES +// ================================================================================================ + +#[derive(Debug, Clone, Default)] +/// Contains account changes to apply to the store after a sync request. +pub struct AccountUpdates { + /// Updated public accounts. + updated_public_accounts: Vec, + /// Account hashes received from the network that don't match the currently locally-tracked + /// state of the private accounts. + /// + /// These updates may represent a stale account hash (meaning that the latest local state + /// hasn't been committed). If this is not the case, the account may be locked until the state + /// is restored manually. + mismatched_private_accounts: Vec<(AccountId, Digest)>, +} + +impl AccountUpdates { + /// Creates a new instance of `AccountUpdates`. + pub fn new( + updated_public_accounts: Vec, + mismatched_private_accounts: Vec<(AccountId, Digest)>, + ) -> Self { + Self { + updated_public_accounts, + mismatched_private_accounts, + } + } + + /// Returns the updated public accounts. + pub fn updated_public_accounts(&self) -> &[Account] { + &self.updated_public_accounts + } + + /// Returns the mismatched private accounts. + pub fn mismatched_private_accounts(&self) -> &[(AccountId, Digest)] { + &self.mismatched_private_accounts + } + + pub fn extend(&mut self, other: AccountUpdates) { + self.updated_public_accounts.extend(other.updated_public_accounts); + self.mismatched_private_accounts.extend(other.mismatched_private_accounts); + } +} diff --git a/crates/rust-client/src/transaction/mod.rs b/crates/rust-client/src/transaction/mod.rs index f71c18865..a50952ad3 100644 --- a/crates/rust-client/src/transaction/mod.rs +++ b/crates/rust-client/src/transaction/mod.rs @@ -369,45 +369,6 @@ impl TransactionStoreUpdate { } } -/// Contains transaction changes to apply to the store. -#[derive(Default)] -pub struct TransactionUpdates { - /// Transaction updates for any transaction that was committed between the sync request's block - /// number and the response's block number. - committed_transactions: Vec, - /// Transaction IDs for any transactions that were discarded in the sync. - discarded_transactions: Vec, -} - -impl TransactionUpdates { - /// Creates a new [`TransactionUpdate`] - pub fn new( - committed_transactions: Vec, - discarded_transactions: Vec, - ) -> Self { - Self { - committed_transactions, - discarded_transactions, - } - } - - /// Extends the transaction update information with `other`. - pub fn extend(&mut self, other: Self) { - self.committed_transactions.extend(other.committed_transactions); - self.discarded_transactions.extend(other.discarded_transactions); - } - - /// Returns a reference to committed transactions. - pub fn committed_transactions(&self) -> &[TransactionUpdate] { - &self.committed_transactions - } - - /// Returns a reference to discarded transactions. - pub fn discarded_transactions(&self) -> &[TransactionId] { - &self.discarded_transactions - } -} - /// Transaction management methods impl Client { // TRANSACTION DATA RETRIEVAL From 3dc16d5c17a362666e81f5d9518d19035f647f04 Mon Sep 17 00:00:00 2001 From: tomyrd Date: Wed, 5 Mar 2025 16:29:59 -0300 Subject: [PATCH 34/34] feat: add check nullifiers request --- crates/rust-client/src/store/mod.rs | 12 -- .../rust-client/src/store/sqlite_store/mod.rs | 13 -- .../src/store/sqlite_store/sync.rs | 19 +-- crates/rust-client/src/store/web_store/mod.rs | 10 -- .../src/store/web_store/sync/mod.rs | 10 -- crates/rust-client/src/sync/state_sync.rs | 153 +++++++++--------- 6 files changed, 81 insertions(+), 136 deletions(-) diff --git a/crates/rust-client/src/store/mod.rs b/crates/rust-client/src/store/mod.rs index 14198e9ea..c7d46ee9a 100644 --- a/crates/rust-client/src/store/mod.rs +++ b/crates/rust-client/src/store/mod.rs @@ -33,12 +33,10 @@ use miden_objects::{ block::{BlockHeader, BlockNumber}, crypto::merkle::{InOrderIndex, MmrPeaks}, note::{NoteId, NoteTag, Nullifier}, - transaction::TransactionId, Digest, Word, }; use crate::{ - note::NoteUpdates, sync::{NoteTagRecord, StateSyncUpdate}, transaction::{TransactionRecord, TransactionStoreUpdate}, }; @@ -313,16 +311,6 @@ pub trait Store: Send + Sync { /// - Storing new MMR authentication nodes. /// - Updating the tracked public accounts. async fn apply_state_sync(&self, state_sync_update: StateSyncUpdate) -> Result<(), StoreError>; - - /// Applies nullifier updates to database. - /// Nullifiers are retrieved after completing a `StateSync`. - /// - /// This operation is temporary, to be removed as part of miden-client/650. - async fn apply_nullifiers( - &self, - note_updates: NoteUpdates, - transactions_to_discard: Vec, - ) -> Result<(), StoreError>; } // CHAIN MMR NODE FILTER diff --git a/crates/rust-client/src/store/sqlite_store/mod.rs b/crates/rust-client/src/store/sqlite_store/mod.rs index 1f94b3835..da555e80a 100644 --- a/crates/rust-client/src/store/sqlite_store/mod.rs +++ b/crates/rust-client/src/store/sqlite_store/mod.rs @@ -17,7 +17,6 @@ use miden_objects::{ block::{BlockHeader, BlockNumber}, crypto::merkle::{InOrderIndex, MmrPeaks}, note::{NoteTag, Nullifier}, - transaction::TransactionId, Digest, Word, }; use rusqlite::{types::Value, vtab::array, Connection}; @@ -28,7 +27,6 @@ use super::{ OutputNoteRecord, Store, TransactionFilter, }; use crate::{ - note::NoteUpdates, store::StoreError, sync::{NoteTagRecord, StateSyncUpdate}, transaction::{TransactionRecord, TransactionStoreUpdate}, @@ -329,17 +327,6 @@ impl Store for SqliteStore { self.interact_with_connection(SqliteStore::get_unspent_input_note_nullifiers) .await } - - async fn apply_nullifiers( - &self, - note_updates: NoteUpdates, - transactions_to_discard: Vec, - ) -> Result<(), StoreError> { - self.interact_with_connection(move |conn| { - SqliteStore::apply_nullifiers(conn, ¬e_updates, &transactions_to_discard) - }) - .await - } } // UTILS diff --git a/crates/rust-client/src/store/sqlite_store/sync.rs b/crates/rust-client/src/store/sqlite_store/sync.rs index fa35f60c5..cc1f5c674 100644 --- a/crates/rust-client/src/store/sqlite_store/sync.rs +++ b/crates/rust-client/src/store/sqlite_store/sync.rs @@ -2,13 +2,12 @@ use alloc::{collections::BTreeSet, vec::Vec}; -use miden_objects::{block::BlockNumber, note::NoteTag, transaction::TransactionId}; +use miden_objects::{block::BlockNumber, note::NoteTag}; use miden_tx::utils::{Deserializable, Serializable}; use rusqlite::{params, Connection, Transaction}; use super::SqliteStore; use crate::{ - note::NoteUpdates, store::{ sqlite_store::{ account::{lock_account, update_account}, @@ -172,22 +171,6 @@ impl SqliteStore { Ok(()) } - - pub(super) fn apply_nullifiers( - conn: &mut Connection, - note_updates: &NoteUpdates, - transactions_to_discard: &[TransactionId], - ) -> Result<(), StoreError> { - let tx = conn.transaction()?; - - apply_note_updates_tx(&tx, note_updates)?; - - Self::mark_transactions_as_discarded(&tx, transactions_to_discard)?; - - tx.commit()?; - - Ok(()) - } } pub(super) fn add_note_tag_tx(tx: &Transaction<'_>, tag: &NoteTagRecord) -> Result<(), StoreError> { diff --git a/crates/rust-client/src/store/web_store/mod.rs b/crates/rust-client/src/store/web_store/mod.rs index 451c19cfb..0fb9443db 100644 --- a/crates/rust-client/src/store/web_store/mod.rs +++ b/crates/rust-client/src/store/web_store/mod.rs @@ -14,7 +14,6 @@ use miden_objects::{ block::{BlockHeader, BlockNumber}, crypto::merkle::{InOrderIndex, MmrPeaks}, note::Nullifier, - transaction::TransactionId, Digest, Word, }; use tonic::async_trait; @@ -26,7 +25,6 @@ use super::{ OutputNoteRecord, Store, StoreError, TransactionFilter, }; use crate::{ - note::NoteUpdates, sync::{NoteTagRecord, StateSyncUpdate}, transaction::{TransactionRecord, TransactionStoreUpdate}, }; @@ -87,14 +85,6 @@ impl Store for WebStore { self.apply_state_sync(state_sync_update).await } - async fn apply_nullifiers( - &self, - note_updates: NoteUpdates, - transactions_to_discard: Vec, - ) -> Result<(), StoreError> { - self.apply_nullifiers(note_updates, transactions_to_discard).await - } - // TRANSACTIONS // -------------------------------------------------------------------------------------------- diff --git a/crates/rust-client/src/store/web_store/sync/mod.rs b/crates/rust-client/src/store/web_store/sync/mod.rs index c01553d06..12c7764c5 100644 --- a/crates/rust-client/src/store/web_store/sync/mod.rs +++ b/crates/rust-client/src/store/web_store/sync/mod.rs @@ -7,7 +7,6 @@ use miden_objects::{ account::AccountId, block::BlockNumber, note::{NoteId, NoteTag}, - transaction::TransactionId, }; use miden_tx::utils::{Deserializable, Serializable}; use serde_wasm_bindgen::from_value; @@ -20,7 +19,6 @@ use super::{ WebStore, }; use crate::{ - note::NoteUpdates, store::StoreError, sync::{NoteTagRecord, NoteTagSource, StateSyncUpdate}, }; @@ -195,12 +193,4 @@ impl WebStore { Ok(()) } - - pub(super) async fn apply_nullifiers( - &self, - note_updates: NoteUpdates, - _transactions_to_discard: Vec, - ) -> Result<(), StoreError> { - apply_note_updates_tx(¬e_updates).await - } } diff --git a/crates/rust-client/src/sync/state_sync.rs b/crates/rust-client/src/sync/state_sync.rs index 56e207877..d21d19c95 100644 --- a/crates/rust-client/src/sync/state_sync.rs +++ b/crates/rust-client/src/sync/state_sync.rs @@ -3,7 +3,7 @@ use core::{future::Future, pin::Pin}; use miden_objects::{ account::{Account, AccountHeader, AccountId}, - block::BlockHeader, + block::{BlockHeader, BlockNumber}, crypto::merkle::{InOrderIndex, MmrDelta, MmrPeaks, PartialMmr}, note::{NoteId, NoteTag}, Digest, @@ -13,10 +13,7 @@ use tracing::info; use super::{AccountUpdates, BlockUpdates, StateSyncUpdate, TransactionUpdates}; use crate::{ note::{NoteScreener, NoteUpdates}, - rpc::{ - domain::{note::CommittedNote, nullifier::NullifierUpdate, transaction::TransactionUpdate}, - NodeRpcClient, - }, + rpc::{domain::note::CommittedNote, NodeRpcClient}, store::{InputNoteRecord, NoteFilter, OutputNoteRecord, Store, StoreError}, ClientError, }; @@ -104,51 +101,32 @@ impl StateSync { unspent_input_notes: Vec, unspent_output_notes: Vec, ) -> Result { - let unspent_nullifiers = unspent_input_notes - .iter() - .map(InputNoteRecord::nullifier) - .chain(unspent_output_notes.iter().filter_map(OutputNoteRecord::nullifier)); - - // To receive information about added nullifiers, we reduce them to the higher 16 bits - // Note that besides filtering by nullifier prefixes, the node also filters by block number - // (it only returns nullifiers from current_block_num until - // response.block_header.block_num()) - let mut nullifiers_tags: Vec = - unspent_nullifiers.map(|nullifier| nullifier.prefix()).collect(); + let block_num = (u32::try_from(current_partial_mmr.num_leaves() - 1) + .expect("The number of leaves in the MMR should be greater than 0 and less than 2^32")) + .into(); let mut state_sync_update = StateSyncUpdate { - block_num: (u32::try_from(current_partial_mmr.num_leaves() - 1).expect( - "The number of leaves in the MMR should be greater than 0 and less than 2^32", - )) - .into(), + block_num, note_updates: NoteUpdates::new(unspent_input_notes, unspent_output_notes), ..Default::default() }; - while self - .sync_state_step( - &mut state_sync_update, - &mut current_partial_mmr, - &accounts, - ¬e_tags, - &nullifiers_tags, - ) - .await? - { - // New nullfiers should be added for new untracked notes that were added in previous - // steps - nullifiers_tags.append( - &mut state_sync_update - .note_updates - .updated_input_notes() - .filter(|note| { - note.is_committed() && !nullifiers_tags.contains(¬e.nullifier().prefix()) - }) - .map(|note| note.nullifier().prefix()) - .collect::>(), - ); + loop { + if !self + .sync_state_step( + &mut state_sync_update, + &mut current_partial_mmr, + &accounts, + ¬e_tags, + ) + .await? + { + break; + } } + self.sync_nullifiers(&mut state_sync_update, block_num).await?; + Ok(state_sync_update) } @@ -167,7 +145,6 @@ impl StateSync { current_partial_mmr: &mut PartialMmr, accounts: &[AccountHeader], note_tags: &[NoteTag], - _nullifiers_tags: &[u16], ) -> Result { let account_ids: Vec = accounts.iter().map(AccountHeader::id).collect(); @@ -189,18 +166,20 @@ impl StateSync { state_sync_update.account_updates.extend(account_updates); - let (found_relevant_note, transaction_updates) = self + // Track the transaction updates for transactions that were committed. Some of these might + // be tracked by the client and need to be marked as committed. + state_sync_update + .transaction_updates + .extend(TransactionUpdates::new(response.transactions, vec![])); + + let found_relevant_note = self .note_state_sync( &mut state_sync_update.note_updates, response.note_inclusions, - response.transactions, - vec![], &response.block_header, ) .await?; - state_sync_update.transaction_updates.extend(transaction_updates); - let (new_mmr_peaks, new_authentication_nodes) = apply_mmr_changes( &response.block_header, found_relevant_note, @@ -282,7 +261,7 @@ impl StateSync { } /// Applies the changes received from the sync response to the notes and transactions tracked - /// by the client and updates the `state_sync_update` accordingly. + /// by the client and updates the `note_updates` accordingly. /// /// This method uses the callbacks provided to the [`StateSync`] component to check if the /// updates received are relevant to the client. @@ -292,27 +271,18 @@ impl StateSync { /// * Tracked expected notes that were committed in the block. /// * Tracked notes that were being processed by a transaction that got committed. /// * Tracked notes that were nullified by an external transaction. - /// - /// The transaction updates might include: - /// * Transactions that were committed in the block. Some of these might me tracked by the - /// client and need to be marked as committed. - /// * Local tracked transactions that were discarded because the notes that they were processing - /// were nullified by an another transaction. async fn note_state_sync( &mut self, note_updates: &mut NoteUpdates, note_inclusions: Vec, - transactions: Vec, - nullifiers: Vec, block_header: &BlockHeader, - ) -> Result<(bool, TransactionUpdates), ClientError> { + ) -> Result { let public_note_ids: Vec = note_inclusions .iter() .filter_map(|note| (!note.metadata().is_private()).then_some(*note.note_id())) .collect(); let mut found_relevant_note = false; - let mut discarded_transactions = vec![]; // Process note inclusions let new_public_notes = @@ -334,20 +304,7 @@ impl StateSync { } } - // Process nullifiers - for nullifier_update in nullifiers { - let discarded_transaction = note_updates - .apply_nullifiers_state_transitions(&nullifier_update, &transactions)?; - - if let Some(transaction_id) = discarded_transaction { - discarded_transactions.push(transaction_id); - } - } - - Ok(( - found_relevant_note, - TransactionUpdates::new(transactions, discarded_transactions), - )) + Ok(found_relevant_note) } /// Queries the node for all received notes that aren't being locally tracked in the client. @@ -372,6 +329,56 @@ impl StateSync { Ok(return_notes) } + + /// Collects the nullifier tags for the notes that were updated in the sync response and uses + /// the `check_nullifiers_by_prefix` endpoint to check if there are new nullifiers for these + /// notes. It then processes the nullifiers to apply the state transitions on the note updates. + /// + /// The `state_sync_update` field will be updated to track the new discarded transactions. + async fn sync_nullifiers( + &self, + state_sync_update: &mut StateSyncUpdate, + current_block_num: BlockNumber, + ) -> Result<(), ClientError> { + // To receive information about added nullifiers, we reduce them to the higher 16 bits + // Note that besides filtering by nullifier prefixes, the node also filters by block number + // (it only returns nullifiers from current_block_num until + // response.block_header.block_num()) + + // Check for new nullifiers for input notes that were updated + let nullifiers_tags: Vec = state_sync_update + .note_updates + .updated_input_notes() + .map(|note| note.nullifier().prefix()) + .collect(); + + let new_nullifiers = self + .rpc_api + .check_nullifiers_by_prefix(&nullifiers_tags, current_block_num) + .await?; + + // Process nullifiers and track the updates of local tracked transactions that were + // discarded because the notes that they were processing were nullified by an + // another transaction. + let mut discarded_transactions = vec![]; + + for nullifier_update in new_nullifiers { + let discarded_transaction = + state_sync_update.note_updates.apply_nullifiers_state_transitions( + &nullifier_update, + state_sync_update.transaction_updates.committed_transactions(), + )?; + + if let Some(transaction_id) = discarded_transaction { + discarded_transactions.push(transaction_id); + } + } + + let transaction_updates = TransactionUpdates::new(vec![], discarded_transactions); + state_sync_update.transaction_updates.extend(transaction_updates); + + Ok(()) + } } // HELPERS