diff --git a/Cargo.lock b/Cargo.lock index 0b91e7717..e52728e62 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -409,6 +409,15 @@ dependencies = [ "bytemuck", ] +[[package]] +name = "atomic-polyfill" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cf2bce30dfe09ef0bfaef228b9d414faaf7e563035494d7fe092dba54b300f4" +dependencies = [ + "critical-section", +] + [[package]] name = "autocfg" version = "1.2.0" @@ -598,6 +607,12 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d6d68c57235a3a081186990eca2867354726650f42f7516ca50c28d6281fd15" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.6.0" @@ -703,6 +718,12 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" +[[package]] +name = "cobs" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15" + [[package]] name = "colorchoice" version = "1.0.0" @@ -768,6 +789,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "critical-section" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7059fff8937831a9ae6f0fe4d658ffabf58f2ca96aa9dec1c889f936f705f216" + [[package]] name = "crossbeam-deque" version = "0.8.5" @@ -925,6 +952,12 @@ version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + [[package]] name = "encoding_rs" version = "0.8.33" @@ -934,6 +967,29 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "env_filter" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a009aa4810eb158359dda09d0c87378e4bbb89b5a801f016885a4707ba24f7ea" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b35839ba51819680ba087cd351788c9a3c476841207e0b8cee0b04722343b9" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "humantime", + "log", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -1113,6 +1169,15 @@ dependencies = [ "tracing", ] +[[package]] +name = "hash32" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" +dependencies = [ + "byteorder", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -1138,6 +1203,20 @@ dependencies = [ "hashbrown 0.14.3", ] +[[package]] +name = "heapless" +version = "0.7.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f" +dependencies = [ + "atomic-polyfill", + "hash32", + "rustc_version", + "serde", + "spin", + "stable_deref_trait", +] + [[package]] name = "heck" version = "0.4.1" @@ -1211,6 +1290,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + [[package]] name = "hyper" version = "0.14.28" @@ -1394,7 +1479,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19" dependencies = [ "cfg-if", - "windows-targets 0.52.4", + "windows-targets 0.48.5", ] [[package]] @@ -1638,6 +1723,7 @@ dependencies = [ "num-complex", "rand", "rand_core", + "serde", "sha3", "winter-crypto", "winter-math", @@ -1658,13 +1744,30 @@ dependencies = [ [[package]] name = "miden-lib" version = "0.2.0" -source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#d6af2ae8ce3da19e9dcd66ce3ad81a724a8d8a7a" +source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#61b0bf91576f932d92ffb1dd30a4450e37b3db0b" dependencies = [ "miden-assembly 0.9.1", "miden-objects 0.2.0", "miden-stdlib 0.9.1", ] +[[package]] +name = "miden-mock" +version = "0.2.0" +source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#61b0bf91576f932d92ffb1dd30a4450e37b3db0b" +dependencies = [ + "env_logger", + "hex", + "miden-lib 0.2.0", + "miden-objects 0.2.0", + "miden-processor 0.9.1", + "miden-prover 0.9.1", + "postcard", + "rand", + "rand_pcg", + "winter-rand-utils", +] + [[package]] name = "miden-node" version = "0.2.0" @@ -1805,6 +1908,7 @@ dependencies = [ "figment", "hex", "miden-lib 0.2.0", + "miden-mock", "miden-node-proto 0.2.0", "miden-node-utils 0.2.0", "miden-objects 0.2.0", @@ -1819,6 +1923,7 @@ dependencies = [ "tonic", "tracing", "tracing-subscriber", + "winter-rand-utils", ] [[package]] @@ -1876,13 +1981,15 @@ dependencies = [ [[package]] name = "miden-objects" version = "0.2.0" -source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#d6af2ae8ce3da19e9dcd66ce3ad81a724a8d8a7a" +source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#61b0bf91576f932d92ffb1dd30a4450e37b3db0b" dependencies = [ + "log", "miden-assembly 0.9.1", "miden-core 0.9.1", "miden-crypto 0.9.1", "miden-processor 0.9.1", "miden-verifier 0.9.1", + "serde", "winter-rand-utils", ] @@ -1968,7 +2075,7 @@ dependencies = [ [[package]] name = "miden-tx" version = "0.2.0" -source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#d6af2ae8ce3da19e9dcd66ce3ad81a724a8d8a7a" +source = "git+https://github.com/0xPolygonMiden/miden-base.git?branch=next#61b0bf91576f932d92ffb1dd30a4450e37b3db0b" dependencies = [ "miden-lib 0.2.0", "miden-objects 0.2.0", @@ -2353,6 +2460,18 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" +[[package]] +name = "postcard" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a55c51ee6c0db07e68448e336cf8ea4131a620edefebf9893e759b2d793420f8" +dependencies = [ + "cobs", + "embedded-io", + "heapless", + "serde", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -2565,6 +2684,16 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rand_pcg" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59cad018caf63deb318e5a4586d99a24424a364f40f1e5778c29aca23f4fc73e" +dependencies = [ + "rand_core", + "serde", +] + [[package]] name = "rand_xorshift" version = "0.3.0" @@ -2880,6 +3009,21 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "strsim" version = "0.11.0" diff --git a/Cargo.toml b/Cargo.toml index 02576560d..34ddf9f61 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ exclude = [".github/"] [workspace.dependencies] miden-air = { version = "0.9", default-features = false } miden-lib = { git = "https://github.com/0xPolygonMiden/miden-base.git", branch = "next" } +miden-mock = { git = "https://github.com/0xPolygonMiden/miden-base.git", branch = "next" } miden-objects = { git = "https://github.com/0xPolygonMiden/miden-base.git", branch = "next" } miden-processor = { version = "0.9" } miden-stdlib = { version = "0.9", default-features = false } @@ -36,3 +37,4 @@ tracing-subscriber = { version = "0.3", features = [ "json", "env-filter", ] } +winter-rand-utils = "0.8.3" diff --git a/block-producer/src/batch_builder/batch.rs b/block-producer/src/batch_builder/batch.rs index b5784c433..1514d5d09 100644 --- a/block-producer/src/batch_builder/batch.rs +++ b/block-producer/src/batch_builder/batch.rs @@ -1,11 +1,12 @@ use std::collections::BTreeMap; -use miden_node_proto::domain::accounts::UpdatedAccount; +use miden_node_proto::domain::accounts::AccountDetailsUpdate; use miden_objects::{ accounts::AccountId, batches::BatchNoteTree, crypto::hash::blake::{Blake3Digest, Blake3_256}, notes::{NoteEnvelope, Nullifier}, + transaction::AccountDetails, Digest, MAX_NOTES_PER_BATCH, }; use tracing::instrument; @@ -54,6 +55,7 @@ impl TransactionBatch { AccountStates { initial_state: tx.initial_account_hash(), final_state: tx.final_account_hash(), + details: tx.account_details().cloned(), }, ) }) @@ -107,15 +109,15 @@ impl TransactionBatch { .map(|(account_id, account_states)| (*account_id, account_states.initial_state)) } - /// Returns an iterator over (account_id, new_state_hash) tuples for accounts that were + /// Returns an iterator over (account_id, details, new_state_hash) tuples for accounts that were /// modified in this transaction batch. - pub fn updated_accounts(&self) -> impl Iterator + '_ { + pub fn updated_accounts(&self) -> impl Iterator + '_ { self.updated_accounts .iter() - .map(|(&account_id, account_states)| UpdatedAccount { + .map(|(&account_id, account_states)| AccountDetailsUpdate { account_id, final_state_hash: account_states.final_state, - details: None, // TODO: In the next PR: account_states.details.clone(), + details: account_states.details.clone(), }) } @@ -150,8 +152,9 @@ impl TransactionBatch { /// account. /// /// TODO: should this be moved into domain objects? -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, Clone, PartialEq, Eq)] struct AccountStates { initial_state: Digest, final_state: Digest, + details: Option, } diff --git a/block-producer/src/block.rs b/block-producer/src/block.rs index 6e43674c5..41467d292 100644 --- a/block-producer/src/block.rs +++ b/block-producer/src/block.rs @@ -1,7 +1,7 @@ use std::collections::BTreeMap; use miden_node_proto::{ - domain::accounts::UpdatedAccount, + domain::accounts::AccountDetailsUpdate, errors::{ConversionError, MissingFieldHelper}, generated::responses::GetBlockInputsResponse, AccountInputRecord, NullifierWitness, @@ -18,11 +18,10 @@ use crate::store::BlockInputsError; #[derive(Debug, Clone)] pub struct Block { pub header: BlockHeader, - pub updated_accounts: Vec, + pub updated_accounts: Vec, pub created_notes: Vec<(usize, usize, NoteEnvelope)>, pub produced_nullifiers: Vec, // TODO: - // - full states for updated public accounts // - full states for created public notes // - zk proof } diff --git a/block-producer/src/block_builder/mod.rs b/block-producer/src/block_builder/mod.rs index cb0502528..eb92612b4 100644 --- a/block-producer/src/block_builder/mod.rs +++ b/block-producer/src/block_builder/mod.rs @@ -119,7 +119,7 @@ where info!(target: COMPONENT, block_num, %block_hash, "block built"); debug!(target: COMPONENT, ?block); - self.state_view.apply_block(block).await?; + self.state_view.apply_block(&block).await?; info!(target: COMPONENT, block_num, %block_hash, "block committed"); diff --git a/block-producer/src/block_builder/prover/block_witness.rs b/block-producer/src/block_builder/prover/block_witness.rs index c72c84ddc..8c57e4a64 100644 --- a/block-producer/src/block_builder/prover/block_witness.rs +++ b/block-producer/src/block_builder/prover/block_witness.rs @@ -1,6 +1,6 @@ use std::collections::{BTreeMap, BTreeSet}; -use miden_node_proto::domain::accounts::UpdatedAccount; +use miden_node_proto::domain::accounts::AccountDetailsUpdate; use miden_objects::{ accounts::AccountId, crypto::merkle::{EmptySubtreeRoots, MerklePath, MerkleStore, MmrPeaks, SmtProof}, @@ -38,7 +38,7 @@ impl BlockWitness { let updated_accounts = { let mut account_initial_states: BTreeMap = - batches.iter().flat_map(|batch| batch.account_initial_states()).collect(); + batches.iter().flat_map(TransactionBatch::account_initial_states).collect(); let mut account_merkle_proofs: BTreeMap = block_inputs .accounts @@ -48,9 +48,9 @@ impl BlockWitness { batches .iter() - .flat_map(|batch| batch.updated_accounts()) + .flat_map(TransactionBatch::updated_accounts) .map( - |UpdatedAccount { + |AccountDetailsUpdate { account_id, final_state_hash, .. diff --git a/block-producer/src/block_builder/prover/tests.rs b/block-producer/src/block_builder/prover/tests.rs index 3869064d1..762b56e6e 100644 --- a/block-producer/src/block_builder/prover/tests.rs +++ b/block-producer/src/block_builder/prover/tests.rs @@ -1,6 +1,6 @@ use std::{collections::BTreeMap, iter}; -use miden_node_proto::domain::accounts::UpdatedAccount; +use miden_node_proto::domain::accounts::AccountDetailsUpdate; use miden_objects::{ accounts::AccountId, crypto::merkle::{ @@ -236,7 +236,7 @@ async fn test_compute_account_root_success() { account_ids .iter() .zip(account_final_states.iter()) - .map(|(&account_id, &account_hash)| UpdatedAccount { + .map(|(&account_id, &account_hash)| AccountDetailsUpdate { account_id, final_state_hash: account_hash.into(), details: None, diff --git a/block-producer/src/state_view/mod.rs b/block-producer/src/state_view/mod.rs index 8af9ebff6..89fde23a3 100644 --- a/block-producer/src/state_view/mod.rs +++ b/block-producer/src/state_view/mod.rs @@ -119,9 +119,9 @@ where #[instrument(target = "miden-block-producer", skip_all, err)] async fn apply_block( &self, - block: Block, + block: &Block, ) -> Result<(), ApplyBlockError> { - self.store.apply_block(block.clone()).await?; + self.store.apply_block(block).await?; let mut locked_accounts_in_flight = self.accounts_in_flight.write().await; let mut locked_nullifiers_in_flight = self.nullifiers_in_flight.write().await; diff --git a/block-producer/src/state_view/tests/apply_block.rs b/block-producer/src/state_view/tests/apply_block.rs index 0db199277..8261215dc 100644 --- a/block-producer/src/state_view/tests/apply_block.rs +++ b/block-producer/src/state_view/tests/apply_block.rs @@ -6,7 +6,7 @@ use std::iter; -use miden_node_proto::domain::accounts::UpdatedAccount; +use miden_node_proto::domain::accounts::AccountDetailsUpdate; use super::*; use crate::test_utils::{block::MockBlockBuilder, MockStoreSuccessBuilder}; @@ -34,7 +34,7 @@ async fn test_apply_block_ab1() { .await .account_updates( std::iter::once(account) - .map(|mock_account| UpdatedAccount { + .map(|mock_account| AccountDetailsUpdate { account_id: mock_account.id, final_state_hash: mock_account.states[1], details: None, @@ -43,7 +43,7 @@ async fn test_apply_block_ab1() { ) .build(); - let apply_block_res = state_view.apply_block(block).await; + let apply_block_res = state_view.apply_block(&block).await; assert!(apply_block_res.is_ok()); assert_eq!(*store.num_apply_block_called.read().await, 1); @@ -81,7 +81,7 @@ async fn test_apply_block_ab2() { .account_updates( accounts_in_block .into_iter() - .map(|mock_account| UpdatedAccount { + .map(|mock_account| AccountDetailsUpdate { account_id: mock_account.id, final_state_hash: mock_account.states[1], details: None, @@ -90,7 +90,7 @@ async fn test_apply_block_ab2() { ) .build(); - let apply_block_res = state_view.apply_block(block).await; + let apply_block_res = state_view.apply_block(&block).await; assert!(apply_block_res.is_ok()); let accounts_still_in_flight = state_view.accounts_in_flight.read().await; @@ -130,7 +130,7 @@ async fn test_apply_block_ab3() { accounts .clone() .into_iter() - .map(|mock_account| UpdatedAccount { + .map(|mock_account| AccountDetailsUpdate { account_id: mock_account.id, final_state_hash: mock_account.states[1], details: None, @@ -139,7 +139,7 @@ async fn test_apply_block_ab3() { ) .build(); - let apply_block_res = state_view.apply_block(block).await; + let apply_block_res = state_view.apply_block(&block).await; assert!(apply_block_res.is_ok()); // Craft a new transaction which tries to consume the same note that was consumed in the diff --git a/block-producer/src/store/mod.rs b/block-producer/src/store/mod.rs index b02f9fa41..2407f6acd 100644 --- a/block-producer/src/store/mod.rs +++ b/block-producer/src/store/mod.rs @@ -49,7 +49,7 @@ pub trait Store: ApplyBlock { pub trait ApplyBlock: Send + Sync + 'static { async fn apply_block( &self, - block: Block, + block: &Block, ) -> Result<(), ApplyBlockError>; } @@ -131,13 +131,13 @@ impl ApplyBlock for DefaultStore { #[instrument(target = "miden-block-producer", skip_all, err)] async fn apply_block( &self, - block: Block, + block: &Block, ) -> Result<(), ApplyBlockError> { let request = tonic::Request::new(ApplyBlockRequest { - block: Some(block.header.into()), + block: Some((&block.header).into()), accounts: convert(&block.updated_accounts), - nullifiers: convert(block.produced_nullifiers), - notes: convert(block.created_notes), + nullifiers: convert(&block.produced_nullifiers), + notes: convert(&block.created_notes), }); let _ = self diff --git a/block-producer/src/test_utils/block.rs b/block-producer/src/test_utils/block.rs index 02c8a28cd..75c69e114 100644 --- a/block-producer/src/test_utils/block.rs +++ b/block-producer/src/test_utils/block.rs @@ -1,4 +1,4 @@ -use miden_node_proto::domain::accounts::UpdatedAccount; +use miden_node_proto::domain::accounts::AccountDetailsUpdate; use miden_objects::{ block::BlockNoteTree, crypto::merkle::{Mmr, SimpleSmt}, @@ -68,7 +68,7 @@ pub async fn build_actual_block_header( let updated_accounts: Vec<_> = batches.iter().flat_map(TransactionBatch::updated_accounts).collect(); let produced_nullifiers: Vec = - batches.iter().flat_map(|batch| batch.produced_nullifiers()).collect(); + batches.iter().flat_map(TransactionBatch::produced_nullifiers).collect(); let block_inputs_from_store: BlockInputs = store .get_block_inputs( @@ -89,7 +89,7 @@ pub struct MockBlockBuilder { store_chain_mmr: Mmr, last_block_header: BlockHeader, - updated_accounts: Option>, + updated_accounts: Option>, created_notes: Option>, produced_nullifiers: Option>, } @@ -109,7 +109,7 @@ impl MockBlockBuilder { pub fn account_updates( mut self, - updated_accounts: Vec, + updated_accounts: Vec, ) -> Self { for update in &updated_accounts { self.store_accounts diff --git a/block-producer/src/test_utils/store.rs b/block-producer/src/test_utils/store.rs index abd6cba22..41c1f3192 100644 --- a/block-producer/src/test_utils/store.rs +++ b/block-producer/src/test_utils/store.rs @@ -174,7 +174,7 @@ impl MockStoreSuccess { impl ApplyBlock for MockStoreSuccess { async fn apply_block( &self, - block: Block, + block: &Block, ) -> Result<(), ApplyBlockError> { // Intentionally, we take and hold both locks, to prevent calls to `get_tx_inputs()` from going through while we're updating the store's data structure let mut locked_accounts = self.accounts.write().await; @@ -187,7 +187,7 @@ impl ApplyBlock for MockStoreSuccess { debug_assert_eq!(locked_accounts.root(), block.header.account_root()); // update nullifiers - for nullifier in block.produced_nullifiers { + for nullifier in &block.produced_nullifiers { locked_produced_nullifiers .insert(nullifier.inner(), [block.header.block_num().into(), ZERO, ZERO, ZERO]); } @@ -291,7 +291,7 @@ pub struct MockStoreFailure; impl ApplyBlock for MockStoreFailure { async fn apply_block( &self, - _block: Block, + _block: &Block, ) -> Result<(), ApplyBlockError> { Err(ApplyBlockError::GrpcClientError(String::new())) } diff --git a/proto/proto/requests.proto b/proto/proto/requests.proto index f65d6076d..78149ea09 100644 --- a/proto/proto/requests.proto +++ b/proto/proto/requests.proto @@ -10,6 +10,8 @@ import "note.proto"; message AccountUpdate { account.AccountId account_id = 1; digest.Digest account_hash = 2; + // Details for public (on-chain) account. + optional bytes details = 3; } message ApplyBlockRequest { diff --git a/proto/src/domain/accounts.rs b/proto/src/domain/accounts.rs index 9669fea75..e3809e9ee 100644 --- a/proto/src/domain/accounts.rs +++ b/proto/src/domain/accounts.rs @@ -2,7 +2,8 @@ use std::fmt::{Debug, Display, Formatter}; use miden_node_utils::formatting::format_opt; use miden_objects::{ - accounts::AccountId, crypto::merkle::MerklePath, transaction::AccountDetails, Digest, + accounts::AccountId, crypto::merkle::MerklePath, transaction::AccountDetails, + utils::Serializable, Digest, }; use crate::{ @@ -44,6 +45,12 @@ impl From for AccountIdPb { } } +impl From<&AccountId> for AccountIdPb { + fn from(account_id: &AccountId) -> Self { + (*account_id).into() + } +} + impl From for AccountIdPb { fn from(account_id: AccountId) -> Self { Self { @@ -73,18 +80,18 @@ impl TryFrom for AccountId { // ================================================================================================ #[derive(Debug, Clone, PartialEq, Eq)] -pub struct UpdatedAccount { +pub struct AccountDetailsUpdate { pub account_id: AccountId, pub final_state_hash: Digest, pub details: Option, } -impl From<&UpdatedAccount> for AccountUpdate { - fn from(update: &UpdatedAccount) -> Self { +impl From<&AccountDetailsUpdate> for AccountUpdate { + fn from(update: &AccountDetailsUpdate) -> Self { Self { account_id: Some(update.account_id.into()), account_hash: Some(update.final_state_hash.into()), - // details: update.details.to_bytes(), + details: update.details.as_ref().map(|details| details.to_bytes()), } } } diff --git a/proto/src/domain/blocks.rs b/proto/src/domain/blocks.rs index c33ace75f..64ed90af6 100644 --- a/proto/src/domain/blocks.rs +++ b/proto/src/domain/blocks.rs @@ -8,8 +8,8 @@ use crate::{ // BLOCK HEADER // ================================================================================================ -impl From for block_header::BlockHeader { - fn from(header: BlockHeader) -> Self { +impl From<&BlockHeader> for block_header::BlockHeader { + fn from(header: &BlockHeader) -> Self { Self { prev_hash: Some(header.prev_hash().into()), block_num: header.block_num(), @@ -27,6 +27,12 @@ impl From for block_header::BlockHeader { } } +impl From for block_header::BlockHeader { + fn from(header: BlockHeader) -> Self { + (&header).into() + } +} + impl TryFrom<&block_header::BlockHeader> for BlockHeader { type Error = ConversionError; diff --git a/proto/src/domain/notes.rs b/proto/src/domain/notes.rs index d27d6e95a..c3104b279 100644 --- a/proto/src/domain/notes.rs +++ b/proto/src/domain/notes.rs @@ -20,11 +20,11 @@ impl From for note::NoteSyncRecord { // NoteCreated // ================================================================================================ -impl From<(usize, usize, NoteEnvelope)> for note::NoteCreated { - fn from((batch_idx, note_idx, note): (usize, usize, NoteEnvelope)) -> Self { +impl From<&(usize, usize, NoteEnvelope)> for note::NoteCreated { + fn from((batch_idx, note_idx, note): &(usize, usize, NoteEnvelope)) -> Self { Self { - batch_index: batch_idx as u32, - note_index: note_idx as u32, + batch_index: *batch_idx as u32, + note_index: *note_idx as u32, note_id: Some(note.note_id().into()), sender: Some(note.metadata().sender().into()), tag: note.metadata().tag().into(), diff --git a/proto/src/generated/requests.rs b/proto/src/generated/requests.rs index 980a431df..edf5bf0a4 100644 --- a/proto/src/generated/requests.rs +++ b/proto/src/generated/requests.rs @@ -6,6 +6,9 @@ pub struct AccountUpdate { pub account_id: ::core::option::Option, #[prost(message, optional, tag = "2")] pub account_hash: ::core::option::Option, + /// Details for public (on-chain) account. + #[prost(bytes = "vec", optional, tag = "3")] + pub details: ::core::option::Option<::prost::alloc::vec::Vec>, } #[derive(Eq, PartialOrd, Ord, Hash)] #[allow(clippy::derive_partial_eq_without_eq)] diff --git a/store/Cargo.toml b/store/Cargo.toml index db5f8b0a3..b74edb8fc 100644 --- a/store/Cargo.toml +++ b/store/Cargo.toml @@ -42,4 +42,6 @@ tracing-subscriber = { workspace = true } [dev-dependencies] figment = { version = "0.10", features = ["toml", "env", "test"] } +miden-mock = { workspace = true } miden-node-utils = { path = "../utils", version = "0.2", features = ["tracing-forest"] } +winter-rand-utils = { workspace = true } \ No newline at end of file diff --git a/store/src/db/mod.rs b/store/src/db/mod.rs index 4fffac824..5a21b5eab 100644 --- a/store/src/db/mod.rs +++ b/store/src/db/mod.rs @@ -1,12 +1,13 @@ use std::fs::{self, create_dir_all}; use deadpool_sqlite::{Config as SqliteConfig, Hook, HookError, Pool, Runtime}; -use miden_node_proto::generated::account::AccountInfo as AccountInfoPb; +use miden_node_proto::{ + domain::accounts::AccountDetailsUpdate, generated::account::AccountInfo as AccountInfoPb, +}; use miden_objects::{ accounts::Account, crypto::{hash::rpo::RpoDigest, merkle::MerklePath, utils::Deserializable}, notes::Nullifier, - transaction::AccountDetails, utils::Serializable, BlockHeader, GENESIS_BLOCK, }; @@ -276,7 +277,7 @@ impl Db { block_header: BlockHeader, notes: Vec, nullifiers: Vec, - accounts: Vec<(AccountId, Option, RpoDigest)>, + accounts: Vec, ) -> Result<()> { self.pool .get() @@ -359,8 +360,14 @@ impl Db { let transaction = conn.transaction()?; let accounts: Vec<_> = account_smt .leaves() - .map(|(account_id, state_hash)| (account_id, None, state_hash.into())) - .collect(); + .map(|(account_id, state_hash)| { + Ok(AccountDetailsUpdate { + account_id: account_id.try_into()?, + final_state_hash: state_hash.into(), + details: None, + }) + }) + .collect::>()?; sql::apply_block( &transaction, &expected_genesis_header, diff --git a/store/src/db/sql.rs b/store/src/db/sql.rs index 4f5d52aab..2ed6630a1 100644 --- a/store/src/db/sql.rs +++ b/store/src/db/sql.rs @@ -1,15 +1,20 @@ //! Wrapper functions for SQL statements. -use std::rc::Rc; +use std::{borrow::Cow, rc::Rc}; +use miden_node_proto::domain::accounts::AccountDetailsUpdate; use miden_objects::{ - accounts::Account, + accounts::{Account, AccountDelta}, crypto::{hash::rpo::RpoDigest, merkle::MerklePath}, notes::Nullifier, transaction::AccountDetails, utils::serde::{Deserializable, Serializable}, BlockHeader, }; -use rusqlite::{params, types::Value, Connection, Transaction}; +use rusqlite::{ + params, + types::{Value, ValueRef}, + Connection, Transaction, +}; use super::{AccountInfo, Note, NoteCreated, NullifierInfo, Result, StateSyncUpdate}; use crate::{ @@ -153,28 +158,51 @@ pub fn select_account( /// transaction. pub fn upsert_accounts( transaction: &Transaction, - accounts: &[(AccountId, Option, RpoDigest)], + accounts: &[AccountDetailsUpdate], block_num: BlockNumber, ) -> Result { - let mut stmt = transaction.prepare( - " - INSERT OR REPLACE INTO - accounts (account_id, account_hash, block_num, details) - VALUES (?1, ?2, ?3, ?4); - ", + let mut upsert_stmt = transaction.prepare( + "INSERT OR REPLACE INTO accounts (account_id, account_hash, block_num, details) VALUES (?1, ?2, ?3, ?4);", )?; + let mut select_details_stmt = + transaction.prepare("SELECT details FROM accounts WHERE account_id = ?1;")?; let mut count = 0; - for (account_id, details, account_hash) in accounts.iter() { - // TODO: Process account details/delta (in the next PR) + for update in accounts.iter() { + let account_id = update.account_id.into(); + let full_account = match &update.details { + None => None, + Some(AccountDetails::Full(account)) => { + debug_assert!(account.is_new()); + debug_assert_eq!(account_id, u64::from(account.id())); + + Some(Cow::Borrowed(account)) + }, + Some(AccountDetails::Delta(delta)) => { + let mut rows = select_details_stmt.query(params![u64_to_value(account_id)])?; + let Some(row) = rows.next()? else { + return Err(DatabaseError::AccountNotFoundInDb(account_id)); + }; - count += stmt.execute(params![ - u64_to_value(*account_id), - account_hash.to_bytes(), + let account = + apply_delta(account_id, &row.get_ref(0)?, delta, &update.final_state_hash)?; + + Some(Cow::Owned(account)) + }, + }; + + let inserted = upsert_stmt.execute(params![ + u64_to_value(account_id), + update.final_state_hash.to_bytes(), block_num, - details.as_ref().map(|details| details.to_bytes()), - ])? + full_account.as_ref().map(|account| account.to_bytes()), + ])?; + + debug_assert_eq!(inserted, 1); + + count += inserted; } + Ok(count) } @@ -599,7 +627,7 @@ pub fn apply_block( block_header: &BlockHeader, notes: &[Note], nullifiers: &[Nullifier], - accounts: &[(AccountId, Option, RpoDigest)], + accounts: &[AccountDetailsUpdate], ) -> Result { let mut count = 0; count += insert_block_header(transaction, block_header)?; @@ -653,7 +681,7 @@ fn account_info_from_row(row: &rusqlite::Row<'_>) -> Result { let account_hash_data = row.get_ref(1)?.as_blob()?; let account_hash = RpoDigest::read_from_bytes(account_hash_data)?; let block_num = row.get(2)?; - let details = row.get_ref(3)?.as_bytes_or_null()?; + let details = row.get_ref(3)?.as_blob_or_null()?; let details = details.map(Account::read_from_bytes).transpose()?; Ok(AccountInfo { @@ -663,3 +691,30 @@ fn account_info_from_row(row: &rusqlite::Row<'_>) -> Result { details, }) } + +/// Deserializes account and applies account delta. +fn apply_delta( + account_id: u64, + value: &ValueRef<'_>, + delta: &AccountDelta, + final_state_hash: &RpoDigest, +) -> Result { + let account = value.as_blob_or_null()?; + let account = account.map(Account::read_from_bytes).transpose()?; + + let Some(mut account) = account else { + return Err(DatabaseError::AccountNotOnChain(account_id)); + }; + + account.apply_delta(delta)?; + + let actual_hash = account.hash(); + if &actual_hash != final_state_hash { + return Err(DatabaseError::ApplyBlockFailedAccountHashesMismatch { + calculated: actual_hash, + expected: *final_state_hash, + }); + } + + Ok(account) +} diff --git a/store/src/db/tests.rs b/store/src/db/tests.rs index 68f85cb35..0a4801d1a 100644 --- a/store/src/db/tests.rs +++ b/store/src/db/tests.rs @@ -1,11 +1,23 @@ +use miden_lib::transaction::TransactionKernel; +use miden_mock::mock::account::{ + generate_account_seed, mock_account_code, AccountSeedType, ACCOUNT_ID_FUNGIBLE_FAUCET_ON_CHAIN, + ACCOUNT_ID_NON_FUNGIBLE_FAUCET_ON_CHAIN, +}; +use miden_node_proto::domain::accounts::AccountDetailsUpdate; use miden_objects::{ - accounts::AccountId, + accounts::{ + Account, AccountDelta, AccountId, AccountStorage, AccountStorageDelta, AccountType, + AccountVaultDelta, + }, + assets::{Asset, AssetVault, FungibleAsset, NonFungibleAsset, NonFungibleAssetDetails}, block::BlockNoteTree, crypto::{hash::rpo::RpoDigest, merkle::MerklePath}, notes::{NoteMetadata, NoteType, Nullifier}, - BlockHeader, Digest, Felt, FieldElement, ZERO, + transaction::AccountDetails, + BlockHeader, Digest, Felt, FieldElement, Word, ONE, ZERO, }; use rusqlite::{vtab::array, Connection}; +use winter_rand_utils::rand_array; use super::{sql, AccountInfo, Note, NoteCreated, NullifierInfo}; use crate::db::migrations; @@ -157,29 +169,172 @@ fn test_sql_select_accounts() { // test querying empty table let accounts = sql::select_accounts(&mut conn).unwrap(); assert!(accounts.is_empty()); - // test multiple entries let mut state = vec![]; - for i in 0..10 { - let account_id = i; + for i in 0..10_u64 { + let init_seed = rand_array(); + let account_id = AccountId::new_dummy(init_seed, AccountType::RegularAccountImmutableCode); let account_hash = num_to_rpo_digest(i); state.push(AccountInfo { - account_id, + account_id: account_id.into(), account_hash, block_num, details: None, }); let transaction = conn.transaction().unwrap(); - let res = - sql::upsert_accounts(&transaction, &[(account_id, None, account_hash)], block_num); + let res = sql::upsert_accounts( + &transaction, + &[AccountDetailsUpdate { + account_id, + final_state_hash: account_hash, + details: None, + }], + block_num, + ); assert_eq!(res.unwrap(), 1, "One element must have been inserted"); transaction.commit().unwrap(); - let accounts = sql::select_accounts(&mut conn).unwrap(); + let mut accounts = sql::select_accounts(&mut conn).unwrap(); + accounts.sort_by_key(|acc| acc.account_id); + state.sort_by_key(|acc| acc.account_id); assert_eq!(accounts, state); } } +#[test] +fn test_sql_public_account_details() { + let mut conn = create_db(); + + let block_num = 1; + create_block(&mut conn, block_num); + + let (account_id, _seed) = + generate_account_seed(AccountSeedType::RegularAccountUpdatableCodeOnChain); + let fungible_faucet_id = AccountId::try_from(ACCOUNT_ID_FUNGIBLE_FAUCET_ON_CHAIN).unwrap(); + let non_fungible_faucet_id = + AccountId::try_from(ACCOUNT_ID_NON_FUNGIBLE_FAUCET_ON_CHAIN).unwrap(); + + let mut storage = AccountStorage::new(vec![]).unwrap(); + storage.set_item(1, num_to_word(1)).unwrap(); + storage.set_item(3, num_to_word(3)).unwrap(); + storage.set_item(5, num_to_word(5)).unwrap(); + + let nft1 = Asset::NonFungible( + NonFungibleAsset::new( + &NonFungibleAssetDetails::new(non_fungible_faucet_id, vec![1, 2, 3]).unwrap(), + ) + .unwrap(), + ); + + let mut account = Account::new( + account_id, + AssetVault::new(&[ + Asset::Fungible(FungibleAsset::new(fungible_faucet_id, 150).unwrap()), + nft1, + ]) + .unwrap(), + storage, + mock_account_code(&TransactionKernel::assembler()), + ZERO, + ); + + // test querying empty table + let accounts_in_db = sql::select_accounts(&mut conn).unwrap(); + assert!(accounts_in_db.is_empty()); + + let transaction = conn.transaction().unwrap(); + let inserted = sql::upsert_accounts( + &transaction, + &[AccountDetailsUpdate { + account_id, + final_state_hash: account.hash(), + details: Some(AccountDetails::Full(account.clone())), + }], + block_num, + ) + .unwrap(); + + assert_eq!(inserted, 1, "One element must have been inserted"); + + transaction.commit().unwrap(); + + let mut accounts_in_db = sql::select_accounts(&mut conn).unwrap(); + + assert_eq!(accounts_in_db.len(), 1, "One element must have been inserted"); + + let account_read = accounts_in_db.pop().unwrap().details.unwrap(); + // TODO: substitute by a single check, once code imports deserialization is fixed + // (possibly after merging: https://github.com/0xPolygonMiden/miden-vm/pull/1277): + // assert_eq!(account_read, account); + assert_eq!(account_read.id(), account.id()); + assert_eq!(account_read.vault(), account.vault()); + assert_eq!(account_read.storage(), account.storage()); + assert_eq!(account_read.nonce(), account.nonce()); + + let storage_delta = AccountStorageDelta { + cleared_items: vec![3], + updated_items: vec![(4, num_to_word(5)), (5, num_to_word(6))], + }; + + let nft2 = Asset::NonFungible( + NonFungibleAsset::new( + &NonFungibleAssetDetails::new(non_fungible_faucet_id, vec![4, 5, 6]).unwrap(), + ) + .unwrap(), + ); + + let vault_delta = AccountVaultDelta { + added_assets: vec![nft2], + removed_assets: vec![nft1], + }; + + let delta = AccountDelta::new(storage_delta, vault_delta, Some(ONE)).unwrap(); + + account.apply_delta(&delta).unwrap(); + + let transaction = conn.transaction().unwrap(); + let inserted = sql::upsert_accounts( + &transaction, + &[AccountDetailsUpdate { + account_id, + final_state_hash: account.hash(), + details: Some(AccountDetails::Delta(delta.clone())), + }], + block_num, + ) + .unwrap(); + + assert_eq!(inserted, 1, "One element must have been inserted"); + + transaction.commit().unwrap(); + + let mut accounts_in_db = sql::select_accounts(&mut conn).unwrap(); + + assert_eq!(accounts_in_db.len(), 1, "One element must have been inserted"); + + let mut account_read = accounts_in_db.pop().unwrap().details.unwrap(); + + assert_eq!(account_read.id(), account.id()); + assert_eq!(account_read.vault(), account.vault()); + assert_eq!(account_read.nonce(), account.nonce()); + + // Cleared item was not serialized, check it and apply delta only with clear item second time: + assert_eq!(account_read.storage().get_item(3), RpoDigest::default()); + + let storage_delta = AccountStorageDelta { + cleared_items: vec![3], + updated_items: vec![], + }; + account_read + .apply_delta( + &AccountDelta::new(storage_delta, AccountVaultDelta::default(), Some(Felt::new(2))) + .unwrap(), + ) + .unwrap(); + + assert_eq!(account_read.storage(), account.storage()); +} + #[test] fn test_sql_select_nullifiers_by_block_range() { let mut conn = create_db(); @@ -376,18 +531,28 @@ fn test_db_account() { let block_num = 1; create_block(&mut conn, block_num); + let (account_id, _seed) = + generate_account_seed(AccountSeedType::RegularAccountUpdatableCodeOnChain); + // test empty table - let account_ids = vec![0, 1, 2, 3, 4, 5]; + let account_ids = vec![account_id.into(), 1, 2, 3, 4, 5]; let res = sql::select_accounts_by_block_range(&mut conn, 0, u32::MAX, &account_ids).unwrap(); assert!(res.is_empty()); // test insertion - let account_id = 0; let account_hash = num_to_rpo_digest(0); let transaction = conn.transaction().unwrap(); - let row_count = - sql::upsert_accounts(&transaction, &[(account_id, None, account_hash)], block_num).unwrap(); + let row_count = sql::upsert_accounts( + &transaction, + &[AccountDetailsUpdate { + account_id, + final_state_hash: account_hash, + details: None, + }], + block_num, + ) + .unwrap(); transaction.commit().unwrap(); assert_eq!(row_count, 1); @@ -397,7 +562,7 @@ fn test_db_account() { assert_eq!( res, vec![AccountInfo { - account_id, + account_id: account_id.into(), account_hash, block_num, details: None @@ -534,7 +699,11 @@ fn test_notes() { // UTILITIES // ------------------------------------------------------------------------------------------- fn num_to_rpo_digest(n: u64) -> RpoDigest { - RpoDigest::new([Felt::ZERO, Felt::ZERO, Felt::ZERO, Felt::new(n)]) + RpoDigest::new(num_to_word(n)) +} + +fn num_to_word(n: u64) -> Word { + [Felt::ZERO, Felt::ZERO, Felt::ZERO, Felt::new(n)] } fn num_to_nullifier(n: u64) -> Nullifier { diff --git a/store/src/errors.rs b/store/src/errors.rs index 890175128..49a560a4d 100644 --- a/store/src/errors.rs +++ b/store/src/errors.rs @@ -3,14 +3,13 @@ use std::io; use deadpool_sqlite::PoolError; use miden_objects::{ crypto::{ + hash::rpo::RpoDigest, merkle::{MerkleError, MmrError}, utils::DeserializationError, }, notes::Nullifier, AccountError, BlockHeader, NoteError, }; -use miden_objects::accounts::AccountId; -use miden_objects::crypto::hash::rpo::RpoDigest; use rusqlite::types::FromSqlError; use thiserror::Error; use tokio::sync::oneshot::error::RecvError; @@ -48,10 +47,16 @@ pub enum DatabaseError { InteractError(String), #[error("Deserialization of BLOB data from database failed: {0}")] DeserializationError(DeserializationError), + #[error("Corrupted data: {0}")] + CorruptedData(String), + #[error("Account error: {0}")] + AccountError(AccountError), #[error("Block applying was broken because of closed channel on state side: {0}")] ApplyBlockFailedClosedChannel(RecvError), #[error("Account {0} not found in the database")] AccountNotFoundInDb(AccountId), + #[error("Account {0} is not on the chain")] + AccountNotOnChain(AccountId), #[error("Failed to apply block because of on-chain account final hashes mismatch (expected {expected}, \ but calculated is {calculated}")] ApplyBlockFailedAccountHashesMismatch { @@ -66,6 +71,12 @@ impl From for DatabaseError { } } +impl From for DatabaseError { + fn from(value: AccountError) -> Self { + Self::AccountError(value) + } +} + // INITIALIZATION ERRORS // ================================================================================================= diff --git a/store/src/server/api.rs b/store/src/server/api.rs index 7c2d28cf0..df43b2189 100644 --- a/store/src/server/api.rs +++ b/store/src/server/api.rs @@ -2,6 +2,7 @@ use std::sync::Arc; use miden_node_proto::{ convert, + domain::accounts::AccountDetailsUpdate, errors::ConversionError, generated::{ self, @@ -23,7 +24,9 @@ use miden_node_proto::{ }, AccountState, }; -use miden_objects::{notes::Nullifier, BlockHeader, Felt, ZERO}; +use miden_objects::{ + notes::Nullifier, transaction::AccountDetails, utils::Deserializable, BlockHeader, Felt, ZERO, +}; use tonic::{Response, Status}; use tracing::{debug, info, instrument}; @@ -214,18 +217,38 @@ impl api_server::Api for StoreApi { let nullifiers = validate_nullifiers(&request.nullifiers)?; let accounts = request .accounts - .into_iter() + .iter() .map(|account_update| { let account_state: AccountState = account_update .try_into() .map_err(|err: ConversionError| Status::invalid_argument(err.to_string()))?; - Ok(( - account_state.account_id.into(), - None, // TODO: Process account details (next PR) - account_state + + match (account_state.account_id.is_on_chain(), account_update.details.is_some()) { + (true, false) => { + return Err(Status::invalid_argument("On-chain account must have details")); + }, + (false, true) => { + return Err(Status::invalid_argument( + "Off-chain account must not have details", + )); + }, + _ => (), + } + + let details = account_update + .details + .as_ref() + .map(|data| AccountDetails::read_from_bytes(data)) + .transpose() + .map_err(|err| Status::invalid_argument(err.to_string()))?; + + Ok(AccountDetailsUpdate { + account_id: account_state.account_id, + details, + final_state_hash: account_state .account_hash .ok_or(invalid_argument("Account update missing account hash"))?, - )) + }) }) .collect::, Status>>()?; diff --git a/store/src/state.rs b/store/src/state.rs index 2a39dfd08..194d72d7f 100644 --- a/store/src/state.rs +++ b/store/src/state.rs @@ -4,7 +4,9 @@ //! data is atomically written, and that reads are consistent. use std::{mem, sync::Arc}; -use miden_node_proto::{AccountInputRecord, NullifierWitness}; +use miden_node_proto::{ + domain::accounts::AccountDetailsUpdate, AccountInputRecord, NullifierWitness, +}; use miden_node_utils::formatting::{format_account_id, format_array}; use miden_objects::{ block::BlockNoteTree, @@ -13,7 +15,6 @@ use miden_objects::{ merkle::{LeafIndex, Mmr, MmrDelta, MmrPeaks, SimpleSmt, SmtProof, ValuePath}, }, notes::{NoteMetadata, NoteType, Nullifier}, - transaction::AccountDetails, AccountError, BlockHeader, NoteError, ACCOUNT_TREE_DEPTH, ZERO, }; use tokio::{ @@ -107,7 +108,7 @@ impl State { &self, block_header: BlockHeader, nullifiers: Vec, - accounts: Vec<(AccountId, Option, RpoDigest)>, + accounts: Vec, notes: Vec, ) -> Result<(), ApplyBlockError> { let _ = self.writer.try_lock().map_err(|_| ApplyBlockError::ConcurrentWrite)?; @@ -181,8 +182,11 @@ impl State { // update account tree let mut account_tree = inner.account_tree.clone(); - for (account_id, _details, account_hash) in accounts.iter() { - account_tree.insert(LeafIndex::new_max_depth(*account_id), account_hash.into()); + for update in &accounts { + account_tree.insert( + LeafIndex::new_max_depth(update.account_id.into()), + update.final_state_hash.into(), + ); } if account_tree.root() != block_header.account_root() {