From f7e43791e0427d3a5d0a0391011d6ee6edfa905c Mon Sep 17 00:00:00 2001 From: Louise Poole Date: Thu, 19 Dec 2024 14:38:33 +0200 Subject: [PATCH 1/6] feat: add benchmark example --- examples/benchmark/Readme.md | 19 +++ examples/benchmark/main.rs | 271 +++++++++++++++++++++++++++++++++++ 2 files changed, 290 insertions(+) create mode 100644 examples/benchmark/Readme.md create mode 100644 examples/benchmark/main.rs diff --git a/examples/benchmark/Readme.md b/examples/benchmark/Readme.md new file mode 100644 index 00000000..f299738e --- /dev/null +++ b/examples/benchmark/Readme.md @@ -0,0 +1,19 @@ +# Benchmark + +This example benchmarks swap simulation times for given protocols + +## How to run + +```bash +cargo run --release --example benchmark -- --exchange uniswap_v2 --exchange uniswap_v3 +``` + +### To see all config options: +```bash +cargo run --release --example benchmark -- help +``` + +### To print out individual swap logs: +```bash +RUST_LOG=info cargo run --release --example benchmark -- --exchange uniswap_v2 +``` \ No newline at end of file diff --git a/examples/benchmark/main.rs b/examples/benchmark/main.rs new file mode 100644 index 00000000..8ea0b7ea --- /dev/null +++ b/examples/benchmark/main.rs @@ -0,0 +1,271 @@ +use std::{collections::HashMap, env, time::Instant}; + +use clap::Parser; +use futures::{stream::BoxStream, StreamExt}; +use num_bigint::BigUint; +use tracing::info; +use tracing_subscriber::EnvFilter; +use tycho_simulation::{ + evm::{ + decoder::StreamDecodeError, + engine_db::tycho_db::PreCachedDB, + protocol::{ + filters::{balancer_pool_filter, curve_pool_filter}, + uniswap_v2::state::UniswapV2State, + uniswap_v3::state::UniswapV3State, + vm::state::EVMPoolState, + }, + stream::ProtocolStreamBuilder, + }, + models::Token, + protocol::models::BlockUpdate, + tycho_client::feed::component_tracker::ComponentFilter, + tycho_core::dto::Chain, + utils::load_all_tokens, +}; + +#[derive(Parser, Debug, Clone, PartialEq)] +struct Cli { + /// The exchanges to benchmark + #[clap(long, number_of_values = 1)] + exchange: Vec, + + /// The number of swaps to benchmark + #[clap(long, default_value = "100")] + n_swaps: usize, + + /// The tvl threshold to filter the pools by + #[clap(long, default_value = "1000.0")] + tvl_threshold: f64, +} + +#[tokio::main] +async fn main() { + tracing_subscriber::fmt() + .with_env_filter(EnvFilter::from_default_env()) + .with_target(false) + .init(); + + let cli = Cli::parse(); + + let tycho_url = + env::var("TYCHO_URL").unwrap_or_else(|_| "tycho-beta.propellerheads.xyz".to_string()); + let tycho_api_key: String = + env::var("TYCHO_API_KEY").unwrap_or_else(|_| "sampletoken".to_string()); + + let tvl_filter = ComponentFilter::with_tvl_range(cli.tvl_threshold, cli.tvl_threshold); + + let all_tokens = load_all_tokens(tycho_url.as_str(), false, Some(tycho_api_key.as_str())).await; + + let mut results = HashMap::new(); + + for protocol in cli.exchange { + { + let stream = match protocol.as_str() { + "uniswap_v2" => ProtocolStreamBuilder::new(&tycho_url, Chain::Ethereum) + .exchange::("uniswap_v2", tvl_filter.clone(), None) + .auth_key(Some(tycho_api_key.clone())) + .set_tokens(all_tokens.clone()) + .await + .build() + .await + .expect("Failed building Uniswap V2 protocol stream") + .boxed(), + "uniswap_v3" => ProtocolStreamBuilder::new(&tycho_url, Chain::Ethereum) + .exchange::("uniswap_v3", tvl_filter.clone(), None) + .auth_key(Some(tycho_api_key.clone())) + .set_tokens(all_tokens.clone()) + .await + .build() + .await + .expect("Failed building Uniswap V3 protocol stream") + .boxed(), + "balancer_v2" => ProtocolStreamBuilder::new(&tycho_url, Chain::Ethereum) + .exchange::>( + "vm:balancer_v2", + tvl_filter.clone(), + Some(balancer_pool_filter), + ) + .auth_key(Some(tycho_api_key.clone())) + .set_tokens(all_tokens.clone()) + .await + .build() + .await + .expect("Failed building Balancer V2 protocol stream") + .boxed(), + "curve" => ProtocolStreamBuilder::new(&tycho_url, Chain::Ethereum) + .exchange::>( + "vm:curve", + tvl_filter.clone(), + Some(curve_pool_filter), + ) + .auth_key(Some(tycho_api_key.clone())) + .set_tokens(all_tokens.clone()) + .await + .build() + .await + .expect("Failed building Curve protocol stream") + .boxed(), + _ => { + eprintln!("Unknown protocol: {}", protocol); + continue; + } + }; + + info!("BENCHMARKING {} protocol on {} swaps", protocol, cli.n_swaps); + let times = benchmark_swaps(stream, cli.n_swaps).await; + results.insert(protocol, times); + } + // Add a small delay to ensure the WebSocket disconnection completes + tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; + } + + analyze_results(&results, cli.n_swaps); +} + +async fn benchmark_swaps( + mut protocol_stream: BoxStream<'_, Result>, + n: usize, +) -> Vec { + let mut times = Vec::new(); + let mut pairs: HashMap> = HashMap::new(); + + if let Some(Ok(message)) = protocol_stream.next().await { + for (id, comp) in message.new_pairs.iter() { + pairs + .entry(id.clone()) + .or_insert_with(|| comp.tokens.clone()); + } + + if message.states.is_empty() { + return times; + } + + for (id, tokens) in pairs.iter().cycle() { + if let Some(state) = message.states.get(id) { + let amount_in = + BigUint::from(1u32) * BigUint::from(10u32).pow(tokens[0].decimals as u32); + + let start = Instant::now(); + let _ = state.get_amount_out(amount_in.clone(), &tokens[0], &tokens[1]); + let duration = start.elapsed().as_nanos(); + + times.push(duration); + + info!("Swap {} -> {} took {} ns", tokens[0].symbol, tokens[1].symbol, duration); + + if times.len() >= n { + break; + } + } + } + } + + times +} + +fn calculate_std_dev(times: &[u128], avg: f64) -> f64 { + let variance = times + .iter() + .map(|&time| (time as f64 - avg).powi(2)) + .sum::() / + times.len() as f64; + variance.sqrt() +} + +/// Calculate the trimmed mean of a dataset. +/// +/// The trimmed mean is calculated by removing outliers from the dataset. Outliers are defined as +/// values that are below the first quartile minus 1.5 times the interquartile range, or above the +/// third quartile plus 1.5 times the interquartile range. This gives us a more robust estimate of +/// the central tendency of the data. +fn calculate_trimmed_mean(times: &[u128]) -> Option { + if times.is_empty() { + return None; + } + + // Sort the data + let mut sorted_times = times.to_vec(); + sorted_times.sort_unstable(); + + // Calculate quartiles + let q1_index = sorted_times.len() / 4; + let q3_index = 3 * sorted_times.len() / 4; + + let q1 = sorted_times[q1_index]; + let q3 = sorted_times[q3_index]; + let iqr = q3 - q1; + + // Convert to floating point for multiplication + let lower_bound = (q1 as f64 - 1.5 * iqr as f64).max(0.0) as u128; + let upper_bound = (q3 as f64 + 1.5 * iqr as f64) as u128; + + // Filter out outliers + let filtered_times: Vec<&u128> = sorted_times + .iter() + .filter(|&&t| t >= lower_bound && t <= upper_bound) + .collect(); + + if filtered_times.is_empty() { + None + } else { + // Calculate the trimmed mean + Some( + filtered_times + .iter() + .map(|&&t| t as f64) + .sum::() / + filtered_times.len() as f64, + ) + } +} + +fn analyze_results(results: &HashMap>, n_swaps: usize) { + println!("\n========== Benchmark Results on {} swaps ==========", n_swaps); + + for (protocol, times) in results { + let avg = times.iter().sum::() as f64 / times.len() as f64; + let max = times.iter().max().unwrap_or(&0); + let min = times.iter().min().unwrap_or(&0); + let std_dev = calculate_std_dev(times, avg); + let trimmed_mean = calculate_trimmed_mean(times).unwrap_or(f64::NAN); + + println!( + "\n{} - Mean Time: {:.2} ns, Trimmed Mean Time: {:.2} ns, Max Time: {} ns, Min Time: {} ns, Std Dev: {:.2} ns", + protocol, avg, trimmed_mean, max, min, std_dev + ); + + generate_histogram(times, 10); + + println!("\n---------------------------------------"); + } +} + +fn generate_histogram(data: &[u128], num_bins: usize) { + if data.is_empty() { + println!("No data to display in histogram."); + return; + } + + let min = *data.iter().min().unwrap(); + let max = *data.iter().max().unwrap(); + let range = max - min; + let bin_width = (range as f64 / num_bins as f64).ceil() as u128; + + // Initialize bins + let mut bins = vec![0; num_bins]; + + // Count data into bins + for &value in data { + let bin_index = ((value - min) / bin_width).min(num_bins as u128 - 1) as usize; + bins[bin_index] += 1; + } + + // Display the histogram + println!("\nHistogram:"); + for (i, &count) in bins.iter().enumerate() { + let lower_bound = min + (i as u128 * bin_width); + let upper_bound = lower_bound + bin_width - 1; + println!("{:>8} - {:<8} | {}", lower_bound, upper_bound, "*".repeat(count)); + } +} From adbdfb42afadaacdf46dfb0aac2092888f900493 Mon Sep 17 00:00:00 2001 From: Louise Poole Date: Thu, 19 Dec 2024 18:23:22 +0200 Subject: [PATCH 2/6] docs: add require env vars to readme instructions --- examples/benchmark/Readme.md | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/benchmark/Readme.md b/examples/benchmark/Readme.md index f299738e..4cc2c8f7 100644 --- a/examples/benchmark/Readme.md +++ b/examples/benchmark/Readme.md @@ -5,6 +5,7 @@ This example benchmarks swap simulation times for given protocols ## How to run ```bash +export RPC_URL= cargo run --release --example benchmark -- --exchange uniswap_v2 --exchange uniswap_v3 ``` From c43437678bebadd90d61c52d1cf69edab998c2d1 Mon Sep 17 00:00:00 2001 From: Louise Poole Date: Fri, 20 Dec 2024 15:39:10 +0200 Subject: [PATCH 3/6] chore: clean up file structure and validate exchanges cli arg --- examples/benchmark/main.rs | 64 ++++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 26 deletions(-) diff --git a/examples/benchmark/main.rs b/examples/benchmark/main.rs index 8ea0b7ea..52481ce7 100644 --- a/examples/benchmark/main.rs +++ b/examples/benchmark/main.rs @@ -27,7 +27,7 @@ use tycho_simulation::{ #[derive(Parser, Debug, Clone, PartialEq)] struct Cli { /// The exchanges to benchmark - #[clap(long, number_of_values = 1)] + #[clap(long, number_of_values = 1, value_parser = validate_exchange)] exchange: Vec, /// The number of swaps to benchmark @@ -39,6 +39,18 @@ struct Cli { tvl_threshold: f64, } +fn validate_exchange(exchange: &str) -> Result { + const SUPPORTED_EXCHANGES: &[&str] = &["uniswap_v2", "uniswap_v3", "balancer_v2", "curve"]; + if SUPPORTED_EXCHANGES.contains(&exchange) { + Ok(exchange.to_string()) + } else { + Err(format!( + "Unsupported exchange '{}'. Supported exchanges are: {:?}", + exchange, SUPPORTED_EXCHANGES + )) + } +} + #[tokio::main] async fn main() { tracing_subscriber::fmt() @@ -67,6 +79,7 @@ async fn main() { .auth_key(Some(tycho_api_key.clone())) .set_tokens(all_tokens.clone()) .await + .skip_state_decode_failures(true) .build() .await .expect("Failed building Uniswap V2 protocol stream") @@ -76,6 +89,7 @@ async fn main() { .auth_key(Some(tycho_api_key.clone())) .set_tokens(all_tokens.clone()) .await + .skip_state_decode_failures(true) .build() .await .expect("Failed building Uniswap V3 protocol stream") @@ -89,6 +103,7 @@ async fn main() { .auth_key(Some(tycho_api_key.clone())) .set_tokens(all_tokens.clone()) .await + .skip_state_decode_failures(true) .build() .await .expect("Failed building Balancer V2 protocol stream") @@ -102,6 +117,7 @@ async fn main() { .auth_key(Some(tycho_api_key.clone())) .set_tokens(all_tokens.clone()) .await + .skip_state_decode_failures(true) .build() .await .expect("Failed building Curve protocol stream") @@ -173,6 +189,27 @@ fn calculate_std_dev(times: &[u128], avg: f64) -> f64 { variance.sqrt() } +fn analyze_results(results: &HashMap>, n_swaps: usize) { + println!("\n========== Benchmark Results on {} swaps ==========", n_swaps); + + for (protocol, times) in results { + let avg = times.iter().sum::() as f64 / times.len() as f64; + let max = times.iter().max().unwrap_or(&0); + let min = times.iter().min().unwrap_or(&0); + let std_dev = calculate_std_dev(times, avg); + let trimmed_mean = calculate_trimmed_mean(times).unwrap_or(f64::NAN); + + println!( + "\n{} - Mean Time: {:.2} ns, Trimmed Mean Time: {:.2} ns, Max Time: {} ns, Min Time: {} ns, Std Dev: {:.2} ns", + protocol, avg, trimmed_mean, max, min, std_dev + ); + + generate_histogram(times, 10); + + println!("\n---------------------------------------"); + } +} + /// Calculate the trimmed mean of a dataset. /// /// The trimmed mean is calculated by removing outliers from the dataset. Outliers are defined as @@ -184,7 +221,6 @@ fn calculate_trimmed_mean(times: &[u128]) -> Option { return None; } - // Sort the data let mut sorted_times = times.to_vec(); sorted_times.sort_unstable(); @@ -196,7 +232,6 @@ fn calculate_trimmed_mean(times: &[u128]) -> Option { let q3 = sorted_times[q3_index]; let iqr = q3 - q1; - // Convert to floating point for multiplication let lower_bound = (q1 as f64 - 1.5 * iqr as f64).max(0.0) as u128; let upper_bound = (q3 as f64 + 1.5 * iqr as f64) as u128; @@ -220,27 +255,6 @@ fn calculate_trimmed_mean(times: &[u128]) -> Option { } } -fn analyze_results(results: &HashMap>, n_swaps: usize) { - println!("\n========== Benchmark Results on {} swaps ==========", n_swaps); - - for (protocol, times) in results { - let avg = times.iter().sum::() as f64 / times.len() as f64; - let max = times.iter().max().unwrap_or(&0); - let min = times.iter().min().unwrap_or(&0); - let std_dev = calculate_std_dev(times, avg); - let trimmed_mean = calculate_trimmed_mean(times).unwrap_or(f64::NAN); - - println!( - "\n{} - Mean Time: {:.2} ns, Trimmed Mean Time: {:.2} ns, Max Time: {} ns, Min Time: {} ns, Std Dev: {:.2} ns", - protocol, avg, trimmed_mean, max, min, std_dev - ); - - generate_histogram(times, 10); - - println!("\n---------------------------------------"); - } -} - fn generate_histogram(data: &[u128], num_bins: usize) { if data.is_empty() { println!("No data to display in histogram."); @@ -252,10 +266,8 @@ fn generate_histogram(data: &[u128], num_bins: usize) { let range = max - min; let bin_width = (range as f64 / num_bins as f64).ceil() as u128; - // Initialize bins let mut bins = vec![0; num_bins]; - // Count data into bins for &value in data { let bin_index = ((value - min) / bin_width).min(num_bins as u128 - 1) as usize; bins[bin_index] += 1; From 7ae5cbda59b1d678c4c1fc348337e8e6982b624b Mon Sep 17 00:00:00 2001 From: Louise Poole Date: Tue, 4 Feb 2025 14:27:16 +0200 Subject: [PATCH 4/6] feat: add uniswap_v4 --- examples/benchmark/main.rs | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/examples/benchmark/main.rs b/examples/benchmark/main.rs index 52481ce7..67e12f6a 100644 --- a/examples/benchmark/main.rs +++ b/examples/benchmark/main.rs @@ -10,9 +10,10 @@ use tycho_simulation::{ decoder::StreamDecodeError, engine_db::tycho_db::PreCachedDB, protocol::{ - filters::{balancer_pool_filter, curve_pool_filter}, + filters::{balancer_pool_filter, curve_pool_filter, uniswap_v4_pool_with_hook_filter}, uniswap_v2::state::UniswapV2State, uniswap_v3::state::UniswapV3State, + uniswap_v4::state::UniswapV4State, vm::state::EVMPoolState, }, stream::ProtocolStreamBuilder, @@ -40,7 +41,8 @@ struct Cli { } fn validate_exchange(exchange: &str) -> Result { - const SUPPORTED_EXCHANGES: &[&str] = &["uniswap_v2", "uniswap_v3", "balancer_v2", "curve"]; + const SUPPORTED_EXCHANGES: &[&str] = + &["uniswap_v2", "uniswap_v3", "balancer_v2", "curve", "uniswap_v4"]; if SUPPORTED_EXCHANGES.contains(&exchange) { Ok(exchange.to_string()) } else { @@ -94,6 +96,20 @@ async fn main() { .await .expect("Failed building Uniswap V3 protocol stream") .boxed(), + "uniswap_v4" => ProtocolStreamBuilder::new(&tycho_url, Chain::Ethereum) + .exchange::( + "uniswap_v4", + tvl_filter.clone(), + Some(uniswap_v4_pool_with_hook_filter), + ) + .auth_key(Some(tycho_api_key.clone())) + .set_tokens(all_tokens.clone()) + .await + .skip_state_decode_failures(true) + .build() + .await + .expect("Failed building Uniswap V3 protocol stream") + .boxed(), "balancer_v2" => ProtocolStreamBuilder::new(&tycho_url, Chain::Ethereum) .exchange::>( "vm:balancer_v2", @@ -153,6 +169,8 @@ async fn benchmark_swaps( .or_insert_with(|| comp.tokens.clone()); } + info!("Got {} pairs", pairs.len()); + if message.states.is_empty() { return times; } From 81d4812798b2ced8003227930e3df1a6a218c69f Mon Sep 17 00:00:00 2001 From: Louise Poole Date: Tue, 4 Feb 2025 14:27:29 +0200 Subject: [PATCH 5/6] feat: calculate median --- examples/benchmark/main.rs | 45 +++++++++++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/examples/benchmark/main.rs b/examples/benchmark/main.rs index 67e12f6a..1d72a39c 100644 --- a/examples/benchmark/main.rs +++ b/examples/benchmark/main.rs @@ -215,11 +215,11 @@ fn analyze_results(results: &HashMap>, n_swaps: usize) { let max = times.iter().max().unwrap_or(&0); let min = times.iter().min().unwrap_or(&0); let std_dev = calculate_std_dev(times, avg); - let trimmed_mean = calculate_trimmed_mean(times).unwrap_or(f64::NAN); + let median = calculate_median(times).unwrap_or(f64::NAN); println!( - "\n{} - Mean Time: {:.2} ns, Trimmed Mean Time: {:.2} ns, Max Time: {} ns, Min Time: {} ns, Std Dev: {:.2} ns", - protocol, avg, trimmed_mean, max, min, std_dev + "\n{} - Mean Time: {:.2} ns, Median Time: {:.2} ns, Max Time: {} ns, Min Time: {} ns, Std Dev: {:.2} ns", + protocol, avg, median, max, min, std_dev ); generate_histogram(times, 10); @@ -273,6 +273,45 @@ fn calculate_trimmed_mean(times: &[u128]) -> Option { } } +fn calculate_median(times: &[u128]) -> Option { + if times.is_empty() { + return None; + } + + let mut sorted_times = times.to_vec(); + sorted_times.sort_unstable(); + + // Calculate quartiles + let q1_index = sorted_times.len() / 4; + let q3_index = 3 * sorted_times.len() / 4; + + let q1 = sorted_times[q1_index]; + let q3 = sorted_times[q3_index]; + let iqr = q3 - q1; + + let lower_bound = (q1 as f64 - 1.5 * iqr as f64).max(0.0) as u128; + let upper_bound = (q3 as f64 + 1.5 * iqr as f64) as u128; + + // Filter out outliers + let filtered_times: Vec<&u128> = sorted_times + .iter() + .filter(|&&t| t >= lower_bound && t <= upper_bound) + .collect(); + + if filtered_times.is_empty() { + None + } else { + // Calculate the trimmed mean + Some( + filtered_times + .iter() + .map(|&&t| t as f64) + .sum::() / + filtered_times.len() as f64, + ) + } +} + fn generate_histogram(data: &[u128], num_bins: usize) { if data.is_empty() { println!("No data to display in histogram."); From 567172b2ee51bfc1946228814507581b13d3cfb8 Mon Sep 17 00:00:00 2001 From: Louise Poole Date: Tue, 4 Feb 2025 15:54:35 +0200 Subject: [PATCH 6/6] chore: remove unused fn --- examples/benchmark/main.rs | 45 -------------------------------------- 1 file changed, 45 deletions(-) diff --git a/examples/benchmark/main.rs b/examples/benchmark/main.rs index 1d72a39c..f84fb657 100644 --- a/examples/benchmark/main.rs +++ b/examples/benchmark/main.rs @@ -228,51 +228,6 @@ fn analyze_results(results: &HashMap>, n_swaps: usize) { } } -/// Calculate the trimmed mean of a dataset. -/// -/// The trimmed mean is calculated by removing outliers from the dataset. Outliers are defined as -/// values that are below the first quartile minus 1.5 times the interquartile range, or above the -/// third quartile plus 1.5 times the interquartile range. This gives us a more robust estimate of -/// the central tendency of the data. -fn calculate_trimmed_mean(times: &[u128]) -> Option { - if times.is_empty() { - return None; - } - - let mut sorted_times = times.to_vec(); - sorted_times.sort_unstable(); - - // Calculate quartiles - let q1_index = sorted_times.len() / 4; - let q3_index = 3 * sorted_times.len() / 4; - - let q1 = sorted_times[q1_index]; - let q3 = sorted_times[q3_index]; - let iqr = q3 - q1; - - let lower_bound = (q1 as f64 - 1.5 * iqr as f64).max(0.0) as u128; - let upper_bound = (q3 as f64 + 1.5 * iqr as f64) as u128; - - // Filter out outliers - let filtered_times: Vec<&u128> = sorted_times - .iter() - .filter(|&&t| t >= lower_bound && t <= upper_bound) - .collect(); - - if filtered_times.is_empty() { - None - } else { - // Calculate the trimmed mean - Some( - filtered_times - .iter() - .map(|&&t| t as f64) - .sum::() / - filtered_times.len() as f64, - ) - } -} - fn calculate_median(times: &[u128]) -> Option { if times.is_empty() { return None;