From 5fe0dd6c59e10e2fc7f09755e7e288857445dcbc Mon Sep 17 00:00:00 2001 From: ilya Date: Wed, 29 Jan 2025 12:27:58 +0000 Subject: [PATCH 01/82] Solver participation validator --- Cargo.lock | 1 + crates/autopilot/Cargo.toml | 1 + crates/autopilot/src/arguments.rs | 20 +++ crates/autopilot/src/database/competition.rs | 21 +++ .../autopilot/src/domain/competition/mod.rs | 6 +- .../competition/solver_participation_guard.rs | 157 ++++++++++++++++++ .../src/domain/settlement/observer.rs | 21 ++- crates/autopilot/src/run.rs | 19 ++- crates/autopilot/src/run_loop.rs | 33 ++-- crates/database/src/solver_competition.rs | 51 ++++++ 10 files changed, 308 insertions(+), 22 deletions(-) create mode 100644 crates/autopilot/src/domain/competition/solver_participation_guard.rs diff --git a/Cargo.lock b/Cargo.lock index 7b4421db5d..2e208f3ec9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -284,6 +284,7 @@ dependencies = [ "clap", "contracts", "cow-amm", + "dashmap", "database", "derive_more 1.0.0", "ethcontract", diff --git a/crates/autopilot/Cargo.toml b/crates/autopilot/Cargo.toml index 9cdfb90295..7fa92c9ace 100644 --- a/crates/autopilot/Cargo.toml +++ b/crates/autopilot/Cargo.toml @@ -25,6 +25,7 @@ chrono = { workspace = true } clap = { workspace = true } contracts = { path = "../contracts" } cow-amm = { path = "../cow-amm" } +dashmap = { workspace = true } database = { path = "../database" } derive_more = { workspace = true } ethcontract = { workspace = true } diff --git a/crates/autopilot/src/arguments.rs b/crates/autopilot/src/arguments.rs index 5d7b155cb7..ed5aa92657 100644 --- a/crates/autopilot/src/arguments.rs +++ b/crates/autopilot/src/arguments.rs @@ -242,6 +242,14 @@ pub struct Arguments { /// Archive node URL used to index CoW AMM #[clap(long, env)] pub archive_node_url: Option, + + #[clap(long, env, default_value = "5m", value_parser = humantime::parse_duration)] + /// The time-to-live for the solver participation blacklist cache. + pub solver_blacklist_cache_ttl: Duration, + + #[clap(long, env, default_value = "3")] + /// The number of last auctions to check solver participation eligibility. + pub solver_last_auctions_participation_count: u32, } impl std::fmt::Display for Arguments { @@ -287,6 +295,8 @@ impl std::fmt::Display for Arguments { max_winners_per_auction, archive_node_url, max_solutions_per_solver, + solver_blacklist_cache_ttl, + solver_last_auctions_participation_count, } = self; write!(f, "{}", shared)?; @@ -370,6 +380,16 @@ impl std::fmt::Display for Arguments { "max_solutions_per_solver: {:?}", max_solutions_per_solver )?; + writeln!( + f, + "solver_blacklist_cache_ttl: {:?}", + solver_blacklist_cache_ttl + )?; + writeln!( + f, + "solver_last_auctions_participation_count: {:?}", + solver_last_auctions_participation_count + )?; Ok(()) } } diff --git a/crates/autopilot/src/database/competition.rs b/crates/autopilot/src/database/competition.rs index 81cc5e63ef..2411e74f0f 100644 --- a/crates/autopilot/src/database/competition.rs +++ b/crates/autopilot/src/database/competition.rs @@ -139,4 +139,25 @@ impl super::Postgres { Ok(()) } + + pub async fn find_non_settling_solvers( + &self, + last_auctions_count: u32, + current_block: u64, + ) -> anyhow::Result> { + let mut ex = self.pool.acquire().await.context("acquire")?; + + let _timer = super::Metrics::get() + .database_queries + .with_label_values(&["find_non_settling_solvers"]) + .start_timer(); + + database::solver_competition::find_non_settling_solvers( + &mut ex, + last_auctions_count, + current_block, + ) + .await + .context("solver_competition::find_non_settling_solvers") + } } diff --git a/crates/autopilot/src/domain/competition/mod.rs b/crates/autopilot/src/domain/competition/mod.rs index 52cd69df22..bcd5210536 100644 --- a/crates/autopilot/src/domain/competition/mod.rs +++ b/crates/autopilot/src/domain/competition/mod.rs @@ -6,8 +6,12 @@ use { }; mod participant; +mod solver_participation_guard; -pub use participant::{Participant, Ranked, Unranked}; +pub use { + participant::{Participant, Ranked, Unranked}, + solver_participation_guard::{DatabaseSolverParticipationValidator, SolverParticipationGuard}, +}; type SolutionId = u64; diff --git a/crates/autopilot/src/domain/competition/solver_participation_guard.rs b/crates/autopilot/src/domain/competition/solver_participation_guard.rs new file mode 100644 index 0000000000..1ec1379098 --- /dev/null +++ b/crates/autopilot/src/domain/competition/solver_participation_guard.rs @@ -0,0 +1,157 @@ +use { + crate::{database::Postgres, domain::eth, infra::Ethereum}, + ethrpc::block_stream::CurrentBlockWatcher, + futures::future::try_join_all, + std::{ + sync::Arc, + time::{Duration, Instant}, + }, +}; + +#[derive(Clone)] +pub struct SolverParticipationGuard(Arc); + +struct Inner { + validators: Vec>, +} + +impl SolverParticipationGuard { + pub fn new( + eth: Ethereum, + db: Postgres, + settlement_updates_receiver: tokio::sync::mpsc::UnboundedReceiver<()>, + ttl: Duration, + last_auctions_count: u32, + ) -> Self { + let current_block = eth.current_block().clone(); + let onchain_solver_participation_validator = OnchainSolverParticipationValidator { eth }; + let database_solver_participation_validator = DatabaseSolverParticipationValidator::new( + db, + current_block, + settlement_updates_receiver, + ttl, + last_auctions_count, + ); + + Self(Arc::new(Inner { + validators: vec![ + Box::new(database_solver_participation_validator), + Box::new(onchain_solver_participation_validator), + ], + })) + } + + pub async fn can_participate(&self, solver: ð::Address) -> anyhow::Result { + try_join_all( + self.0 + .validators + .iter() + .map(|strategy| strategy.can_participate(solver)), + ) + .await + .map(|results| results.into_iter().all(|can_participate| can_participate)) + } +} + +#[async_trait::async_trait] +trait SolverParticipationValidator: Send + Sync { + async fn can_participate(&self, solver: ð::Address) -> anyhow::Result; +} + +#[derive(Clone)] +pub struct DatabaseSolverParticipationValidator(Arc); + +struct DatabaseSolverParticipationValidatorInner { + db: Postgres, + cache: dashmap::DashMap, + ttl: Duration, + last_auctions_count: u32, +} + +impl DatabaseSolverParticipationValidator { + pub fn new( + db: Postgres, + current_block: CurrentBlockWatcher, + settlement_updates_receiver: tokio::sync::mpsc::UnboundedReceiver<()>, + ttl: Duration, + last_auctions_count: u32, + ) -> Self { + let self_ = Self(Arc::new(DatabaseSolverParticipationValidatorInner { + db, + cache: Default::default(), + ttl, + last_auctions_count, + })); + + self_.start_maintenance(settlement_updates_receiver, current_block); + + self_ + } + + fn start_maintenance( + &self, + mut settlement_updates_receiver: tokio::sync::mpsc::UnboundedReceiver<()>, + current_block: CurrentBlockWatcher, + ) { + let self_ = self.0.clone(); + tokio::spawn(async move { + while settlement_updates_receiver.recv().await.is_some() { + let current_block = current_block.borrow().number; + match self_ + .db + .find_non_settling_solvers(self_.last_auctions_count, current_block) + .await + { + Ok(non_settling_solvers) => { + let non_settling_solvers = non_settling_solvers + .into_iter() + .map(|solver| eth::Address(solver.0.into())) + .collect::>(); + + tracing::debug!(?non_settling_solvers, "found non-settling solvers",); + + let now = Instant::now(); + for solver in non_settling_solvers { + self_.cache.insert(solver, now); + } + } + Err(err) => { + tracing::warn!(?err, "error while searching for non-settling solvers") + } + } + } + }); + } +} + +#[async_trait::async_trait] +impl SolverParticipationValidator for DatabaseSolverParticipationValidator { + async fn can_participate(&self, solver: ð::Address) -> anyhow::Result { + if let Some(entry) = self.0.cache.get(solver) { + if Instant::now().duration_since(*entry.value()) < self.0.ttl { + return Ok(false); + } else { + self.0.cache.remove(solver); + } + } + + Ok(true) + } +} + +struct OnchainSolverParticipationValidator { + eth: Ethereum, +} + +#[async_trait::async_trait] +impl SolverParticipationValidator for OnchainSolverParticipationValidator { + async fn can_participate(&self, solver: ð::Address) -> anyhow::Result { + Ok(self + .eth + .contracts() + .authenticator() + .is_solver(solver.0) + .call() + .await?) + } +} diff --git a/crates/autopilot/src/domain/settlement/observer.rs b/crates/autopilot/src/domain/settlement/observer.rs index d142646728..0374465a1f 100644 --- a/crates/autopilot/src/domain/settlement/observer.rs +++ b/crates/autopilot/src/domain/settlement/observer.rs @@ -19,23 +19,34 @@ use { pub struct Observer { eth: infra::Ethereum, persistence: infra::Persistence, + settlement_updates_sender: tokio::sync::mpsc::UnboundedSender<()>, } impl Observer { /// Creates a new Observer and asynchronously schedules the first update /// run. - pub fn new(eth: infra::Ethereum, persistence: infra::Persistence) -> Self { - Self { eth, persistence } + pub fn new( + eth: infra::Ethereum, + persistence: infra::Persistence, + settlement_updates_sender: tokio::sync::mpsc::UnboundedSender<()>, + ) -> Self { + Self { + eth, + persistence, + settlement_updates_sender, + } } /// Fetches all the available missing data needed for bookkeeping. /// This needs to get called after indexing a new settlement event /// since this code needs that data to already be present in the DB. pub async fn update(&self) { + let mut updated = false; loop { match self.single_update().await { Ok(true) => { tracing::debug!("on settlement event updater ran and processed event"); + updated = true; // There might be more pending updates, continue immediately. continue; } @@ -49,6 +60,12 @@ impl Observer { } } } + if updated { + // Notify the solver participation guard that a settlement has been updated. + if let Err(err) = self.settlement_updates_sender.send(()) { + tracing::error!(?err, "failed to notify solver participation guard"); + } + } } /// Update database for settlement events that have not been processed yet. diff --git a/crates/autopilot/src/run.rs b/crates/autopilot/src/run.rs index 1ea1f6de93..ede9f38045 100644 --- a/crates/autopilot/src/run.rs +++ b/crates/autopilot/src/run.rs @@ -15,7 +15,7 @@ use { }, Postgres, }, - domain, + domain::{self, competition::SolverParticipationGuard}, event_updater::EventUpdater, infra, maintenance::Maintenance, @@ -367,8 +367,20 @@ pub async fn run(args: Arguments) { let persistence = infra::persistence::Persistence::new(args.s3.into().unwrap(), Arc::new(db.clone())).await; - let settlement_observer = - crate::domain::settlement::Observer::new(eth.clone(), persistence.clone()); + let (settlement_updates_sender, settlement_updates_receiver) = + tokio::sync::mpsc::unbounded_channel(); + let settlement_observer = crate::domain::settlement::Observer::new( + eth.clone(), + persistence.clone(), + settlement_updates_sender, + ); + let solver_participation_guard = SolverParticipationGuard::new( + eth.clone(), + db.clone(), + settlement_updates_receiver, + args.solver_blacklist_cache_ttl, + args.solver_last_auctions_participation_count, + ); let settlement_contract_start_index = if let Some(DeploymentInformation::BlockNumber(settlement_contract_start_index)) = eth.contracts().settlement().deployment_information() @@ -574,6 +586,7 @@ pub async fn run(args: Arguments) { eth, persistence.clone(), drivers, + solver_participation_guard, solvable_orders_cache, trusted_tokens, liveness.clone(), diff --git a/crates/autopilot/src/run_loop.rs b/crates/autopilot/src/run_loop.rs index fa0d2a5c0f..492ca18751 100644 --- a/crates/autopilot/src/run_loop.rs +++ b/crates/autopilot/src/run_loop.rs @@ -4,7 +4,14 @@ use { domain::{ self, auction::Id, - competition::{self, Solution, SolutionError, TradedOrder, Unranked}, + competition::{ + self, + Solution, + SolutionError, + SolverParticipationGuard, + TradedOrder, + Unranked, + }, eth::{self, TxId}, OrderUid, }, @@ -59,6 +66,7 @@ pub struct RunLoop { eth: infra::Ethereum, persistence: infra::Persistence, drivers: Vec>, + solver_participation_guard: SolverParticipationGuard, solvable_orders_cache: Arc, trusted_tokens: AutoUpdatingTokenList, in_flight_orders: Arc>>, @@ -75,6 +83,7 @@ impl RunLoop { eth: infra::Ethereum, persistence: infra::Persistence, drivers: Vec>, + solver_participation_guard: SolverParticipationGuard, solvable_orders_cache: Arc, trusted_tokens: AutoUpdatingTokenList, liveness: Arc, @@ -85,6 +94,7 @@ impl RunLoop { eth, persistence, drivers, + solver_participation_guard, solvable_orders_cache, trusted_tokens, in_flight_orders: Default::default(), @@ -726,23 +736,14 @@ impl RunLoop { request: &solve::Request, ) -> Result>, SolveError> { - let authenticator = self.eth.contracts().authenticator(); - let is_allowed = authenticator - .is_solver(driver.submission_address.into()) - .call() - .await - .map_err(|err| { - tracing::warn!( - driver = driver.name, - ?driver.submission_address, - ?err, - "failed to check if solver is deny listed" - ); + let can_participate = self.solver_participation_guard.can_participate(&driver.submission_address).await.map_err(|err| { + tracing::error!(?err, driver = %driver.name, ?driver.submission_address, "solver participation check failed"); SolveError::SolverDenyListed - })?; + } + )?; - // Do not send the request to the driver if the solver is denied - if !is_allowed { + // Do not send the request to the driver if the solver is deny-listed + if !can_participate { return Err(SolveError::SolverDenyListed); } diff --git a/crates/database/src/solver_competition.rs b/crates/database/src/solver_competition.rs index 01e0183858..6edd700c70 100644 --- a/crates/database/src/solver_competition.rs +++ b/crates/database/src/solver_competition.rs @@ -97,6 +97,57 @@ GROUP BY sc.id sqlx::query_as(QUERY).bind(tx_hash).fetch_optional(ex).await } +pub async fn find_non_settling_solvers( + ex: &mut PgConnection, + last_auctions_count: u32, + current_block: u64, +) -> Result, sqlx::Error> { + const QUERY: &str = r#" +WITH + last_auctions AS ( + SELECT ps.auction_id, ps.solver + FROM ( + SELECT DISTINCT ca.id AS auction_id + FROM competition_auctions ca + WHERE ca.deadline <= $1 + ORDER BY ca.id DESC + LIMIT $2 + ) latest_auctions + JOIN proposed_solutions ps ON ps.auction_id = latest_auctions.auction_id + WHERE ps.is_winner = true + ), + unsuccessful_solvers AS ( + SELECT la.auction_id, la.solver + FROM last_auctions la + LEFT JOIN settlements s + ON la.auction_id = s.auction_id AND la.solver = s.solver + WHERE s.auction_id IS NULL + ), + solver_appearance_count AS ( + SELECT solver, COUNT(DISTINCT auction_id) AS appearance_count + FROM unsuccessful_solvers + GROUP BY solver + ), + auction_count AS ( + SELECT COUNT(DISTINCT auction_id) AS total_auctions + FROM last_auctions + ), + consistent_solvers AS ( + SELECT sa.solver + FROM solver_appearance_count sa, auction_count ac + WHERE sa.appearance_count = ac.total_auctions + ) +SELECT DISTINCT solver +FROM consistent_solvers; + "#; + + sqlx::query_scalar(QUERY) + .bind(sqlx::types::BigDecimal::from(current_block)) + .bind(i64::from(last_auctions_count)) + .fetch_all(ex) + .await +} + #[derive(Clone, Debug, PartialEq, Default)] pub struct Solution { // Unique Id generated by the autopilot to uniquely identify the solution within Auction From 53199457ffed674c60a66bb990bb989792bd0ade Mon Sep 17 00:00:00 2001 From: ilya Date: Wed, 29 Jan 2025 13:25:22 +0000 Subject: [PATCH 02/82] Test --- crates/database/src/solver_competition.rs | 166 +++++++++++++++++++++- 1 file changed, 165 insertions(+), 1 deletion(-) diff --git a/crates/database/src/solver_competition.rs b/crates/database/src/solver_competition.rs index 6edd700c70..0139610671 100644 --- a/crates/database/src/solver_competition.rs +++ b/crates/database/src/solver_competition.rs @@ -368,8 +368,10 @@ mod tests { use { super::*, crate::{ + auction, byte_array::ByteArray, - events::{EventIndex, Settlement}, + events::{self, EventIndex, Settlement}, + settlements, }, sqlx::{Connection, Row}, }; @@ -584,4 +586,166 @@ mod tests { // inserted (2 fetched from "proposed_jit_orders" and 1 from "orders" table) assert!(fetched_solutions[2].orders.len() == 3); } + + #[tokio::test] + #[ignore] + async fn postgres_non_settling_solvers_roundtrip() { + let mut db = PgConnection::connect("postgresql://").await.unwrap(); + let mut db = db.begin().await.unwrap(); + crate::clear_DANGER_(&mut db).await.unwrap(); + + let non_settling_solver = ByteArray([1u8; 20]); + + let deadline_block = 100u64; + let last_auctions_count = 3i64; + // competition_auctions + // Insert auctions within the deadline + for auction_id in 1..=4 { + let auction = auction::Auction { + id: auction_id, + block: auction_id, + deadline: i64::try_from(deadline_block).unwrap(), + order_uids: Default::default(), + price_tokens: Default::default(), + price_values: Default::default(), + surplus_capturing_jit_order_owners: Default::default(), + }; + auction::save(&mut db, auction).await.unwrap(); + } + + // Insert auctions outside the deadline + for auction_id in 5..=6 { + let auction = auction::Auction { + id: auction_id, + block: auction_id, + deadline: i64::try_from(deadline_block).unwrap() + auction_id, + order_uids: Default::default(), + price_tokens: Default::default(), + price_values: Default::default(), + surplus_capturing_jit_order_owners: Default::default(), + }; + auction::save(&mut db, auction).await.unwrap(); + } + + // proposed_solutions + // Non-settling solver wins all auctions within the deadline + for auction_id in 2..=4 { + let solutions = vec![Solution { + uid: auction_id, + id: auction_id.into(), + solver: non_settling_solver, + is_winner: true, + score: Default::default(), + orders: Default::default(), + price_tokens: Default::default(), + price_values: Default::default(), + }]; + save_solutions(&mut db, auction_id, &solutions) + .await + .unwrap(); + } + + // Non-settling solver wins not all the auctions within the deadline + for auction_id in 2..=4 { + let solutions = vec![Solution { + uid: auction_id, + id: auction_id.into(), + solver: ByteArray([2u8; 20]), + is_winner: auction_id != 2, + score: Default::default(), + orders: Default::default(), + price_tokens: Default::default(), + price_values: Default::default(), + }]; + save_solutions(&mut db, auction_id, &solutions) + .await + .unwrap(); + } + + // Another non-settling solver has `last_auctions_count` winning auctions but + // not consecutive + for auction_id in 1..=4 { + // Break the sequence + if auction_id == 2 { + continue; + } + let solutions = vec![Solution { + uid: auction_id, + id: auction_id.into(), + solver: ByteArray([3u8; 20]), + is_winner: true, + score: Default::default(), + orders: Default::default(), + price_tokens: Default::default(), + price_values: Default::default(), + }]; + save_solutions(&mut db, auction_id, &solutions) + .await + .unwrap(); + } + + // One more non-settling solver has `last_auctions_count` winning auctions but + // some of them are outside the deadline + for auction_id in 3..=5 { + let solutions = vec![Solution { + uid: auction_id, + id: auction_id.into(), + solver: ByteArray([4u8; 20]), + is_winner: true, + score: Default::default(), + orders: Default::default(), + price_tokens: Default::default(), + price_values: Default::default(), + }]; + save_solutions(&mut db, auction_id, &solutions) + .await + .unwrap(); + } + + // Verify only the non-settling solver is returned + let result = find_non_settling_solvers( + &mut db, + u32::try_from(last_auctions_count).unwrap(), + deadline_block, + ) + .await + .unwrap(); + assert_eq!(result, vec![non_settling_solver]); + + // Non-settling solver settles one of the auctions + let event = EventIndex { + block_number: 4, + log_index: 0, + }; + let settlement = Settlement { + solver: non_settling_solver, + transaction_hash: ByteArray([0u8; 32]), + }; + events::insert_settlement(&mut db, &event, &settlement) + .await + .unwrap(); + + // The same result until the auction_id is updated in the settlements table + let result = find_non_settling_solvers( + &mut db, + u32::try_from(last_auctions_count).unwrap(), + deadline_block, + ) + .await + .unwrap(); + assert_eq!(result, vec![non_settling_solver]); + + settlements::update_settlement_auction(&mut db, 4, 0, 4) + .await + .unwrap(); + + let result = find_non_settling_solvers( + &mut db, + u32::try_from(last_auctions_count).unwrap(), + deadline_block, + ) + .await + .unwrap(); + assert!(result.is_empty()); + } } From e65c328fea34ad8cb59f8941db87222e9a509d6e Mon Sep 17 00:00:00 2001 From: ilya Date: Wed, 29 Jan 2025 13:30:34 +0000 Subject: [PATCH 03/82] Avoid rpc calls every time --- .../competition/solver_participation_guard.rs | 37 ++++++++++++------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/crates/autopilot/src/domain/competition/solver_participation_guard.rs b/crates/autopilot/src/domain/competition/solver_participation_guard.rs index 1ec1379098..1ee4252f99 100644 --- a/crates/autopilot/src/domain/competition/solver_participation_guard.rs +++ b/crates/autopilot/src/domain/competition/solver_participation_guard.rs @@ -1,7 +1,6 @@ use { crate::{database::Postgres, domain::eth, infra::Ethereum}, ethrpc::block_stream::CurrentBlockWatcher, - futures::future::try_join_all, std::{ sync::Arc, time::{Duration, Instant}, @@ -12,7 +11,8 @@ use { pub struct SolverParticipationGuard(Arc); struct Inner { - validators: Vec>, + onchain_solver_participation_validator: OnchainSolverParticipationValidator, + database_solver_participation_validator: DatabaseSolverParticipationValidator, } impl SolverParticipationGuard { @@ -34,22 +34,31 @@ impl SolverParticipationGuard { ); Self(Arc::new(Inner { - validators: vec![ - Box::new(database_solver_participation_validator), - Box::new(onchain_solver_participation_validator), - ], + onchain_solver_participation_validator, + database_solver_participation_validator, })) } pub async fn can_participate(&self, solver: ð::Address) -> anyhow::Result { - try_join_all( - self.0 - .validators - .iter() - .map(|strategy| strategy.can_participate(solver)), - ) - .await - .map(|results| results.into_iter().all(|can_participate| can_participate)) + if !self + .0 + .database_solver_participation_validator + .can_participate(solver) + .await? + { + return Ok(false); + } + + if !self + .0 + .onchain_solver_participation_validator + .can_participate(solver) + .await? + { + return Ok(false); + } + + Ok(true) } } From fc3321baf1cb4c7ed5054576cdc0d2085433e025 Mon Sep 17 00:00:00 2001 From: ilya Date: Wed, 29 Jan 2025 13:44:26 +0000 Subject: [PATCH 04/82] Typo --- crates/autopilot/src/database/competition.rs | 1 - .../src/domain/competition/solver_participation_guard.rs | 2 +- crates/driver/src/domain/competition/bad_tokens/metrics.rs | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/autopilot/src/database/competition.rs b/crates/autopilot/src/database/competition.rs index 2411e74f0f..88c9a86fca 100644 --- a/crates/autopilot/src/database/competition.rs +++ b/crates/autopilot/src/database/competition.rs @@ -146,7 +146,6 @@ impl super::Postgres { current_block: u64, ) -> anyhow::Result> { let mut ex = self.pool.acquire().await.context("acquire")?; - let _timer = super::Metrics::get() .database_queries .with_label_values(&["find_non_settling_solvers"]) diff --git a/crates/autopilot/src/domain/competition/solver_participation_guard.rs b/crates/autopilot/src/domain/competition/solver_participation_guard.rs index 1ee4252f99..8d8b961a58 100644 --- a/crates/autopilot/src/domain/competition/solver_participation_guard.rs +++ b/crates/autopilot/src/domain/competition/solver_participation_guard.rs @@ -117,7 +117,7 @@ impl DatabaseSolverParticipationValidator { .map(|solver| eth::Address(solver.0.into())) .collect::>(); - tracing::debug!(?non_settling_solvers, "found non-settling solvers",); + tracing::debug!(?non_settling_solvers, "found non-settling solvers"); let now = Instant::now(); for solver in non_settling_solvers { diff --git a/crates/driver/src/domain/competition/bad_tokens/metrics.rs b/crates/driver/src/domain/competition/bad_tokens/metrics.rs index daaad1ee7c..a0444169b9 100644 --- a/crates/driver/src/domain/competition/bad_tokens/metrics.rs +++ b/crates/driver/src/domain/competition/bad_tokens/metrics.rs @@ -110,7 +110,7 @@ impl Detector { flagged_unsupported_at: None, }); - // token neeeds to be frozen as unsupported for a while + // token needs to be frozen as unsupported for a while if self.quality_based_on_stats(&stats) == Quality::Unsupported && stats .flagged_unsupported_at From 0fbd61c05a638a0dc4b221c8300e7cfa8ba623b8 Mon Sep 17 00:00:00 2001 From: ilya Date: Wed, 29 Jan 2025 14:28:33 +0000 Subject: [PATCH 05/82] Docs --- crates/autopilot/src/database/competition.rs | 4 ++ .../competition/solver_participation_guard.rs | 61 ++++++++++--------- 2 files changed, 37 insertions(+), 28 deletions(-) diff --git a/crates/autopilot/src/database/competition.rs b/crates/autopilot/src/database/competition.rs index 88c9a86fca..682ac81d1d 100644 --- a/crates/autopilot/src/database/competition.rs +++ b/crates/autopilot/src/database/competition.rs @@ -140,6 +140,10 @@ impl super::Postgres { Ok(()) } + /// Finds solvers that won `last_auctions_count` consecutive auctions but + /// never settled any of them. The current block is used to prevent + /// selecting auctions with deadline after the current block since they + /// still can be settled. pub async fn find_non_settling_solvers( &self, last_auctions_count: u32, diff --git a/crates/autopilot/src/domain/competition/solver_participation_guard.rs b/crates/autopilot/src/domain/competition/solver_participation_guard.rs index 8d8b961a58..71eacf0e40 100644 --- a/crates/autopilot/src/domain/competition/solver_participation_guard.rs +++ b/crates/autopilot/src/domain/competition/solver_participation_guard.rs @@ -7,12 +7,14 @@ use { }, }; +/// This struct checks whether a solver can participate in the competition by +/// using different validators. #[derive(Clone)] pub struct SolverParticipationGuard(Arc); struct Inner { - onchain_solver_participation_validator: OnchainSolverParticipationValidator, - database_solver_participation_validator: DatabaseSolverParticipationValidator, + /// Stores the validators in order they will be called. + validators: Vec>, } impl SolverParticipationGuard { @@ -34,28 +36,25 @@ impl SolverParticipationGuard { ); Self(Arc::new(Inner { - onchain_solver_participation_validator, - database_solver_participation_validator, + validators: vec![ + Box::new(database_solver_participation_validator), + Box::new(onchain_solver_participation_validator), + ], })) } + /// Checks if a solver can participate in the competition. + /// Sequentially asks internal validators to avoid redundant RPC calls in + /// the following order: + /// 1. DatabaseSolverParticipationValidator - operates fast since it uses + /// in-memory cache. + /// 2. OnchainSolverParticipationValidator - only then calls the + /// Authenticator contract. pub async fn can_participate(&self, solver: ð::Address) -> anyhow::Result { - if !self - .0 - .database_solver_participation_validator - .can_participate(solver) - .await? - { - return Ok(false); - } - - if !self - .0 - .onchain_solver_participation_validator - .can_participate(solver) - .await? - { - return Ok(false); + for validator in &self.0.validators { + if !validator.is_allowed(solver).await? { + return Ok(false); + } } Ok(true) @@ -64,15 +63,17 @@ impl SolverParticipationGuard { #[async_trait::async_trait] trait SolverParticipationValidator: Send + Sync { - async fn can_participate(&self, solver: ð::Address) -> anyhow::Result; + async fn is_allowed(&self, solver: ð::Address) -> anyhow::Result; } +/// Checks the DB by searching for solvers that won N last consecutive auctions +/// but never settled any of them. #[derive(Clone)] pub struct DatabaseSolverParticipationValidator(Arc); struct DatabaseSolverParticipationValidatorInner { db: Postgres, - cache: dashmap::DashMap, + banned_solvers: dashmap::DashMap, ttl: Duration, last_auctions_count: u32, } @@ -87,7 +88,7 @@ impl DatabaseSolverParticipationValidator { ) -> Self { let self_ = Self(Arc::new(DatabaseSolverParticipationValidatorInner { db, - cache: Default::default(), + banned_solvers: Default::default(), ttl, last_auctions_count, })); @@ -97,6 +98,8 @@ impl DatabaseSolverParticipationValidator { self_ } + /// Update the internal cache only once the settlement table is updated to + /// avoid redundant DB queries. fn start_maintenance( &self, mut settlement_updates_receiver: tokio::sync::mpsc::UnboundedReceiver<()>, @@ -121,7 +124,7 @@ impl DatabaseSolverParticipationValidator { let now = Instant::now(); for solver in non_settling_solvers { - self_.cache.insert(solver, now); + self_.banned_solvers.insert(solver, now); } } Err(err) => { @@ -135,12 +138,12 @@ impl DatabaseSolverParticipationValidator { #[async_trait::async_trait] impl SolverParticipationValidator for DatabaseSolverParticipationValidator { - async fn can_participate(&self, solver: ð::Address) -> anyhow::Result { - if let Some(entry) = self.0.cache.get(solver) { + async fn is_allowed(&self, solver: ð::Address) -> anyhow::Result { + if let Some(entry) = self.0.banned_solvers.get(solver) { if Instant::now().duration_since(*entry.value()) < self.0.ttl { return Ok(false); } else { - self.0.cache.remove(solver); + self.0.banned_solvers.remove(solver); } } @@ -148,13 +151,15 @@ impl SolverParticipationValidator for DatabaseSolverParticipationValidator { } } +/// Calls Authenticator contract to check if a solver has a sufficient +/// permission. struct OnchainSolverParticipationValidator { eth: Ethereum, } #[async_trait::async_trait] impl SolverParticipationValidator for OnchainSolverParticipationValidator { - async fn can_participate(&self, solver: ð::Address) -> anyhow::Result { + async fn is_allowed(&self, solver: ð::Address) -> anyhow::Result { Ok(self .eth .contracts() From b1abfa0be8a1282ad008011c546c8029e695bfc1 Mon Sep 17 00:00:00 2001 From: ilya Date: Wed, 29 Jan 2025 16:57:08 +0000 Subject: [PATCH 06/82] Metrics --- .../competition/solver_participation_guard.rs | 16 ++++++++++++++-- crates/autopilot/src/domain/mod.rs | 15 +++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/crates/autopilot/src/domain/competition/solver_participation_guard.rs b/crates/autopilot/src/domain/competition/solver_participation_guard.rs index 71eacf0e40..75cd0e4b82 100644 --- a/crates/autopilot/src/domain/competition/solver_participation_guard.rs +++ b/crates/autopilot/src/domain/competition/solver_participation_guard.rs @@ -1,5 +1,9 @@ use { - crate::{database::Postgres, domain::eth, infra::Ethereum}, + crate::{ + database::Postgres, + domain::{eth, Metrics}, + infra::Ethereum, + }, ethrpc::block_stream::CurrentBlockWatcher, std::{ sync::Arc, @@ -117,7 +121,15 @@ impl DatabaseSolverParticipationValidator { Ok(non_settling_solvers) => { let non_settling_solvers = non_settling_solvers .into_iter() - .map(|solver| eth::Address(solver.0.into())) + .map(|solver| { + let address = eth::Address(solver.0.into()); + + Metrics::get() + .non_settling_solver + .with_label_values(&[&format!("{:#x}", address.0)]); + + address + }) .collect::>(); tracing::debug!(?non_settling_solvers, "found non-settling solvers"); diff --git a/crates/autopilot/src/domain/mod.rs b/crates/autopilot/src/domain/mod.rs index ceaae58e2a..ad7d0077af 100644 --- a/crates/autopilot/src/domain/mod.rs +++ b/crates/autopilot/src/domain/mod.rs @@ -14,3 +14,18 @@ pub use { fee::ProtocolFees, quote::Quote, }; + +#[derive(prometheus_metric_storage::MetricStorage)] +#[metric(subsystem = "domain")] +pub struct Metrics { + /// How many times the solver marked as non-settling based on the database + /// statistics. + #[metric(labels("solver"))] + pub non_settling_solver: prometheus::IntCounterVec, +} + +impl Metrics { + fn get() -> &'static Self { + Metrics::instance(observe::metrics::get_storage_registry()).unwrap() + } +} From 292dcffb9538b993bba7d16ffe49c0338ad42580 Mon Sep 17 00:00:00 2001 From: ilya Date: Wed, 29 Jan 2025 17:42:56 +0000 Subject: [PATCH 07/82] Configurable validators --- crates/autopilot/src/arguments.rs | 53 +++++++++++++++---- .../competition/solver_participation_guard.rs | 43 ++++++++------- crates/autopilot/src/run.rs | 19 +++---- 3 files changed, 77 insertions(+), 38 deletions(-) diff --git a/crates/autopilot/src/arguments.rs b/crates/autopilot/src/arguments.rs index ed5aa92657..2134bd3329 100644 --- a/crates/autopilot/src/arguments.rs +++ b/crates/autopilot/src/arguments.rs @@ -243,15 +243,52 @@ pub struct Arguments { #[clap(long, env)] pub archive_node_url: Option, - #[clap(long, env, default_value = "5m", value_parser = humantime::parse_duration)] + /// Configuration for the solver participation guard. + #[clap(flatten)] + pub solver_participation_guard: SolverParticipationGuardConfig, +} + +#[derive(Debug, clap::Parser)] +pub struct SolverParticipationGuardConfig { + #[clap(flatten)] + pub db_based_validator: DbBasedValidatorConfig, + + #[clap(flatten)] + pub onchain_based_validator: OnchainBasedValidatorConfig, +} + +#[derive(Debug, clap::Parser)] +pub struct DbBasedValidatorConfig { + /// Enables or disables the solver participation guard + #[clap( + long, + env, + name = "db_based_solver_participation_guard_enabled", + default_value = "true" + )] + pub enabled: bool, + /// The time-to-live for the solver participation blacklist cache. + #[clap(long, env, default_value = "5m", value_parser = humantime::parse_duration)] pub solver_blacklist_cache_ttl: Duration, - #[clap(long, env, default_value = "3")] /// The number of last auctions to check solver participation eligibility. + #[clap(long, env, default_value = "3")] pub solver_last_auctions_participation_count: u32, } +#[derive(Debug, Clone, clap::Parser)] +pub struct OnchainBasedValidatorConfig { + /// Enables or disables the solver participation guard + #[clap( + long, + env, + name = "onchain_based_solver_participation_guard_enabled", + default_value = "true" + )] + pub enabled: bool, +} + impl std::fmt::Display for Arguments { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let Self { @@ -295,8 +332,7 @@ impl std::fmt::Display for Arguments { max_winners_per_auction, archive_node_url, max_solutions_per_solver, - solver_blacklist_cache_ttl, - solver_last_auctions_participation_count, + solver_participation_guard, } = self; write!(f, "{}", shared)?; @@ -382,13 +418,8 @@ impl std::fmt::Display for Arguments { )?; writeln!( f, - "solver_blacklist_cache_ttl: {:?}", - solver_blacklist_cache_ttl - )?; - writeln!( - f, - "solver_last_auctions_participation_count: {:?}", - solver_last_auctions_participation_count + "solver_participation_guard: {:?}", + solver_participation_guard )?; Ok(()) } diff --git a/crates/autopilot/src/domain/competition/solver_participation_guard.rs b/crates/autopilot/src/domain/competition/solver_participation_guard.rs index 75cd0e4b82..f02029370d 100644 --- a/crates/autopilot/src/domain/competition/solver_participation_guard.rs +++ b/crates/autopilot/src/domain/competition/solver_participation_guard.rs @@ -1,5 +1,6 @@ use { crate::{ + arguments::SolverParticipationGuardConfig, database::Postgres, domain::{eth, Metrics}, infra::Ethereum, @@ -26,25 +27,31 @@ impl SolverParticipationGuard { eth: Ethereum, db: Postgres, settlement_updates_receiver: tokio::sync::mpsc::UnboundedReceiver<()>, - ttl: Duration, - last_auctions_count: u32, + config: SolverParticipationGuardConfig, ) -> Self { - let current_block = eth.current_block().clone(); - let onchain_solver_participation_validator = OnchainSolverParticipationValidator { eth }; - let database_solver_participation_validator = DatabaseSolverParticipationValidator::new( - db, - current_block, - settlement_updates_receiver, - ttl, - last_auctions_count, - ); - - Self(Arc::new(Inner { - validators: vec![ - Box::new(database_solver_participation_validator), - Box::new(onchain_solver_participation_validator), - ], - })) + let mut validators: Vec> = Vec::new(); + + if config.db_based_validator.enabled { + let current_block = eth.current_block().clone(); + let database_solver_participation_validator = DatabaseSolverParticipationValidator::new( + db, + current_block, + settlement_updates_receiver, + config.db_based_validator.solver_blacklist_cache_ttl, + config + .db_based_validator + .solver_last_auctions_participation_count, + ); + validators.push(Box::new(database_solver_participation_validator)); + } + + if config.onchain_based_validator.enabled { + let onchain_solver_participation_validator = + OnchainSolverParticipationValidator { eth }; + validators.push(Box::new(onchain_solver_participation_validator)); + } + + Self(Arc::new(Inner { validators })) } /// Checks if a solver can participate in the competition. diff --git a/crates/autopilot/src/run.rs b/crates/autopilot/src/run.rs index ede9f38045..cf538343c8 100644 --- a/crates/autopilot/src/run.rs +++ b/crates/autopilot/src/run.rs @@ -365,21 +365,22 @@ pub async fn run(args: Arguments) { None }; - let persistence = - infra::persistence::Persistence::new(args.s3.into().unwrap(), Arc::new(db.clone())).await; let (settlement_updates_sender, settlement_updates_receiver) = tokio::sync::mpsc::unbounded_channel(); - let settlement_observer = crate::domain::settlement::Observer::new( - eth.clone(), - persistence.clone(), - settlement_updates_sender, - ); + let solver_participation_guard = SolverParticipationGuard::new( eth.clone(), db.clone(), settlement_updates_receiver, - args.solver_blacklist_cache_ttl, - args.solver_last_auctions_participation_count, + args.solver_participation_guard, + ); + + let persistence = + infra::persistence::Persistence::new(args.s3.into().unwrap(), Arc::new(db.clone())).await; + let settlement_observer = crate::domain::settlement::Observer::new( + eth.clone(), + persistence.clone(), + settlement_updates_sender, ); let settlement_contract_start_index = if let Some(DeploymentInformation::BlockNumber(settlement_contract_start_index)) = From fe9ef5b506e2b13074937b89f141d9d043f40b85 Mon Sep 17 00:00:00 2001 From: ilya Date: Wed, 29 Jan 2025 17:58:13 +0000 Subject: [PATCH 08/82] Fixed clap config --- crates/autopilot/src/arguments.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/autopilot/src/arguments.rs b/crates/autopilot/src/arguments.rs index 2134bd3329..e0fcb13489 100644 --- a/crates/autopilot/src/arguments.rs +++ b/crates/autopilot/src/arguments.rs @@ -261,9 +261,9 @@ pub struct SolverParticipationGuardConfig { pub struct DbBasedValidatorConfig { /// Enables or disables the solver participation guard #[clap( - long, - env, - name = "db_based_solver_participation_guard_enabled", + id = "db_enabled", + long = "db-based-solver-participation-guard-enabled", + env = "DB_BASED_SOLVER_PARTICIPATION_GUARD_ENABLED", default_value = "true" )] pub enabled: bool, @@ -281,9 +281,9 @@ pub struct DbBasedValidatorConfig { pub struct OnchainBasedValidatorConfig { /// Enables or disables the solver participation guard #[clap( - long, - env, - name = "onchain_based_solver_participation_guard_enabled", + id = "onchain_enabled", + long = "onchain-based-solver-participation-guard-enabled", + env = "ONCHAIN_BASED_SOLVER_PARTICIPATION_GUARD_ENABLED", default_value = "true" )] pub enabled: bool, From c5e350235e3455e55357ed34a907801f800e8467 Mon Sep 17 00:00:00 2001 From: ilya Date: Thu, 30 Jan 2025 12:36:57 +0000 Subject: [PATCH 09/82] Refactoring --- .../autopilot/src/domain/competition/mod.rs | 4 +- .../competition/participation_guard/db.rs | 104 ++++++++++ .../competition/participation_guard/mod.rs | 74 +++++++ .../participation_guard/onchain.rs | 20 ++ .../competition/solver_participation_guard.rs | 190 ------------------ 5 files changed, 200 insertions(+), 192 deletions(-) create mode 100644 crates/autopilot/src/domain/competition/participation_guard/db.rs create mode 100644 crates/autopilot/src/domain/competition/participation_guard/mod.rs create mode 100644 crates/autopilot/src/domain/competition/participation_guard/onchain.rs delete mode 100644 crates/autopilot/src/domain/competition/solver_participation_guard.rs diff --git a/crates/autopilot/src/domain/competition/mod.rs b/crates/autopilot/src/domain/competition/mod.rs index bcd5210536..7ef2d6d06f 100644 --- a/crates/autopilot/src/domain/competition/mod.rs +++ b/crates/autopilot/src/domain/competition/mod.rs @@ -6,11 +6,11 @@ use { }; mod participant; -mod solver_participation_guard; +mod participation_guard; pub use { participant::{Participant, Ranked, Unranked}, - solver_participation_guard::{DatabaseSolverParticipationValidator, SolverParticipationGuard}, + participation_guard::SolverParticipationGuard, }; type SolutionId = u64; diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs new file mode 100644 index 0000000000..4eb3ce7c17 --- /dev/null +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -0,0 +1,104 @@ +use { + crate::{ + database::Postgres, + domain::{eth, Metrics}, + }, + ethrpc::block_stream::CurrentBlockWatcher, + std::{ + sync::Arc, + time::{Duration, Instant}, + }, +}; + +/// Checks the DB by searching for solvers that won N last consecutive auctions +/// but never settled any of them. +#[derive(Clone)] +pub(super) struct Validator(Arc); + +struct Inner { + db: Postgres, + banned_solvers: dashmap::DashMap, + ttl: Duration, + last_auctions_count: u32, +} + +impl Validator { + pub fn new( + db: Postgres, + current_block: CurrentBlockWatcher, + settlement_updates_receiver: tokio::sync::mpsc::UnboundedReceiver<()>, + ttl: Duration, + last_auctions_count: u32, + ) -> Self { + let self_ = Self(Arc::new(Inner { + db, + banned_solvers: Default::default(), + ttl, + last_auctions_count, + })); + + self_.start_maintenance(settlement_updates_receiver, current_block); + + self_ + } + + /// Update the internal cache only once the settlement table is updated to + /// avoid redundant DB queries. + fn start_maintenance( + &self, + mut settlement_updates_receiver: tokio::sync::mpsc::UnboundedReceiver<()>, + current_block: CurrentBlockWatcher, + ) { + let self_ = self.0.clone(); + tokio::spawn(async move { + while settlement_updates_receiver.recv().await.is_some() { + let current_block = current_block.borrow().number; + match self_ + .db + .find_non_settling_solvers(self_.last_auctions_count, current_block) + .await + { + Ok(non_settling_solvers) => { + let non_settling_solvers = non_settling_solvers + .into_iter() + .map(|solver| { + let address = eth::Address(solver.0.into()); + + Metrics::get() + .non_settling_solver + .with_label_values(&[&format!("{:#x}", address.0)]); + + address + }) + .collect::>(); + + tracing::debug!(?non_settling_solvers, "found non-settling solvers"); + + let now = Instant::now(); + for solver in non_settling_solvers { + self_.banned_solvers.insert(solver, now); + } + } + Err(err) => { + tracing::warn!(?err, "error while searching for non-settling solvers") + } + } + } + }); + } +} + +#[async_trait::async_trait] +impl super::Validator for Validator { + async fn is_allowed(&self, solver: ð::Address) -> anyhow::Result { + if let Some(entry) = self.0.banned_solvers.get(solver) { + if Instant::now().duration_since(*entry.value()) < self.0.ttl { + return Ok(false); + } else { + self.0.banned_solvers.remove(solver); + } + } + + Ok(true) + } +} diff --git a/crates/autopilot/src/domain/competition/participation_guard/mod.rs b/crates/autopilot/src/domain/competition/participation_guard/mod.rs new file mode 100644 index 0000000000..7450dc6126 --- /dev/null +++ b/crates/autopilot/src/domain/competition/participation_guard/mod.rs @@ -0,0 +1,74 @@ +mod db; +mod onchain; + +use { + crate::{ + arguments::SolverParticipationGuardConfig, + database::Postgres, + domain::eth, + infra::Ethereum, + }, + std::sync::Arc, +}; + +/// This struct checks whether a solver can participate in the competition by +/// using different validators. +#[derive(Clone)] +pub struct SolverParticipationGuard(Arc); + +struct Inner { + /// Stores the validators in order they will be called. + validators: Vec>, +} + +impl SolverParticipationGuard { + pub fn new( + eth: Ethereum, + db: Postgres, + settlement_updates_receiver: tokio::sync::mpsc::UnboundedReceiver<()>, + config: SolverParticipationGuardConfig, + ) -> Self { + let mut validators: Vec> = Vec::new(); + + if config.db_based_validator.enabled { + let current_block = eth.current_block().clone(); + let database_solver_participation_validator = db::Validator::new( + db, + current_block, + settlement_updates_receiver, + config.db_based_validator.solver_blacklist_cache_ttl, + config + .db_based_validator + .solver_last_auctions_participation_count, + ); + validators.push(Box::new(database_solver_participation_validator)); + } + + if config.onchain_based_validator.enabled { + let onchain_solver_participation_validator = onchain::Validator { eth }; + validators.push(Box::new(onchain_solver_participation_validator)); + } + + Self(Arc::new(Inner { validators })) + } + + /// Checks if a solver can participate in the competition. + /// Sequentially asks internal validators to avoid redundant RPC calls in + /// the following order: + /// 1. DB-based validator: operates fast since it uses in-memory cache. + /// 2. Onchain-based validator: only then calls the Authenticator contract. + pub async fn can_participate(&self, solver: ð::Address) -> anyhow::Result { + for validator in &self.0.validators { + if !validator.is_allowed(solver).await? { + return Ok(false); + } + } + + Ok(true) + } +} + +#[async_trait::async_trait] +trait Validator: Send + Sync { + async fn is_allowed(&self, solver: ð::Address) -> anyhow::Result; +} diff --git a/crates/autopilot/src/domain/competition/participation_guard/onchain.rs b/crates/autopilot/src/domain/competition/participation_guard/onchain.rs new file mode 100644 index 0000000000..82d0ef3fb7 --- /dev/null +++ b/crates/autopilot/src/domain/competition/participation_guard/onchain.rs @@ -0,0 +1,20 @@ +use crate::{domain::eth, infra::Ethereum}; + +/// Calls Authenticator contract to check if a solver has a sufficient +/// permission. +pub(super) struct Validator { + pub eth: Ethereum, +} + +#[async_trait::async_trait] +impl super::Validator for Validator { + async fn is_allowed(&self, solver: ð::Address) -> anyhow::Result { + Ok(self + .eth + .contracts() + .authenticator() + .is_solver(solver.0) + .call() + .await?) + } +} diff --git a/crates/autopilot/src/domain/competition/solver_participation_guard.rs b/crates/autopilot/src/domain/competition/solver_participation_guard.rs deleted file mode 100644 index f02029370d..0000000000 --- a/crates/autopilot/src/domain/competition/solver_participation_guard.rs +++ /dev/null @@ -1,190 +0,0 @@ -use { - crate::{ - arguments::SolverParticipationGuardConfig, - database::Postgres, - domain::{eth, Metrics}, - infra::Ethereum, - }, - ethrpc::block_stream::CurrentBlockWatcher, - std::{ - sync::Arc, - time::{Duration, Instant}, - }, -}; - -/// This struct checks whether a solver can participate in the competition by -/// using different validators. -#[derive(Clone)] -pub struct SolverParticipationGuard(Arc); - -struct Inner { - /// Stores the validators in order they will be called. - validators: Vec>, -} - -impl SolverParticipationGuard { - pub fn new( - eth: Ethereum, - db: Postgres, - settlement_updates_receiver: tokio::sync::mpsc::UnboundedReceiver<()>, - config: SolverParticipationGuardConfig, - ) -> Self { - let mut validators: Vec> = Vec::new(); - - if config.db_based_validator.enabled { - let current_block = eth.current_block().clone(); - let database_solver_participation_validator = DatabaseSolverParticipationValidator::new( - db, - current_block, - settlement_updates_receiver, - config.db_based_validator.solver_blacklist_cache_ttl, - config - .db_based_validator - .solver_last_auctions_participation_count, - ); - validators.push(Box::new(database_solver_participation_validator)); - } - - if config.onchain_based_validator.enabled { - let onchain_solver_participation_validator = - OnchainSolverParticipationValidator { eth }; - validators.push(Box::new(onchain_solver_participation_validator)); - } - - Self(Arc::new(Inner { validators })) - } - - /// Checks if a solver can participate in the competition. - /// Sequentially asks internal validators to avoid redundant RPC calls in - /// the following order: - /// 1. DatabaseSolverParticipationValidator - operates fast since it uses - /// in-memory cache. - /// 2. OnchainSolverParticipationValidator - only then calls the - /// Authenticator contract. - pub async fn can_participate(&self, solver: ð::Address) -> anyhow::Result { - for validator in &self.0.validators { - if !validator.is_allowed(solver).await? { - return Ok(false); - } - } - - Ok(true) - } -} - -#[async_trait::async_trait] -trait SolverParticipationValidator: Send + Sync { - async fn is_allowed(&self, solver: ð::Address) -> anyhow::Result; -} - -/// Checks the DB by searching for solvers that won N last consecutive auctions -/// but never settled any of them. -#[derive(Clone)] -pub struct DatabaseSolverParticipationValidator(Arc); - -struct DatabaseSolverParticipationValidatorInner { - db: Postgres, - banned_solvers: dashmap::DashMap, - ttl: Duration, - last_auctions_count: u32, -} - -impl DatabaseSolverParticipationValidator { - pub fn new( - db: Postgres, - current_block: CurrentBlockWatcher, - settlement_updates_receiver: tokio::sync::mpsc::UnboundedReceiver<()>, - ttl: Duration, - last_auctions_count: u32, - ) -> Self { - let self_ = Self(Arc::new(DatabaseSolverParticipationValidatorInner { - db, - banned_solvers: Default::default(), - ttl, - last_auctions_count, - })); - - self_.start_maintenance(settlement_updates_receiver, current_block); - - self_ - } - - /// Update the internal cache only once the settlement table is updated to - /// avoid redundant DB queries. - fn start_maintenance( - &self, - mut settlement_updates_receiver: tokio::sync::mpsc::UnboundedReceiver<()>, - current_block: CurrentBlockWatcher, - ) { - let self_ = self.0.clone(); - tokio::spawn(async move { - while settlement_updates_receiver.recv().await.is_some() { - let current_block = current_block.borrow().number; - match self_ - .db - .find_non_settling_solvers(self_.last_auctions_count, current_block) - .await - { - Ok(non_settling_solvers) => { - let non_settling_solvers = non_settling_solvers - .into_iter() - .map(|solver| { - let address = eth::Address(solver.0.into()); - - Metrics::get() - .non_settling_solver - .with_label_values(&[&format!("{:#x}", address.0)]); - - address - }) - .collect::>(); - - tracing::debug!(?non_settling_solvers, "found non-settling solvers"); - - let now = Instant::now(); - for solver in non_settling_solvers { - self_.banned_solvers.insert(solver, now); - } - } - Err(err) => { - tracing::warn!(?err, "error while searching for non-settling solvers") - } - } - } - }); - } -} - -#[async_trait::async_trait] -impl SolverParticipationValidator for DatabaseSolverParticipationValidator { - async fn is_allowed(&self, solver: ð::Address) -> anyhow::Result { - if let Some(entry) = self.0.banned_solvers.get(solver) { - if Instant::now().duration_since(*entry.value()) < self.0.ttl { - return Ok(false); - } else { - self.0.banned_solvers.remove(solver); - } - } - - Ok(true) - } -} - -/// Calls Authenticator contract to check if a solver has a sufficient -/// permission. -struct OnchainSolverParticipationValidator { - eth: Ethereum, -} - -#[async_trait::async_trait] -impl SolverParticipationValidator for OnchainSolverParticipationValidator { - async fn is_allowed(&self, solver: ð::Address) -> anyhow::Result { - Ok(self - .eth - .contracts() - .authenticator() - .is_solver(solver.0) - .call() - .await?) - } -} From a9e6a3fe9311c524ce85672480e14a8fff67ff10 Mon Sep 17 00:00:00 2001 From: ilya Date: Thu, 30 Jan 2025 16:07:06 +0000 Subject: [PATCH 10/82] Config per solver --- crates/autopilot/src/arguments.rs | 92 +++++++++++++------ .../competition/participation_guard/mod.rs | 18 ++-- crates/autopilot/src/run.rs | 2 +- 3 files changed, 70 insertions(+), 42 deletions(-) diff --git a/crates/autopilot/src/arguments.rs b/crates/autopilot/src/arguments.rs index e0fcb13489..e70b5ba0a3 100644 --- a/crates/autopilot/src/arguments.rs +++ b/crates/autopilot/src/arguments.rs @@ -245,20 +245,11 @@ pub struct Arguments { /// Configuration for the solver participation guard. #[clap(flatten)] - pub solver_participation_guard: SolverParticipationGuardConfig, + pub db_based_solver_participation_guard: DbBasedSolverParticipationGuardConfig, } #[derive(Debug, clap::Parser)] -pub struct SolverParticipationGuardConfig { - #[clap(flatten)] - pub db_based_validator: DbBasedValidatorConfig, - - #[clap(flatten)] - pub onchain_based_validator: OnchainBasedValidatorConfig, -} - -#[derive(Debug, clap::Parser)] -pub struct DbBasedValidatorConfig { +pub struct DbBasedSolverParticipationGuardConfig { /// Enables or disables the solver participation guard #[clap( id = "db_enabled", @@ -277,18 +268,6 @@ pub struct DbBasedValidatorConfig { pub solver_last_auctions_participation_count: u32, } -#[derive(Debug, Clone, clap::Parser)] -pub struct OnchainBasedValidatorConfig { - /// Enables or disables the solver participation guard - #[clap( - id = "onchain_enabled", - long = "onchain-based-solver-participation-guard-enabled", - env = "ONCHAIN_BASED_SOLVER_PARTICIPATION_GUARD_ENABLED", - default_value = "true" - )] - pub enabled: bool, -} - impl std::fmt::Display for Arguments { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let Self { @@ -332,7 +311,7 @@ impl std::fmt::Display for Arguments { max_winners_per_auction, archive_node_url, max_solutions_per_solver, - solver_participation_guard, + db_based_solver_participation_guard, } = self; write!(f, "{}", shared)?; @@ -418,8 +397,8 @@ impl std::fmt::Display for Arguments { )?; writeln!( f, - "solver_participation_guard: {:?}", - solver_participation_guard + "db_based_solver_participation_guard: {:?}", + db_based_solver_participation_guard )?; Ok(()) } @@ -432,6 +411,7 @@ pub struct Solver { pub url: Url, pub submission_account: Account, pub fairness_threshold: Option, + pub accepts_unsettled_blocking: bool, } #[derive(Debug, Clone, PartialEq, Eq, Hash)] @@ -480,18 +460,34 @@ impl FromStr for Solver { Account::Address(H160::from_str(parts[2]).context("failed to parse submission")?) }; - let fairness_threshold = match parts.get(3) { - Some(value) => { - Some(U256::from_dec_str(value).context("failed to parse fairness threshold")?) + let mut fairness_threshold: Option = Default::default(); + let mut accepts_unsettled_blocking = false; + + if let Some(value) = parts.get(3) { + match U256::from_dec_str(value) { + Ok(parsed_fairness_threshold) => { + fairness_threshold = Some(parsed_fairness_threshold); + } + Err(_) => { + accepts_unsettled_blocking = value + .parse() + .context("failed to parse solver's third arg param")? + } } - None => None, }; + if let Some(value) = parts.get(4) { + accepts_unsettled_blocking = value + .parse() + .context("failed to parse `accepts_unsettled_blocking` flag")?; + } + Ok(Self { name: name.to_owned(), url, fairness_threshold, submission_account, + accepts_unsettled_blocking, }) } } @@ -688,6 +684,7 @@ mod test { name: "name1".into(), url: Url::parse("http://localhost:8080").unwrap(), fairness_threshold: None, + accepts_unsettled_blocking: false, submission_account: Account::Address(H160::from_slice(&hex!( "C02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2" ))), @@ -703,6 +700,7 @@ mod test { name: "name1".into(), url: Url::parse("http://localhost:8080").unwrap(), fairness_threshold: None, + accepts_unsettled_blocking: false, submission_account: Account::Kms( Arn::from_str("arn:aws:kms:supersecretstuff").unwrap(), ), @@ -721,6 +719,40 @@ mod test { "C02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2" ))), fairness_threshold: Some(U256::exp10(18)), + accepts_unsettled_blocking: false, + }; + assert_eq!(driver, expected); + } + + #[test] + fn parse_driver_with_accepts_unsettled_blocking_flag() { + let argument = + "name1|http://localhost:8080|0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2|true"; + let driver = Solver::from_str(argument).unwrap(); + let expected = Solver { + name: "name1".into(), + url: Url::parse("http://localhost:8080").unwrap(), + submission_account: Account::Address(H160::from_slice(&hex!( + "C02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2" + ))), + fairness_threshold: None, + accepts_unsettled_blocking: true, + }; + assert_eq!(driver, expected); + } + + #[test] + fn parse_driver_with_threshold_and_accepts_unsettled_blocking_flag() { + let argument = "name1|http://localhost:8080|0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2|1000000000000000000|true"; + let driver = Solver::from_str(argument).unwrap(); + let expected = Solver { + name: "name1".into(), + url: Url::parse("http://localhost:8080").unwrap(), + submission_account: Account::Address(H160::from_slice(&hex!( + "C02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2" + ))), + fairness_threshold: Some(U256::exp10(18)), + accepts_unsettled_blocking: true, }; assert_eq!(driver, expected); } diff --git a/crates/autopilot/src/domain/competition/participation_guard/mod.rs b/crates/autopilot/src/domain/competition/participation_guard/mod.rs index 7450dc6126..07f08f210e 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/mod.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/mod.rs @@ -3,7 +3,7 @@ mod onchain; use { crate::{ - arguments::SolverParticipationGuardConfig, + arguments::DbBasedSolverParticipationGuardConfig, database::Postgres, domain::eth, infra::Ethereum, @@ -26,28 +26,24 @@ impl SolverParticipationGuard { eth: Ethereum, db: Postgres, settlement_updates_receiver: tokio::sync::mpsc::UnboundedReceiver<()>, - config: SolverParticipationGuardConfig, + db_based_validator_config: DbBasedSolverParticipationGuardConfig, ) -> Self { let mut validators: Vec> = Vec::new(); - if config.db_based_validator.enabled { + if db_based_validator_config.enabled { let current_block = eth.current_block().clone(); let database_solver_participation_validator = db::Validator::new( db, current_block, settlement_updates_receiver, - config.db_based_validator.solver_blacklist_cache_ttl, - config - .db_based_validator - .solver_last_auctions_participation_count, + db_based_validator_config.solver_blacklist_cache_ttl, + db_based_validator_config.solver_last_auctions_participation_count, ); validators.push(Box::new(database_solver_participation_validator)); } - if config.onchain_based_validator.enabled { - let onchain_solver_participation_validator = onchain::Validator { eth }; - validators.push(Box::new(onchain_solver_participation_validator)); - } + let onchain_solver_participation_validator = onchain::Validator { eth }; + validators.push(Box::new(onchain_solver_participation_validator)); Self(Arc::new(Inner { validators })) } diff --git a/crates/autopilot/src/run.rs b/crates/autopilot/src/run.rs index cf538343c8..e3a08c5400 100644 --- a/crates/autopilot/src/run.rs +++ b/crates/autopilot/src/run.rs @@ -372,7 +372,7 @@ pub async fn run(args: Arguments) { eth.clone(), db.clone(), settlement_updates_receiver, - args.solver_participation_guard, + args.db_based_solver_participation_guard, ); let persistence = From 9a55fe2fb2ab6c6ac84dafd2197448a1c0e1236d Mon Sep 17 00:00:00 2001 From: ilya Date: Thu, 30 Jan 2025 16:29:03 +0000 Subject: [PATCH 11/82] Start using the new config --- .../competition/participation_guard/db.rs | 16 +++++++++++++ .../competition/participation_guard/mod.rs | 4 +++- crates/autopilot/src/infra/solvers/mod.rs | 3 +++ crates/autopilot/src/run.rs | 24 ++++++++++++------- 4 files changed, 38 insertions(+), 9 deletions(-) diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index 4eb3ce7c17..d37849c0aa 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -5,6 +5,7 @@ use { }, ethrpc::block_stream::CurrentBlockWatcher, std::{ + collections::HashMap, sync::Arc, time::{Duration, Instant}, }, @@ -20,6 +21,7 @@ struct Inner { banned_solvers: dashmap::DashMap, ttl: Duration, last_auctions_count: u32, + db_validator_acceptance_by_solver: HashMap, } impl Validator { @@ -29,12 +31,14 @@ impl Validator { settlement_updates_receiver: tokio::sync::mpsc::UnboundedReceiver<()>, ttl: Duration, last_auctions_count: u32, + db_validator_acceptance_by_solver: HashMap, ) -> Self { let self_ = Self(Arc::new(Inner { db, banned_solvers: Default::default(), ttl, last_auctions_count, + db_validator_acceptance_by_solver, })); self_.start_maintenance(settlement_updates_receiver, current_block); @@ -91,6 +95,18 @@ impl Validator { #[async_trait::async_trait] impl super::Validator for Validator { async fn is_allowed(&self, solver: ð::Address) -> anyhow::Result { + // Check if solver accepted this feature. This should be removed once a CIP is + // approved. + if !self + .0 + .db_validator_acceptance_by_solver + .get(solver) + .copied() + .unwrap_or_default() + { + return Ok(true); + } + if let Some(entry) = self.0.banned_solvers.get(solver) { if Instant::now().duration_since(*entry.value()) < self.0.ttl { return Ok(false); diff --git a/crates/autopilot/src/domain/competition/participation_guard/mod.rs b/crates/autopilot/src/domain/competition/participation_guard/mod.rs index 07f08f210e..f4fb14a9a5 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/mod.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/mod.rs @@ -8,7 +8,7 @@ use { domain::eth, infra::Ethereum, }, - std::sync::Arc, + std::{collections::HashMap, sync::Arc}, }; /// This struct checks whether a solver can participate in the competition by @@ -27,6 +27,7 @@ impl SolverParticipationGuard { db: Postgres, settlement_updates_receiver: tokio::sync::mpsc::UnboundedReceiver<()>, db_based_validator_config: DbBasedSolverParticipationGuardConfig, + db_validator_acceptance_by_solver: HashMap, ) -> Self { let mut validators: Vec> = Vec::new(); @@ -38,6 +39,7 @@ impl SolverParticipationGuard { settlement_updates_receiver, db_based_validator_config.solver_blacklist_cache_ttl, db_based_validator_config.solver_last_auctions_participation_count, + db_validator_acceptance_by_solver, ); validators.push(Box::new(database_solver_participation_validator)); } diff --git a/crates/autopilot/src/infra/solvers/mod.rs b/crates/autopilot/src/infra/solvers/mod.rs index b84dc79654..aae623ebaa 100644 --- a/crates/autopilot/src/infra/solvers/mod.rs +++ b/crates/autopilot/src/infra/solvers/mod.rs @@ -21,6 +21,7 @@ pub struct Driver { // another driver solved with surplus exceeding this driver's surplus by `threshold` pub fairness_threshold: Option, pub submission_address: eth::Address, + pub accepts_unsettled_blocking: bool, client: Client, } @@ -38,6 +39,7 @@ impl Driver { name: String, fairness_threshold: Option, submission_account: Account, + accepts_unsettled_blocking: bool, ) -> Result { let submission_address = match submission_account { Account::Kms(key_id) => { @@ -70,6 +72,7 @@ impl Driver { .build() .map_err(Error::FailedToBuildClient)?, submission_address: submission_address.into(), + accepts_unsettled_blocking, }) } diff --git a/crates/autopilot/src/run.rs b/crates/autopilot/src/run.rs index e3a08c5400..cfa031f81a 100644 --- a/crates/autopilot/src/run.rs +++ b/crates/autopilot/src/run.rs @@ -52,6 +52,7 @@ use { token_list::{AutoUpdatingTokenList, TokenListConfiguration}, }, std::{ + collections::HashMap, sync::{Arc, RwLock}, time::{Duration, Instant}, }, @@ -368,13 +369,6 @@ pub async fn run(args: Arguments) { let (settlement_updates_sender, settlement_updates_receiver) = tokio::sync::mpsc::unbounded_channel(); - let solver_participation_guard = SolverParticipationGuard::new( - eth.clone(), - db.clone(), - settlement_updates_receiver, - args.db_based_solver_participation_guard, - ); - let persistence = infra::persistence::Persistence::new(args.s3.into().unwrap(), Arc::new(db.clone())).await; let settlement_observer = crate::domain::settlement::Observer::new( @@ -561,6 +555,7 @@ pub async fn run(args: Arguments) { max_winners_per_auction: args.max_winners_per_auction, max_solutions_per_solver: args.max_solutions_per_solver, }; + let drivers_futures = args .drivers .into_iter() @@ -570,6 +565,7 @@ pub async fn run(args: Arguments) { driver.name.clone(), driver.fairness_threshold.map(Into::into), driver.submission_account, + driver.accepts_unsettled_blocking, ) .await .map(Arc::new) @@ -577,11 +573,22 @@ pub async fn run(args: Arguments) { }) .collect::>(); - let drivers = futures::future::join_all(drivers_futures) + let drivers: Vec<_> = futures::future::join_all(drivers_futures) .await .into_iter() .collect(); + let solver_participation_guard = SolverParticipationGuard::new( + eth.clone(), + db.clone(), + settlement_updates_receiver, + args.db_based_solver_participation_guard, + drivers + .iter() + .map(|driver| (driver.submission_address, driver.accepts_unsettled_blocking)) + .collect::>(), + ); + let run = RunLoop::new( run_loop_config, eth, @@ -613,6 +620,7 @@ async fn shadow_mode(args: Arguments) -> ! { driver.name.clone(), driver.fairness_threshold.map(Into::into), driver.submission_account, + driver.accepts_unsettled_blocking, ) .await .map(Arc::new) From f9bdafdba9dc78295b171d3ee40e73bc8ad134a6 Mon Sep 17 00:00:00 2001 From: ilya Date: Thu, 30 Jan 2025 16:32:53 +0000 Subject: [PATCH 12/82] Simplify to hashset --- .../domain/competition/participation_guard/db.rs | 16 +++++----------- .../competition/participation_guard/mod.rs | 6 +++--- crates/autopilot/src/run.rs | 8 +++++--- 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index d37849c0aa..f765b39418 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -5,7 +5,7 @@ use { }, ethrpc::block_stream::CurrentBlockWatcher, std::{ - collections::HashMap, + collections::HashSet, sync::Arc, time::{Duration, Instant}, }, @@ -21,7 +21,7 @@ struct Inner { banned_solvers: dashmap::DashMap, ttl: Duration, last_auctions_count: u32, - db_validator_acceptance_by_solver: HashMap, + db_validator_accepted_solvers: HashSet, } impl Validator { @@ -31,14 +31,14 @@ impl Validator { settlement_updates_receiver: tokio::sync::mpsc::UnboundedReceiver<()>, ttl: Duration, last_auctions_count: u32, - db_validator_acceptance_by_solver: HashMap, + db_validator_accepted_solvers: HashSet, ) -> Self { let self_ = Self(Arc::new(Inner { db, banned_solvers: Default::default(), ttl, last_auctions_count, - db_validator_acceptance_by_solver, + db_validator_accepted_solvers, })); self_.start_maintenance(settlement_updates_receiver, current_block); @@ -97,13 +97,7 @@ impl super::Validator for Validator { async fn is_allowed(&self, solver: ð::Address) -> anyhow::Result { // Check if solver accepted this feature. This should be removed once a CIP is // approved. - if !self - .0 - .db_validator_acceptance_by_solver - .get(solver) - .copied() - .unwrap_or_default() - { + if !self.0.db_validator_accepted_solvers.contains(solver) { return Ok(true); } diff --git a/crates/autopilot/src/domain/competition/participation_guard/mod.rs b/crates/autopilot/src/domain/competition/participation_guard/mod.rs index f4fb14a9a5..d0127e68d5 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/mod.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/mod.rs @@ -8,7 +8,7 @@ use { domain::eth, infra::Ethereum, }, - std::{collections::HashMap, sync::Arc}, + std::{collections::HashSet, sync::Arc}, }; /// This struct checks whether a solver can participate in the competition by @@ -27,7 +27,7 @@ impl SolverParticipationGuard { db: Postgres, settlement_updates_receiver: tokio::sync::mpsc::UnboundedReceiver<()>, db_based_validator_config: DbBasedSolverParticipationGuardConfig, - db_validator_acceptance_by_solver: HashMap, + db_validator_accepted_solvers: HashSet, ) -> Self { let mut validators: Vec> = Vec::new(); @@ -39,7 +39,7 @@ impl SolverParticipationGuard { settlement_updates_receiver, db_based_validator_config.solver_blacklist_cache_ttl, db_based_validator_config.solver_last_auctions_participation_count, - db_validator_acceptance_by_solver, + db_validator_accepted_solvers, ); validators.push(Box::new(database_solver_participation_validator)); } diff --git a/crates/autopilot/src/run.rs b/crates/autopilot/src/run.rs index cfa031f81a..87bfa4d942 100644 --- a/crates/autopilot/src/run.rs +++ b/crates/autopilot/src/run.rs @@ -52,7 +52,7 @@ use { token_list::{AutoUpdatingTokenList, TokenListConfiguration}, }, std::{ - collections::HashMap, + collections::HashSet, sync::{Arc, RwLock}, time::{Duration, Instant}, }, @@ -585,8 +585,10 @@ pub async fn run(args: Arguments) { args.db_based_solver_participation_guard, drivers .iter() - .map(|driver| (driver.submission_address, driver.accepts_unsettled_blocking)) - .collect::>(), + .filter_map(|driver| { + (driver.accepts_unsettled_blocking).then_some(driver.submission_address) + }) + .collect::>(), ); let run = RunLoop::new( From 5fc831ee08005a8b020f9c23f5642ef05ace29fa Mon Sep 17 00:00:00 2001 From: ilya Date: Thu, 30 Jan 2025 16:37:56 +0000 Subject: [PATCH 13/82] Nit --- crates/autopilot/src/run.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/autopilot/src/run.rs b/crates/autopilot/src/run.rs index 87bfa4d942..8c8f2b7834 100644 --- a/crates/autopilot/src/run.rs +++ b/crates/autopilot/src/run.rs @@ -586,7 +586,9 @@ pub async fn run(args: Arguments) { drivers .iter() .filter_map(|driver| { - (driver.accepts_unsettled_blocking).then_some(driver.submission_address) + driver + .accepts_unsettled_blocking + .then_some(driver.submission_address) }) .collect::>(), ); From 321f9bc73ac2fe87d79f6ae5683c57926961c432 Mon Sep 17 00:00:00 2001 From: ilya Date: Thu, 30 Jan 2025 17:17:55 +0000 Subject: [PATCH 14/82] Use driver directly --- .../competition/participation_guard/db.rs | 13 +++++------ .../competition/participation_guard/mod.rs | 13 +++++------ .../participation_guard/onchain.rs | 6 ++--- crates/autopilot/src/run.rs | 22 +++++-------------- crates/autopilot/src/run_loop.rs | 2 +- 5 files changed, 20 insertions(+), 36 deletions(-) diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index f765b39418..0a8dba9f7d 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -2,10 +2,10 @@ use { crate::{ database::Postgres, domain::{eth, Metrics}, + infra::Driver, }, ethrpc::block_stream::CurrentBlockWatcher, std::{ - collections::HashSet, sync::Arc, time::{Duration, Instant}, }, @@ -21,7 +21,6 @@ struct Inner { banned_solvers: dashmap::DashMap, ttl: Duration, last_auctions_count: u32, - db_validator_accepted_solvers: HashSet, } impl Validator { @@ -31,14 +30,12 @@ impl Validator { settlement_updates_receiver: tokio::sync::mpsc::UnboundedReceiver<()>, ttl: Duration, last_auctions_count: u32, - db_validator_accepted_solvers: HashSet, ) -> Self { let self_ = Self(Arc::new(Inner { db, banned_solvers: Default::default(), ttl, last_auctions_count, - db_validator_accepted_solvers, })); self_.start_maintenance(settlement_updates_receiver, current_block); @@ -94,18 +91,18 @@ impl Validator { #[async_trait::async_trait] impl super::Validator for Validator { - async fn is_allowed(&self, solver: ð::Address) -> anyhow::Result { + async fn is_allowed(&self, driver: &Driver) -> anyhow::Result { // Check if solver accepted this feature. This should be removed once a CIP is // approved. - if !self.0.db_validator_accepted_solvers.contains(solver) { + if !driver.accepts_unsettled_blocking { return Ok(true); } - if let Some(entry) = self.0.banned_solvers.get(solver) { + if let Some(entry) = self.0.banned_solvers.get(&driver.submission_address) { if Instant::now().duration_since(*entry.value()) < self.0.ttl { return Ok(false); } else { - self.0.banned_solvers.remove(solver); + self.0.banned_solvers.remove(&driver.submission_address); } } diff --git a/crates/autopilot/src/domain/competition/participation_guard/mod.rs b/crates/autopilot/src/domain/competition/participation_guard/mod.rs index d0127e68d5..7b4bf53157 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/mod.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/mod.rs @@ -5,10 +5,9 @@ use { crate::{ arguments::DbBasedSolverParticipationGuardConfig, database::Postgres, - domain::eth, - infra::Ethereum, + infra::{Driver, Ethereum}, }, - std::{collections::HashSet, sync::Arc}, + std::sync::Arc, }; /// This struct checks whether a solver can participate in the competition by @@ -27,7 +26,6 @@ impl SolverParticipationGuard { db: Postgres, settlement_updates_receiver: tokio::sync::mpsc::UnboundedReceiver<()>, db_based_validator_config: DbBasedSolverParticipationGuardConfig, - db_validator_accepted_solvers: HashSet, ) -> Self { let mut validators: Vec> = Vec::new(); @@ -39,7 +37,6 @@ impl SolverParticipationGuard { settlement_updates_receiver, db_based_validator_config.solver_blacklist_cache_ttl, db_based_validator_config.solver_last_auctions_participation_count, - db_validator_accepted_solvers, ); validators.push(Box::new(database_solver_participation_validator)); } @@ -55,9 +52,9 @@ impl SolverParticipationGuard { /// the following order: /// 1. DB-based validator: operates fast since it uses in-memory cache. /// 2. Onchain-based validator: only then calls the Authenticator contract. - pub async fn can_participate(&self, solver: ð::Address) -> anyhow::Result { + pub async fn can_participate(&self, driver: &Driver) -> anyhow::Result { for validator in &self.0.validators { - if !validator.is_allowed(solver).await? { + if !validator.is_allowed(driver).await? { return Ok(false); } } @@ -68,5 +65,5 @@ impl SolverParticipationGuard { #[async_trait::async_trait] trait Validator: Send + Sync { - async fn is_allowed(&self, solver: ð::Address) -> anyhow::Result; + async fn is_allowed(&self, driver: &Driver) -> anyhow::Result; } diff --git a/crates/autopilot/src/domain/competition/participation_guard/onchain.rs b/crates/autopilot/src/domain/competition/participation_guard/onchain.rs index 82d0ef3fb7..3e472b298e 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/onchain.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/onchain.rs @@ -1,4 +1,4 @@ -use crate::{domain::eth, infra::Ethereum}; +use crate::infra::{Driver, Ethereum}; /// Calls Authenticator contract to check if a solver has a sufficient /// permission. @@ -8,12 +8,12 @@ pub(super) struct Validator { #[async_trait::async_trait] impl super::Validator for Validator { - async fn is_allowed(&self, solver: ð::Address) -> anyhow::Result { + async fn is_allowed(&self, solver: &Driver) -> anyhow::Result { Ok(self .eth .contracts() .authenticator() - .is_solver(solver.0) + .is_solver(solver.submission_address.0) .call() .await?) } diff --git a/crates/autopilot/src/run.rs b/crates/autopilot/src/run.rs index 8c8f2b7834..2f426a1967 100644 --- a/crates/autopilot/src/run.rs +++ b/crates/autopilot/src/run.rs @@ -52,7 +52,6 @@ use { token_list::{AutoUpdatingTokenList, TokenListConfiguration}, }, std::{ - collections::HashSet, sync::{Arc, RwLock}, time::{Duration, Instant}, }, @@ -369,6 +368,12 @@ pub async fn run(args: Arguments) { let (settlement_updates_sender, settlement_updates_receiver) = tokio::sync::mpsc::unbounded_channel(); + let solver_participation_guard = SolverParticipationGuard::new( + eth.clone(), + db.clone(), + settlement_updates_receiver, + args.db_based_solver_participation_guard, + ); let persistence = infra::persistence::Persistence::new(args.s3.into().unwrap(), Arc::new(db.clone())).await; let settlement_observer = crate::domain::settlement::Observer::new( @@ -578,21 +583,6 @@ pub async fn run(args: Arguments) { .into_iter() .collect(); - let solver_participation_guard = SolverParticipationGuard::new( - eth.clone(), - db.clone(), - settlement_updates_receiver, - args.db_based_solver_participation_guard, - drivers - .iter() - .filter_map(|driver| { - driver - .accepts_unsettled_blocking - .then_some(driver.submission_address) - }) - .collect::>(), - ); - let run = RunLoop::new( run_loop_config, eth, diff --git a/crates/autopilot/src/run_loop.rs b/crates/autopilot/src/run_loop.rs index 492ca18751..4102f675ea 100644 --- a/crates/autopilot/src/run_loop.rs +++ b/crates/autopilot/src/run_loop.rs @@ -736,7 +736,7 @@ impl RunLoop { request: &solve::Request, ) -> Result>, SolveError> { - let can_participate = self.solver_participation_guard.can_participate(&driver.submission_address).await.map_err(|err| { + let can_participate = self.solver_participation_guard.can_participate(driver).await.map_err(|err| { tracing::error!(?err, driver = %driver.name, ?driver.submission_address, "solver participation check failed"); SolveError::SolverDenyListed } From f69e1747ad5150033f0bb94443df4550fb898b70 Mon Sep 17 00:00:00 2001 From: ilya Date: Thu, 30 Jan 2025 19:05:01 +0000 Subject: [PATCH 15/82] Notify external solvers --- .../competition/participation_guard/db.rs | 47 +++++++++++++++++-- .../competition/participation_guard/mod.rs | 13 +++-- .../participation_guard/onchain.rs | 6 +-- crates/autopilot/src/infra/solvers/dto/mod.rs | 1 + .../autopilot/src/infra/solvers/dto/notify.rs | 9 ++++ crates/autopilot/src/infra/solvers/mod.rs | 6 ++- crates/autopilot/src/run.rs | 18 ++++--- 7 files changed, 80 insertions(+), 20 deletions(-) create mode 100644 crates/autopilot/src/infra/solvers/dto/notify.rs diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index 0a8dba9f7d..6c50b96553 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -2,10 +2,12 @@ use { crate::{ database::Postgres, domain::{eth, Metrics}, - infra::Driver, + infra, }, ethrpc::block_stream::CurrentBlockWatcher, + futures::future::join_all, std::{ + collections::HashMap, sync::Arc, time::{Duration, Instant}, }, @@ -21,6 +23,7 @@ struct Inner { banned_solvers: dashmap::DashMap, ttl: Duration, last_auctions_count: u32, + drivers_by_address: HashMap>, } impl Validator { @@ -30,12 +33,14 @@ impl Validator { settlement_updates_receiver: tokio::sync::mpsc::UnboundedReceiver<()>, ttl: Duration, last_auctions_count: u32, + drivers_by_address: HashMap>, ) -> Self { let self_ = Self(Arc::new(Inner { db, banned_solvers: Default::default(), ttl, last_auctions_count, + drivers_by_address, })); self_.start_maintenance(settlement_updates_receiver, current_block); @@ -50,13 +55,14 @@ impl Validator { mut settlement_updates_receiver: tokio::sync::mpsc::UnboundedReceiver<()>, current_block: CurrentBlockWatcher, ) { - let self_ = self.0.clone(); + let self_ = self.clone(); tokio::spawn(async move { while settlement_updates_receiver.recv().await.is_some() { let current_block = current_block.borrow().number; match self_ + .0 .db - .find_non_settling_solvers(self_.last_auctions_count, current_block) + .find_non_settling_solvers(self_.0.last_auctions_count, current_block) .await { Ok(non_settling_solvers) => { @@ -74,10 +80,11 @@ impl Validator { .collect::>(); tracing::debug!(?non_settling_solvers, "found non-settling solvers"); + self_.notify_solvers(&non_settling_solvers); let now = Instant::now(); for solver in non_settling_solvers { - self_.banned_solvers.insert(solver, now); + self_.0.banned_solvers.insert(solver, now); } } Err(err) => { @@ -87,11 +94,41 @@ impl Validator { } }); } + + /// Try to notify all the non-settling external solvers. + fn notify_solvers(&self, non_settling_solvers: &[eth::Address]) { + let futures = non_settling_solvers + .iter() + .cloned() + .map(|solver| { + let self_ = self.0.clone(); + async move { + match self_.drivers_by_address.get(&solver) { + Some(driver) => { + if let Err(err) = driver + .notify(&infra::solvers::dto::notify::Request::UnsettledConsecutiveAuctions) + .await + { + tracing::debug!(?solver, ?err, "unable to notify external solver"); + } + } + None => { + tracing::error!(?solver, "found unrecognized non-settling driver"); + } + } + } + } + ).collect::>(); + + tokio::spawn(async move { + join_all(futures).await; + }); + } } #[async_trait::async_trait] impl super::Validator for Validator { - async fn is_allowed(&self, driver: &Driver) -> anyhow::Result { + async fn is_allowed(&self, driver: &infra::Driver) -> anyhow::Result { // Check if solver accepted this feature. This should be removed once a CIP is // approved. if !driver.accepts_unsettled_blocking { diff --git a/crates/autopilot/src/domain/competition/participation_guard/mod.rs b/crates/autopilot/src/domain/competition/participation_guard/mod.rs index 7b4bf53157..5564cc07f6 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/mod.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/mod.rs @@ -5,9 +5,10 @@ use { crate::{ arguments::DbBasedSolverParticipationGuardConfig, database::Postgres, - infra::{Driver, Ethereum}, + domain::eth, + infra, }, - std::sync::Arc, + std::{collections::HashMap, sync::Arc}, }; /// This struct checks whether a solver can participate in the competition by @@ -22,10 +23,11 @@ struct Inner { impl SolverParticipationGuard { pub fn new( - eth: Ethereum, + eth: infra::Ethereum, db: Postgres, settlement_updates_receiver: tokio::sync::mpsc::UnboundedReceiver<()>, db_based_validator_config: DbBasedSolverParticipationGuardConfig, + drivers_by_address: HashMap>, ) -> Self { let mut validators: Vec> = Vec::new(); @@ -37,6 +39,7 @@ impl SolverParticipationGuard { settlement_updates_receiver, db_based_validator_config.solver_blacklist_cache_ttl, db_based_validator_config.solver_last_auctions_participation_count, + drivers_by_address, ); validators.push(Box::new(database_solver_participation_validator)); } @@ -52,7 +55,7 @@ impl SolverParticipationGuard { /// the following order: /// 1. DB-based validator: operates fast since it uses in-memory cache. /// 2. Onchain-based validator: only then calls the Authenticator contract. - pub async fn can_participate(&self, driver: &Driver) -> anyhow::Result { + pub async fn can_participate(&self, driver: &infra::Driver) -> anyhow::Result { for validator in &self.0.validators { if !validator.is_allowed(driver).await? { return Ok(false); @@ -65,5 +68,5 @@ impl SolverParticipationGuard { #[async_trait::async_trait] trait Validator: Send + Sync { - async fn is_allowed(&self, driver: &Driver) -> anyhow::Result; + async fn is_allowed(&self, driver: &infra::Driver) -> anyhow::Result; } diff --git a/crates/autopilot/src/domain/competition/participation_guard/onchain.rs b/crates/autopilot/src/domain/competition/participation_guard/onchain.rs index 3e472b298e..d1327f0254 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/onchain.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/onchain.rs @@ -1,14 +1,14 @@ -use crate::infra::{Driver, Ethereum}; +use crate::infra; /// Calls Authenticator contract to check if a solver has a sufficient /// permission. pub(super) struct Validator { - pub eth: Ethereum, + pub eth: infra::Ethereum, } #[async_trait::async_trait] impl super::Validator for Validator { - async fn is_allowed(&self, solver: &Driver) -> anyhow::Result { + async fn is_allowed(&self, solver: &infra::Driver) -> anyhow::Result { Ok(self .eth .contracts() diff --git a/crates/autopilot/src/infra/solvers/dto/mod.rs b/crates/autopilot/src/infra/solvers/dto/mod.rs index d3a156294c..5365700ce8 100644 --- a/crates/autopilot/src/infra/solvers/dto/mod.rs +++ b/crates/autopilot/src/infra/solvers/dto/mod.rs @@ -1,6 +1,7 @@ //! Types for communicating with drivers as defined in //! `crates/driver/openapi.yml`. +pub mod notify; pub mod reveal; pub mod settle; pub mod solve; diff --git a/crates/autopilot/src/infra/solvers/dto/notify.rs b/crates/autopilot/src/infra/solvers/dto/notify.rs new file mode 100644 index 0000000000..08165964fd --- /dev/null +++ b/crates/autopilot/src/infra/solvers/dto/notify.rs @@ -0,0 +1,9 @@ +use {serde::Serialize, serde_with::serde_as}; + +#[serde_as] +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum Request { + /// The driver won multiple consecutive auctions but never settled them. + UnsettledConsecutiveAuctions, +} diff --git a/crates/autopilot/src/infra/solvers/mod.rs b/crates/autopilot/src/infra/solvers/mod.rs index aae623ebaa..2490a71b15 100644 --- a/crates/autopilot/src/infra/solvers/mod.rs +++ b/crates/autopilot/src/infra/solvers/mod.rs @@ -1,6 +1,6 @@ use { self::dto::{reveal, settle, solve}, - crate::{arguments::Account, domain::eth, util}, + crate::{arguments::Account, domain::eth, infra::solvers::dto::notify, util}, anyhow::{anyhow, Context, Result}, reqwest::{Client, StatusCode}, std::time::Duration, @@ -116,6 +116,10 @@ impl Driver { Ok(()) } + pub async fn notify(&self, request: ¬ify::Request) -> Result<()> { + self.request_response("notify", request, None).await + } + async fn request_response( &self, path: &str, diff --git a/crates/autopilot/src/run.rs b/crates/autopilot/src/run.rs index 2f426a1967..a6e30dd562 100644 --- a/crates/autopilot/src/run.rs +++ b/crates/autopilot/src/run.rs @@ -52,6 +52,7 @@ use { token_list::{AutoUpdatingTokenList, TokenListConfiguration}, }, std::{ + collections::HashMap, sync::{Arc, RwLock}, time::{Duration, Instant}, }, @@ -368,12 +369,6 @@ pub async fn run(args: Arguments) { let (settlement_updates_sender, settlement_updates_receiver) = tokio::sync::mpsc::unbounded_channel(); - let solver_participation_guard = SolverParticipationGuard::new( - eth.clone(), - db.clone(), - settlement_updates_receiver, - args.db_based_solver_participation_guard, - ); let persistence = infra::persistence::Persistence::new(args.s3.into().unwrap(), Arc::new(db.clone())).await; let settlement_observer = crate::domain::settlement::Observer::new( @@ -583,6 +578,17 @@ pub async fn run(args: Arguments) { .into_iter() .collect(); + let solver_participation_guard = SolverParticipationGuard::new( + eth.clone(), + db.clone(), + settlement_updates_receiver, + args.db_based_solver_participation_guard, + drivers + .iter() + .map(|driver| (driver.submission_address, driver.clone())) + .collect::>(), + ); + let run = RunLoop::new( run_loop_config, eth, From 1504c48bb75ad9dd1c65442d43ac36b0f192ff33 Mon Sep 17 00:00:00 2001 From: ilya Date: Thu, 30 Jan 2025 20:11:23 +0000 Subject: [PATCH 16/82] /notify driver endpoint --- crates/driver/src/infra/api/error.rs | 9 +++ crates/driver/src/infra/api/mod.rs | 1 + crates/driver/src/infra/api/routes/mod.rs | 2 + .../src/infra/api/routes/notify/dto/mod.rs | 3 + .../api/routes/notify/dto/notify_request.rs | 25 ++++++++ .../driver/src/infra/api/routes/notify/mod.rs | 23 +++++++ crates/driver/src/infra/notify/mod.rs | 16 ++--- .../driver/src/infra/notify/notification.rs | 3 + .../src/infra/solver/dto/notification.rs | 2 + crates/driver/src/infra/solver/mod.rs | 64 ++++++++++++++++--- 10 files changed, 130 insertions(+), 18 deletions(-) create mode 100644 crates/driver/src/infra/api/routes/notify/dto/mod.rs create mode 100644 crates/driver/src/infra/api/routes/notify/dto/notify_request.rs create mode 100644 crates/driver/src/infra/api/routes/notify/mod.rs diff --git a/crates/driver/src/infra/api/error.rs b/crates/driver/src/infra/api/error.rs index e788bbe5bf..e95768c616 100644 --- a/crates/driver/src/infra/api/error.rs +++ b/crates/driver/src/infra/api/error.rs @@ -112,3 +112,12 @@ impl From for (hyper::StatusCode, axum::Json) { error.into() } } + +impl From for (hyper::StatusCode, axum::Json) { + fn from(value: api::routes::NotifyError) -> Self { + let error = match value { + api::routes::NotifyError::UnableToNotify => Kind::Unknown, + }; + error.into() + } +} diff --git a/crates/driver/src/infra/api/mod.rs b/crates/driver/src/infra/api/mod.rs index dfbc33fa97..d153b34979 100644 --- a/crates/driver/src/infra/api/mod.rs +++ b/crates/driver/src/infra/api/mod.rs @@ -78,6 +78,7 @@ impl Api { let router = routes::solve(router); let router = routes::reveal(router); let router = routes::settle(router); + let router = routes::notify(router); let bad_token_config = solver.bad_token_detection(); let mut bad_tokens = diff --git a/crates/driver/src/infra/api/routes/mod.rs b/crates/driver/src/infra/api/routes/mod.rs index ee1027b0e9..d4ed5e12e3 100644 --- a/crates/driver/src/infra/api/routes/mod.rs +++ b/crates/driver/src/infra/api/routes/mod.rs @@ -1,6 +1,7 @@ mod healthz; mod info; mod metrics; +mod notify; mod quote; mod reveal; mod settle; @@ -10,6 +11,7 @@ pub(super) use { healthz::healthz, info::info, metrics::metrics, + notify::{notify, NotifyError}, quote::{quote, OrderError}, reveal::reveal, settle::settle, diff --git a/crates/driver/src/infra/api/routes/notify/dto/mod.rs b/crates/driver/src/infra/api/routes/notify/dto/mod.rs new file mode 100644 index 0000000000..3f99a96ff7 --- /dev/null +++ b/crates/driver/src/infra/api/routes/notify/dto/mod.rs @@ -0,0 +1,3 @@ +mod notify_request; + +pub use notify_request::{Error as NotifyError, NotifyRequest}; diff --git a/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs b/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs new file mode 100644 index 0000000000..22be6cbb00 --- /dev/null +++ b/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs @@ -0,0 +1,25 @@ +use {crate::infra::notify, serde::Deserialize, serde_with::serde_as}; + +#[serde_as] +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum NotifyRequest { + /// The driver won multiple consecutive auctions but never settled them. + UnsettledConsecutiveAuctions, +} + +impl From for notify::Kind { + fn from(value: NotifyRequest) -> Self { + match value { + NotifyRequest::UnsettledConsecutiveAuctions => { + notify::Kind::UnsettledConsecutiveAuctions + } + } + } +} + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("Unable to notify solver")] + UnableToNotify, +} diff --git a/crates/driver/src/infra/api/routes/notify/mod.rs b/crates/driver/src/infra/api/routes/notify/mod.rs new file mode 100644 index 0000000000..7683cc499b --- /dev/null +++ b/crates/driver/src/infra/api/routes/notify/mod.rs @@ -0,0 +1,23 @@ +mod dto; + +pub use dto::NotifyError; + +use crate::infra::api::{Error, State}; + +pub(in crate::infra::api) fn notify(router: axum::Router) -> axum::Router { + router.route("/notify", axum::routing::post(route)) +} + +async fn route( + state: axum::extract::State, + req: axum::Json, +) -> Result)> { + let solver = &state.solver().name().0; + tracing::trace!(?req, ?solver, "received a notification"); + state + .solver() + .notify(None, None, req.0.into()) + .await + .map(|_| hyper::StatusCode::OK) + .map_err(|_| NotifyError::UnableToNotify.into()) +} diff --git a/crates/driver/src/infra/notify/mod.rs b/crates/driver/src/infra/notify/mod.rs index edb4c55458..72446e96bf 100644 --- a/crates/driver/src/infra/notify/mod.rs +++ b/crates/driver/src/infra/notify/mod.rs @@ -12,11 +12,11 @@ use { }; pub fn solver_timeout(solver: &Solver, auction_id: Option) { - solver.notify(auction_id, None, notification::Kind::Timeout); + solver.notify_and_forget(auction_id, None, notification::Kind::Timeout); } pub fn empty_solution(solver: &Solver, auction_id: Option, solution: solution::Id) { - solver.notify( + solver.notify_and_forget( auction_id, Some(solution), notification::Kind::EmptySolution, @@ -45,7 +45,7 @@ pub fn scoring_failed( } }; - solver.notify(auction_id, Some(solution_id.clone()), notification); + solver.notify_and_forget(auction_id, Some(solution_id.clone()), notification); } pub fn encoding_failed( @@ -76,7 +76,7 @@ pub fn encoding_failed( solution::Error::Encoding(_) => return, }; - solver.notify(auction_id, Some(solution_id.clone()), notification); + solver.notify_and_forget(auction_id, Some(solution_id.clone()), notification); } pub fn simulation_failed( @@ -94,7 +94,7 @@ pub fn simulation_failed( ), simulator::Error::Other(error) => notification::Kind::DriverError(error.to_string()), }; - solver.notify(auction_id, Some(solution_id.clone()), kind); + solver.notify_and_forget(auction_id, Some(solution_id.clone()), kind); } pub fn executed( @@ -111,7 +111,7 @@ pub fn executed( Err(Error::Other(_) | Error::Disabled) => notification::Settlement::Fail, }; - solver.notify( + solver.notify_and_forget( Some(auction_id), Some(solution_id.clone()), notification::Kind::Settled(kind), @@ -123,7 +123,7 @@ pub fn duplicated_solution_id( auction_id: Option, solution_id: &solution::Id, ) { - solver.notify( + solver.notify_and_forget( auction_id, Some(solution_id.clone()), notification::Kind::DuplicatedSolutionId, @@ -131,5 +131,5 @@ pub fn duplicated_solution_id( } pub fn postprocessing_timed_out(solver: &Solver, auction_id: Option) { - solver.notify(auction_id, None, notification::Kind::PostprocessingTimedOut); + solver.notify_and_forget(auction_id, None, notification::Kind::PostprocessingTimedOut); } diff --git a/crates/driver/src/infra/notify/notification.rs b/crates/driver/src/infra/notify/notification.rs index 0a178e9af5..ca123bb5e8 100644 --- a/crates/driver/src/infra/notify/notification.rs +++ b/crates/driver/src/infra/notify/notification.rs @@ -45,6 +45,9 @@ pub enum Kind { DriverError(String), /// On-chain solution postprocessing timed out. PostprocessingTimedOut, + /// The solver won multiple consecutive auctions but none of the settlement + /// succeeded. + UnsettledConsecutiveAuctions, } #[derive(Debug)] diff --git a/crates/driver/src/infra/solver/dto/notification.rs b/crates/driver/src/infra/solver/dto/notification.rs index 08969c5c35..2c70f50f5a 100644 --- a/crates/driver/src/infra/solver/dto/notification.rs +++ b/crates/driver/src/infra/solver/dto/notification.rs @@ -61,6 +61,7 @@ impl Notification { notify::Settlement::Expired => Kind::Expired, }, notify::Kind::PostprocessingTimedOut => Kind::PostprocessingTimedOut, + notify::Kind::UnsettledConsecutiveAuctions => Kind::UnsettledConsecutiveAuctions, }, } } @@ -144,6 +145,7 @@ pub enum Kind { Expired, Fail, PostprocessingTimedOut, + UnsettledConsecutiveAuctions, } type BlockNo = u64; diff --git a/crates/driver/src/infra/solver/mod.rs b/crates/driver/src/infra/solver/mod.rs index 3e63af9e86..6a6b3f8386 100644 --- a/crates/driver/src/infra/solver/mod.rs +++ b/crates/driver/src/infra/solver/mod.rs @@ -260,27 +260,71 @@ impl Solver { } /// Make a fire and forget POST request to notify the solver about an event. - pub fn notify( + pub fn notify_and_forget( &self, auction_id: Option, solution_id: Option, kind: notify::Kind, ) { + let base_url = self.config.endpoint.clone(); + let client = self.client.clone(); + let response_limit = self.config.response_size_limit_max_bytes; + let future = async move { + if let Err(error) = Self::notify_( + base_url, + client, + response_limit, + auction_id, + solution_id, + kind, + ) + .await + { + tracing::warn!(?error, "failed to notify solver"); + } + }; + + tokio::task::spawn(future.in_current_span()); + } + + pub async fn notify( + &self, + auction_id: Option, + solution_id: Option, + kind: notify::Kind, + ) -> Result<(), crate::util::http::Error> { + let base_url = self.config.endpoint.clone(); + let client = self.client.clone(); + + Self::notify_( + base_url, + client, + self.config.response_size_limit_max_bytes, + auction_id, + solution_id, + kind, + ) + .await + } + + async fn notify_( + base_url: url::Url, + client: reqwest::Client, + response_limit: usize, + auction_id: Option, + solution_id: Option, + kind: notify::Kind, + ) -> Result<(), crate::util::http::Error> { let body = serde_json::to_string(&dto::Notification::new(auction_id, solution_id, kind)).unwrap(); - let url = shared::url::join(&self.config.endpoint, "notify"); + let url = shared::url::join(&base_url, "notify"); super::observe::solver_request(&url, &body); - let mut req = self.client.post(url).body(body); + let mut req = client.post(url).body(body); if let Some(id) = observe::request_id::from_current_span() { req = req.header("X-REQUEST-ID", id); } - let response_size = self.config.response_size_limit_max_bytes; - let future = async move { - if let Err(error) = util::http::send(response_size, req).await { - tracing::warn!(?error, "failed to notify solver"); - } - }; - tokio::task::spawn(future.in_current_span()); + + util::http::send(response_limit, req).await.map(|_| ()) } } From 9f0cbc8462a8110f84c1371dcd7d1ded3b18950a Mon Sep 17 00:00:00 2001 From: ilya Date: Thu, 30 Jan 2025 20:24:11 +0000 Subject: [PATCH 17/82] OpenApi --- crates/driver/openapi.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/crates/driver/openapi.yml b/crates/driver/openapi.yml index a8d43d199f..0383a35130 100644 --- a/crates/driver/openapi.yml +++ b/crates/driver/openapi.yml @@ -138,6 +138,33 @@ paths: $ref: "#/components/responses/BadRequest" "500": $ref: "#/components/responses/InternalServerError" + /notify: + post: + description: | + Receive a notification in case the driver was banned for a certain reason. + requestBody: + required: true + content: + application/json: + schema: + type: string + enum: + - unsettledConsecutiveAuctions + description: |- + A notification that informs about a reasoning why the driver was banned. + responses: + "200": + description: notification successfully received. + "500": + description: Unable to notify solver. + content: + application/json: + schema: + type: object + properties: + error: + type: string + example: "Unable to notify solver" components: schemas: Address: From 7f86000b61b69c513926b2a97d8f8fbceb3bc9e8 Mon Sep 17 00:00:00 2001 From: ilya Date: Thu, 30 Jan 2025 20:24:40 +0000 Subject: [PATCH 18/82] Nit --- .../autopilot/src/domain/competition/participation_guard/db.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index 6c50b96553..969674bc55 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -95,7 +95,7 @@ impl Validator { }); } - /// Try to notify all the non-settling external solvers. + /// Try to notify all the non-settling solvers. fn notify_solvers(&self, non_settling_solvers: &[eth::Address]) { let futures = non_settling_solvers .iter() From d2330942e1a2c7e196f96d64747b960450d73830 Mon Sep 17 00:00:00 2001 From: ilya Date: Thu, 30 Jan 2025 20:35:38 +0000 Subject: [PATCH 19/82] Solver's OpenAPI --- crates/solvers/openapi.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/solvers/openapi.yml b/crates/solvers/openapi.yml index af9562efd2..135f47cfaa 100644 --- a/crates/solvers/openapi.yml +++ b/crates/solvers/openapi.yml @@ -89,6 +89,7 @@ paths: - cancelled - fail - postprocessingTimedOut + - unsettledConsecutiveAuctions responses: "200": description: notification successfully received. From 105b9a7bc90b8f43390ee221e6660c1fd5626e04 Mon Sep 17 00:00:00 2001 From: ilya Date: Fri, 31 Jan 2025 08:52:59 +0000 Subject: [PATCH 20/82] Store drivers accepted for the feature --- .../competition/participation_guard/db.rs | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index 969674bc55..64d7ea6234 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -35,12 +35,16 @@ impl Validator { last_auctions_count: u32, drivers_by_address: HashMap>, ) -> Self { + let accepted_drivers_by_address = drivers_by_address + .into_iter() + .filter(|(_, driver)| driver.accepts_unsettled_blocking) + .collect::>(); let self_ = Self(Arc::new(Inner { db, banned_solvers: Default::default(), ttl, last_auctions_count, - drivers_by_address, + drivers_by_address: accepted_drivers_by_address, })); self_.start_maintenance(settlement_updates_receiver, current_block); @@ -83,9 +87,14 @@ impl Validator { self_.notify_solvers(&non_settling_solvers); let now = Instant::now(); - for solver in non_settling_solvers { - self_.0.banned_solvers.insert(solver, now); - } + non_settling_solvers + .into_iter() + // Check if solver accepted this feature. This should be removed once a CIP is + // approved. + .filter(|solver| self_.0.drivers_by_address.contains_key(solver)) + .for_each(|solver| { + self_.0.banned_solvers.insert(solver, now); + }); } Err(err) => { tracing::warn!(?err, "error while searching for non-settling solvers") @@ -103,22 +112,19 @@ impl Validator { .map(|solver| { let self_ = self.0.clone(); async move { - match self_.drivers_by_address.get(&solver) { - Some(driver) => { - if let Err(err) = driver - .notify(&infra::solvers::dto::notify::Request::UnsettledConsecutiveAuctions) - .await - { - tracing::debug!(?solver, ?err, "unable to notify external solver"); - } - } - None => { - tracing::error!(?solver, "found unrecognized non-settling driver"); + if let Some(driver) = self_.drivers_by_address.get(&solver) { + if let Err(err) = driver + .notify( + &infra::solvers::dto::notify::Request::UnsettledConsecutiveAuctions, + ) + .await + { + tracing::debug!(?solver, ?err, "unable to notify external solver"); } } } - } - ).collect::>(); + }) + .collect::>(); tokio::spawn(async move { join_all(futures).await; @@ -129,12 +135,6 @@ impl Validator { #[async_trait::async_trait] impl super::Validator for Validator { async fn is_allowed(&self, driver: &infra::Driver) -> anyhow::Result { - // Check if solver accepted this feature. This should be removed once a CIP is - // approved. - if !driver.accepts_unsettled_blocking { - return Ok(true); - } - if let Some(entry) = self.0.banned_solvers.get(&driver.submission_address) { if Instant::now().duration_since(*entry.value()) < self.0.ttl { return Ok(false); From 08924daab49dfeca85c4bd1b7f961361ef713e00 Mon Sep 17 00:00:00 2001 From: ilya Date: Fri, 31 Jan 2025 08:53:48 +0000 Subject: [PATCH 21/82] Comment --- .../autopilot/src/domain/competition/participation_guard/db.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index 64d7ea6234..bf5af37a20 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -104,7 +104,7 @@ impl Validator { }); } - /// Try to notify all the non-settling solvers. + /// Try to notify all the non-settling solvers in a background task. fn notify_solvers(&self, non_settling_solvers: &[eth::Address]) { let futures = non_settling_solvers .iter() From 3154cd0827f40ab52563eac2c66f4a6ae5b99a7f Mon Sep 17 00:00:00 2001 From: ilya Date: Fri, 31 Jan 2025 15:18:48 +0000 Subject: [PATCH 22/82] Use driver's name in metrics --- .../competition/participation_guard/db.rs | 45 +++++++++++-------- .../competition/participation_guard/mod.rs | 8 ++-- crates/autopilot/src/run.rs | 10 ++--- 3 files changed, 33 insertions(+), 30 deletions(-) diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index f765b39418..8ddbb28f5f 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -2,10 +2,11 @@ use { crate::{ database::Postgres, domain::{eth, Metrics}, + infra, }, ethrpc::block_stream::CurrentBlockWatcher, std::{ - collections::HashSet, + collections::HashMap, sync::Arc, time::{Duration, Instant}, }, @@ -21,7 +22,7 @@ struct Inner { banned_solvers: dashmap::DashMap, ttl: Duration, last_auctions_count: u32, - db_validator_accepted_solvers: HashSet, + drivers_by_address: HashMap>, } impl Validator { @@ -31,14 +32,19 @@ impl Validator { settlement_updates_receiver: tokio::sync::mpsc::UnboundedReceiver<()>, ttl: Duration, last_auctions_count: u32, - db_validator_accepted_solvers: HashSet, + drivers_by_address: HashMap>, ) -> Self { + // Keep only drivers that accept unsettled blocking. + let drivers_by_address = drivers_by_address + .into_iter() + .filter(|(_, driver)| driver.accepts_unsettled_blocking) + .collect(); let self_ = Self(Arc::new(Inner { db, banned_solvers: Default::default(), ttl, last_auctions_count, - db_validator_accepted_solvers, + drivers_by_address, })); self_.start_maintenance(settlement_updates_receiver, current_block); @@ -53,13 +59,14 @@ impl Validator { mut settlement_updates_receiver: tokio::sync::mpsc::UnboundedReceiver<()>, current_block: CurrentBlockWatcher, ) { - let self_ = self.0.clone(); + let self_ = self.clone(); tokio::spawn(async move { while settlement_updates_receiver.recv().await.is_some() { let current_block = current_block.borrow().number; match self_ + .0 .db - .find_non_settling_solvers(self_.last_auctions_count, current_block) + .find_non_settling_solvers(self_.0.last_auctions_count, current_block) .await { Ok(non_settling_solvers) => { @@ -67,10 +74,11 @@ impl Validator { .into_iter() .map(|solver| { let address = eth::Address(solver.0.into()); - - Metrics::get() - .non_settling_solver - .with_label_values(&[&format!("{:#x}", address.0)]); + if let Some(driver) = self_.0.drivers_by_address.get(&address) { + Metrics::get() + .non_settling_solver + .with_label_values(&[&driver.name]); + } address }) @@ -79,9 +87,14 @@ impl Validator { tracing::debug!(?non_settling_solvers, "found non-settling solvers"); let now = Instant::now(); - for solver in non_settling_solvers { - self_.banned_solvers.insert(solver, now); - } + non_settling_solvers + .into_iter() + // Check if solver accepted this feature. This should be removed once a CIP is + // approved. + .filter(|solver| self_.0.drivers_by_address.contains_key(solver)) + .for_each(|solver| { + self_.0.banned_solvers.insert(solver, now); + }); } Err(err) => { tracing::warn!(?err, "error while searching for non-settling solvers") @@ -95,12 +108,6 @@ impl Validator { #[async_trait::async_trait] impl super::Validator for Validator { async fn is_allowed(&self, solver: ð::Address) -> anyhow::Result { - // Check if solver accepted this feature. This should be removed once a CIP is - // approved. - if !self.0.db_validator_accepted_solvers.contains(solver) { - return Ok(true); - } - if let Some(entry) = self.0.banned_solvers.get(solver) { if Instant::now().duration_since(*entry.value()) < self.0.ttl { return Ok(false); diff --git a/crates/autopilot/src/domain/competition/participation_guard/mod.rs b/crates/autopilot/src/domain/competition/participation_guard/mod.rs index d0127e68d5..925cf9e52e 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/mod.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/mod.rs @@ -6,9 +6,9 @@ use { arguments::DbBasedSolverParticipationGuardConfig, database::Postgres, domain::eth, - infra::Ethereum, + infra::{self, Ethereum}, }, - std::{collections::HashSet, sync::Arc}, + std::{collections::HashMap, sync::Arc}, }; /// This struct checks whether a solver can participate in the competition by @@ -27,7 +27,7 @@ impl SolverParticipationGuard { db: Postgres, settlement_updates_receiver: tokio::sync::mpsc::UnboundedReceiver<()>, db_based_validator_config: DbBasedSolverParticipationGuardConfig, - db_validator_accepted_solvers: HashSet, + drivers_by_address: HashMap>, ) -> Self { let mut validators: Vec> = Vec::new(); @@ -39,7 +39,7 @@ impl SolverParticipationGuard { settlement_updates_receiver, db_based_validator_config.solver_blacklist_cache_ttl, db_based_validator_config.solver_last_auctions_participation_count, - db_validator_accepted_solvers, + drivers_by_address, ); validators.push(Box::new(database_solver_participation_validator)); } diff --git a/crates/autopilot/src/run.rs b/crates/autopilot/src/run.rs index 8c8f2b7834..a6e30dd562 100644 --- a/crates/autopilot/src/run.rs +++ b/crates/autopilot/src/run.rs @@ -52,7 +52,7 @@ use { token_list::{AutoUpdatingTokenList, TokenListConfiguration}, }, std::{ - collections::HashSet, + collections::HashMap, sync::{Arc, RwLock}, time::{Duration, Instant}, }, @@ -585,12 +585,8 @@ pub async fn run(args: Arguments) { args.db_based_solver_participation_guard, drivers .iter() - .filter_map(|driver| { - driver - .accepts_unsettled_blocking - .then_some(driver.submission_address) - }) - .collect::>(), + .map(|driver| (driver.submission_address, driver.clone())) + .collect::>(), ); let run = RunLoop::new( From 47007c100d3676f1f14699c51313503768e86b8f Mon Sep 17 00:00:00 2001 From: ilya Date: Fri, 31 Jan 2025 15:19:28 +0000 Subject: [PATCH 23/82] Nit --- crates/autopilot/src/run.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/autopilot/src/run.rs b/crates/autopilot/src/run.rs index a6e30dd562..bfd3455d71 100644 --- a/crates/autopilot/src/run.rs +++ b/crates/autopilot/src/run.rs @@ -52,7 +52,6 @@ use { token_list::{AutoUpdatingTokenList, TokenListConfiguration}, }, std::{ - collections::HashMap, sync::{Arc, RwLock}, time::{Duration, Instant}, }, @@ -586,7 +585,7 @@ pub async fn run(args: Arguments) { drivers .iter() .map(|driver| (driver.submission_address, driver.clone())) - .collect::>(), + .collect(), ); let run = RunLoop::new( From e37087170b373b17543e1605a50afd202e4a93ad Mon Sep 17 00:00:00 2001 From: ilya Date: Fri, 31 Jan 2025 15:25:18 +0000 Subject: [PATCH 24/82] Fix after merge --- .../src/domain/competition/participation_guard/db.rs | 2 +- .../src/domain/competition/participation_guard/mod.rs | 6 +++--- .../src/domain/competition/participation_guard/onchain.rs | 6 +++--- crates/autopilot/src/run_loop.rs | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index 6a0ae32466..21f7a7d846 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -141,7 +141,7 @@ impl super::Validator for Validator { if Instant::now().duration_since(*entry.value()) < self.0.ttl { return Ok(false); } else { - self.0.banned_solvers.remove(&driver.submission_address); + self.0.banned_solvers.remove(solver); } } diff --git a/crates/autopilot/src/domain/competition/participation_guard/mod.rs b/crates/autopilot/src/domain/competition/participation_guard/mod.rs index 5564cc07f6..eb252f94a1 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/mod.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/mod.rs @@ -55,9 +55,9 @@ impl SolverParticipationGuard { /// the following order: /// 1. DB-based validator: operates fast since it uses in-memory cache. /// 2. Onchain-based validator: only then calls the Authenticator contract. - pub async fn can_participate(&self, driver: &infra::Driver) -> anyhow::Result { + pub async fn can_participate(&self, solver: ð::Address) -> anyhow::Result { for validator in &self.0.validators { - if !validator.is_allowed(driver).await? { + if !validator.is_allowed(solver).await? { return Ok(false); } } @@ -68,5 +68,5 @@ impl SolverParticipationGuard { #[async_trait::async_trait] trait Validator: Send + Sync { - async fn is_allowed(&self, driver: &infra::Driver) -> anyhow::Result; + async fn is_allowed(&self, solver: ð::Address) -> anyhow::Result; } diff --git a/crates/autopilot/src/domain/competition/participation_guard/onchain.rs b/crates/autopilot/src/domain/competition/participation_guard/onchain.rs index d1327f0254..598da9cf26 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/onchain.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/onchain.rs @@ -1,4 +1,4 @@ -use crate::infra; +use crate::{domain::eth, infra}; /// Calls Authenticator contract to check if a solver has a sufficient /// permission. @@ -8,12 +8,12 @@ pub(super) struct Validator { #[async_trait::async_trait] impl super::Validator for Validator { - async fn is_allowed(&self, solver: &infra::Driver) -> anyhow::Result { + async fn is_allowed(&self, solver: ð::Address) -> anyhow::Result { Ok(self .eth .contracts() .authenticator() - .is_solver(solver.submission_address.0) + .is_solver(solver.0) .call() .await?) } diff --git a/crates/autopilot/src/run_loop.rs b/crates/autopilot/src/run_loop.rs index 4102f675ea..492ca18751 100644 --- a/crates/autopilot/src/run_loop.rs +++ b/crates/autopilot/src/run_loop.rs @@ -736,7 +736,7 @@ impl RunLoop { request: &solve::Request, ) -> Result>, SolveError> { - let can_participate = self.solver_participation_guard.can_participate(driver).await.map_err(|err| { + let can_participate = self.solver_participation_guard.can_participate(&driver.submission_address).await.map_err(|err| { tracing::error!(?err, driver = %driver.name, ?driver.submission_address, "solver participation check failed"); SolveError::SolverDenyListed } From bb9059eaba96859de229ce30f215ffe075c6e0fd Mon Sep 17 00:00:00 2001 From: ilya Date: Fri, 31 Jan 2025 16:13:45 +0000 Subject: [PATCH 25/82] Send metrics about each found solver --- .../competition/participation_guard/db.rs | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index 8ddbb28f5f..f43caa804c 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -34,11 +34,6 @@ impl Validator { last_auctions_count: u32, drivers_by_address: HashMap>, ) -> Self { - // Keep only drivers that accept unsettled blocking. - let drivers_by_address = drivers_by_address - .into_iter() - .filter(|(_, driver)| driver.accepts_unsettled_blocking) - .collect(); let self_ = Self(Arc::new(Inner { db, banned_solvers: Default::default(), @@ -74,10 +69,18 @@ impl Validator { .into_iter() .map(|solver| { let address = eth::Address(solver.0.into()); - if let Some(driver) = self_.0.drivers_by_address.get(&address) { - Metrics::get() - .non_settling_solver - .with_label_values(&[&driver.name]); + match self_.0.drivers_by_address.get(&address) { + Some(driver) => { + Metrics::get() + .non_settling_solver + .with_label_values(&[&driver.name]); + } + None => { + tracing::warn!( + ?address, + "unrecognized driver in non-settling solvers", + ); + } } address From 6787d3453fabe99af99374f417c8db2fd8cc6fb6 Mon Sep 17 00:00:00 2001 From: ilya Date: Fri, 31 Jan 2025 16:24:16 +0000 Subject: [PATCH 26/82] Cache only accepted solvers --- .../src/domain/competition/participation_guard/db.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index f43caa804c..9b2731ddec 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -94,7 +94,13 @@ impl Validator { .into_iter() // Check if solver accepted this feature. This should be removed once a CIP is // approved. - .filter(|solver| self_.0.drivers_by_address.contains_key(solver)) + .filter_map(|solver| { + self_ + .0 + .drivers_by_address + .get(&solver) + .filter(|driver| driver.accepts_unsettled_blocking).map(|_| solver) + }) .for_each(|solver| { self_.0.banned_solvers.insert(solver, now); }); From 921692f5eab25efe9bacb430839883b62f7bf3f7 Mon Sep 17 00:00:00 2001 From: ilya Date: Fri, 31 Jan 2025 16:28:08 +0000 Subject: [PATCH 27/82] Notify only accepted solvers --- .../src/domain/competition/participation_guard/db.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index 9b6c05fc48..398802fddd 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -123,7 +123,11 @@ impl Validator { .map(|solver| { let self_ = self.0.clone(); async move { - if let Some(driver) = self_.drivers_by_address.get(&solver) { + if let Some(driver) = self_ + .drivers_by_address + .get(&solver) + .filter(|driver| driver.accepts_unsettled_blocking) + { if let Err(err) = driver .notify( &infra::solvers::dto::notify::Request::UnsettledConsecutiveAuctions, From f3b64155f584354a76b0ee4a193007ee6d1226df Mon Sep 17 00:00:00 2001 From: ilya Date: Fri, 31 Jan 2025 16:53:40 +0000 Subject: [PATCH 28/82] Minor refactoring --- .../competition/participation_guard/db.rs | 48 +++++++++---------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index 398802fddd..6af1239d4d 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -89,23 +89,30 @@ impl Validator { .collect::>(); tracing::debug!(?non_settling_solvers, "found non-settling solvers"); - self_.notify_solvers(&non_settling_solvers); - let now = Instant::now(); - non_settling_solvers + let drivers = non_settling_solvers .into_iter() - // Check if solver accepted this feature. This should be removed once a CIP is - // approved. .filter_map(|solver| { self_ .0 .drivers_by_address .get(&solver) - .filter(|driver| driver.accepts_unsettled_blocking).map(|_| solver) + // Check if solver accepted this feature. This should be removed once a CIP is + // approved. + .filter(|driver| driver.accepts_unsettled_blocking) + .cloned() }) - .for_each(|solver| { - self_.0.banned_solvers.insert(solver, now); - }); + .collect::>(); + + Self::notify_solvers(&drivers); + + let now = Instant::now(); + for driver in drivers { + self_ + .0 + .banned_solvers + .insert(driver.submission_address, now); + } } Err(err) => { tracing::warn!(?err, "error while searching for non-settling solvers") @@ -116,26 +123,17 @@ impl Validator { } /// Try to notify all the non-settling solvers in a background task. - fn notify_solvers(&self, non_settling_solvers: &[eth::Address]) { - let futures = non_settling_solvers + fn notify_solvers(non_settling_drivers: &[Arc]) { + let futures = non_settling_drivers .iter() .cloned() - .map(|solver| { - let self_ = self.0.clone(); + .map(|driver| { async move { - if let Some(driver) = self_ - .drivers_by_address - .get(&solver) - .filter(|driver| driver.accepts_unsettled_blocking) + if let Err(err) = driver + .notify(&infra::solvers::dto::notify::Request::UnsettledConsecutiveAuctions) + .await { - if let Err(err) = driver - .notify( - &infra::solvers::dto::notify::Request::UnsettledConsecutiveAuctions, - ) - .await - { - tracing::debug!(?solver, ?err, "unable to notify external solver"); - } + tracing::debug!(solver = ?driver.name, ?err, "unable to notify external solver"); } } }) From 805946c4e7cfe1ad1830086c3c56764a91929de7 Mon Sep 17 00:00:00 2001 From: ilya Date: Fri, 31 Jan 2025 17:11:11 +0000 Subject: [PATCH 29/82] DB statistics guard --- crates/autopilot/src/database/competition.rs | 22 ++ .../competition/participation_guard/db.rs | 168 +++++++++------ crates/autopilot/src/domain/mod.rs | 4 +- .../autopilot/src/infra/solvers/dto/notify.rs | 4 +- crates/database/src/solver_competition.rs | 196 ++++++++++++++++++ 5 files changed, 333 insertions(+), 61 deletions(-) diff --git a/crates/autopilot/src/database/competition.rs b/crates/autopilot/src/database/competition.rs index 682ac81d1d..ebe533e01f 100644 --- a/crates/autopilot/src/database/competition.rs +++ b/crates/autopilot/src/database/competition.rs @@ -163,4 +163,26 @@ impl super::Postgres { .await .context("solver_competition::find_non_settling_solvers") } + + pub async fn find_low_settling_solvers( + &self, + last_auctions_count: u32, + current_block: u64, + min_success_ratio: f64, + ) -> anyhow::Result> { + let mut ex = self.pool.acquire().await.context("acquire")?; + let _timer = super::Metrics::get() + .database_queries + .with_label_values(&["find_low_settling_solvers"]) + .start_timer(); + + database::solver_competition::find_low_settling_solvers( + &mut ex, + last_auctions_count, + current_block, + min_success_ratio, + ) + .await + .context("solver_competition::find_low_settling_solvers") + } } diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index 21f7a7d846..f0976a1200 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -2,15 +2,16 @@ use { crate::{ database::Postgres, domain::{eth, Metrics}, - infra, + infra::{self, solvers::dto}, }, ethrpc::block_stream::CurrentBlockWatcher, futures::future::join_all, std::{ - collections::HashMap, + collections::{HashMap, HashSet}, sync::Arc, time::{Duration, Instant}, }, + tokio::join, }; /// Checks the DB by searching for solvers that won N last consecutive auctions @@ -35,11 +36,6 @@ impl Validator { last_auctions_count: u32, drivers_by_address: HashMap>, ) -> Self { - // Keep only drivers that accept unsettled blocking. - let drivers_by_address = drivers_by_address - .into_iter() - .filter(|(_, driver)| driver.accepts_unsettled_blocking) - .collect(); let self_ = Self(Arc::new(Inner { db, banned_solvers: Default::default(), @@ -64,65 +60,75 @@ impl Validator { tokio::spawn(async move { while settlement_updates_receiver.recv().await.is_some() { let current_block = current_block.borrow().number; - match self_ - .0 - .db - .find_non_settling_solvers(self_.0.last_auctions_count, current_block) - .await - { - Ok(non_settling_solvers) => { - let non_settling_solvers = non_settling_solvers - .into_iter() - .map(|solver| { - let address = eth::Address(solver.0.into()); - if let Some(driver) = self_.0.drivers_by_address.get(&address) { - Metrics::get() - .non_settling_solver - .with_label_values(&[&driver.name]); - } - - address - }) - .collect::>(); - - tracing::debug!(?non_settling_solvers, "found non-settling solvers"); - self_.notify_solvers(&non_settling_solvers); - - let now = Instant::now(); - non_settling_solvers - .into_iter() - // Check if solver accepted this feature. This should be removed once a CIP is - // approved. - .filter(|solver| self_.0.drivers_by_address.contains_key(solver)) - .for_each(|solver| { - self_.0.banned_solvers.insert(solver, now); - }); - } - Err(err) => { - tracing::warn!(?err, "error while searching for non-settling solvers") - } - } + + let (non_settling_solvers, mut low_settling_solvers) = join!( + self_.find_non_settling_solvers(current_block), + self_.find_low_settling_solvers(current_block) + ); + // Non-settling issue has a higher priority, remove duplicates from low-settling + // solvers. + low_settling_solvers.retain(|solver| !non_settling_solvers.contains(solver)); + + self_.post_process( + &non_settling_solvers, + &dto::notify::Request::UnsettledConsecutiveAuctions, + ); + self_.post_process( + &low_settling_solvers, + &dto::notify::Request::LowSettlingRate, + ); } }); } + async fn find_non_settling_solvers(&self, current_block: u64) -> HashSet { + let last_auctions_count = self.0.last_auctions_count; + match self + .0 + .db + .find_non_settling_solvers(last_auctions_count, current_block) + .await + { + Ok(solvers) => solvers + .into_iter() + .map(|solver| eth::Address(solver.0.into())) + .collect(), + Err(err) => { + tracing::warn!(?err, "error while searching for non-settling solvers"); + Default::default() + } + } + } + + async fn find_low_settling_solvers(&self, current_block: u64) -> HashSet { + let last_auctions_count = self.0.last_auctions_count; + match self + .0 + .db + .find_low_settling_solvers(last_auctions_count, current_block, 1.0) + .await + { + Ok(solvers) => solvers + .into_iter() + .map(|solver| eth::Address(solver.0.into())) + .collect(), + Err(err) => { + tracing::warn!(?err, "error while searching for low-settling solvers"); + Default::default() + } + } + } + /// Try to notify all the non-settling solvers in a background task. - fn notify_solvers(&self, non_settling_solvers: &[eth::Address]) { - let futures = non_settling_solvers + fn notify_solvers(drivers: &[Arc], request: &dto::notify::Request) { + let futures = drivers .iter() .cloned() - .map(|solver| { - let self_ = self.0.clone(); + .map(|driver| { + let request = request.clone(); async move { - if let Some(driver) = self_.drivers_by_address.get(&solver) { - if let Err(err) = driver - .notify( - &infra::solvers::dto::notify::Request::UnsettledConsecutiveAuctions, - ) - .await - { - tracing::debug!(?solver, ?err, "unable to notify external solver"); - } + if let Err(err) = driver.notify(&request).await { + tracing::debug!(solver = ?driver.name, ?err, "unable to notify external solver"); } } }) @@ -132,6 +138,52 @@ impl Validator { join_all(futures).await; }); } + + /// Updates the cache and notifies the solvers. + fn post_process(&self, solvers: &HashSet, request: &dto::notify::Request) { + if solvers.is_empty() { + return; + } + + let drivers = solvers + .iter() + .filter_map(|solver| self.0.drivers_by_address.get(solver).cloned()) + .collect::>(); + + let log_message = match request { + dto::notify::Request::UnsettledConsecutiveAuctions => "found non-settling solvers", + dto::notify::Request::LowSettlingRate => "found low-settling solvers", + }; + let solver_names = drivers + .iter() + .map(|driver| driver.name.clone()) + .collect::>(); + tracing::debug!(solvers = ?solver_names, log_message); + + let reason = match request { + dto::notify::Request::UnsettledConsecutiveAuctions => "non_settling", + dto::notify::Request::LowSettlingRate => "low_settling", + }; + + for solver in solver_names { + Metrics::get() + .banned_solver + .with_label_values(&[&solver, reason]); + } + + let drivers = drivers + .into_iter() + // Notify and block only solvers that accept unsettled blocking feature. This should be removed once a CIP is approved. + .filter(|driver| driver.accepts_unsettled_blocking) + .collect::>(); + + Self::notify_solvers(&drivers, request); + + let now = Instant::now(); + for driver in drivers { + self.0.banned_solvers.insert(driver.submission_address, now); + } + } } #[async_trait::async_trait] diff --git a/crates/autopilot/src/domain/mod.rs b/crates/autopilot/src/domain/mod.rs index ad7d0077af..64fdfbf603 100644 --- a/crates/autopilot/src/domain/mod.rs +++ b/crates/autopilot/src/domain/mod.rs @@ -20,8 +20,8 @@ pub use { pub struct Metrics { /// How many times the solver marked as non-settling based on the database /// statistics. - #[metric(labels("solver"))] - pub non_settling_solver: prometheus::IntCounterVec, + #[metric(labels("solver", "reason"))] + pub banned_solver: prometheus::IntCounterVec, } impl Metrics { diff --git a/crates/autopilot/src/infra/solvers/dto/notify.rs b/crates/autopilot/src/infra/solvers/dto/notify.rs index 08165964fd..e72ae90a12 100644 --- a/crates/autopilot/src/infra/solvers/dto/notify.rs +++ b/crates/autopilot/src/infra/solvers/dto/notify.rs @@ -1,9 +1,11 @@ use {serde::Serialize, serde_with::serde_as}; #[serde_as] -#[derive(Debug, Serialize)] +#[derive(Clone, Debug, Serialize)] #[serde(rename_all = "camelCase")] pub enum Request { /// The driver won multiple consecutive auctions but never settled them. UnsettledConsecutiveAuctions, + /// Driver's success settling rate is below the threshold. + LowSettlingRate, } diff --git a/crates/database/src/solver_competition.rs b/crates/database/src/solver_competition.rs index 0139610671..22844d75cc 100644 --- a/crates/database/src/solver_competition.rs +++ b/crates/database/src/solver_competition.rs @@ -148,6 +148,48 @@ FROM consistent_solvers; .await } +pub async fn find_low_settling_solvers( + ex: &mut PgConnection, + last_auctions_count: u32, + current_block: u64, + min_success_ratio: f64, +) -> Result, sqlx::Error> { + const QUERY: &str = r#" +WITH + last_auctions AS ( + SELECT ps.auction_id, ps.solver + FROM ( + SELECT DISTINCT ca.id AS auction_id + FROM competition_auctions ca + WHERE ca.deadline <= $1 + ORDER BY ca.id DESC + LIMIT $2 + ) latest_auctions + JOIN proposed_solutions ps ON ps.auction_id = latest_auctions.auction_id + WHERE ps.is_winner = true + ), + solver_settlement_counts AS ( + SELECT la.solver, + COUNT(DISTINCT la.auction_id) AS total_wins, + COUNT(DISTINCT s.auction_id) AS total_settlements + FROM last_auctions la + LEFT JOIN settlements s + ON la.auction_id = s.auction_id AND la.solver = s.solver + GROUP BY la.solver + ) +SELECT solver +FROM solver_settlement_counts +WHERE (total_settlements::decimal / NULLIF(total_wins, 0)) < $3; + "#; + + sqlx::query_scalar(QUERY) + .bind(sqlx::types::BigDecimal::from(current_block)) + .bind(i64::from(last_auctions_count)) + .bind(min_success_ratio) + .fetch_all(ex) + .await +} + #[derive(Clone, Debug, PartialEq, Default)] pub struct Solution { // Unique Id generated by the autopilot to uniquely identify the solution within Auction @@ -748,4 +790,158 @@ mod tests { .unwrap(); assert!(result.is_empty()); } + + #[tokio::test] + #[ignore] + async fn postgres_low_settling_solvers_roundtrip() { + let mut db = PgConnection::connect("postgresql://").await.unwrap(); + let mut db = db.begin().await.unwrap(); + crate::clear_DANGER_(&mut db).await.unwrap(); + + let deadline_block = 2u64; + let last_auctions_count = 100i64; + let min_ratio = 0.4; + let mut solution_uid = 0; + + for auction_id in 1..=10 { + let auction = auction::Auction { + id: auction_id, + block: auction_id, + deadline: i64::try_from(deadline_block).unwrap(), + order_uids: Default::default(), + price_tokens: Default::default(), + price_values: Default::default(), + surplus_capturing_jit_order_owners: Default::default(), + }; + auction::save(&mut db, auction).await.unwrap(); + } + + // Settles only 20% of won auctions + let low_settling_solver = ByteArray([1u8; 20]); + for auction_id in 1..=5 { + solution_uid += 1; + let solutions = vec![Solution { + uid: solution_uid, + id: auction_id.into(), + solver: low_settling_solver, + is_winner: true, + score: Default::default(), + orders: Default::default(), + price_tokens: Default::default(), + price_values: Default::default(), + }]; + save_solutions(&mut db, auction_id, &solutions) + .await + .unwrap(); + } + let event = EventIndex { + block_number: 1, + log_index: 0, + }; + let settlement = Settlement { + solver: low_settling_solver, + transaction_hash: ByteArray([0u8; 32]), + }; + events::insert_settlement(&mut db, &event, &settlement) + .await + .unwrap(); + settlements::update_settlement_auction(&mut db, 1, 0, 1) + .await + .unwrap(); + + // Settles 0% of won auctions + let non_settling_solver = ByteArray([2u8; 20]); + for auction_id in 1..=5 { + solution_uid += 1; + let solutions = vec![Solution { + uid: solution_uid, + id: auction_id.into(), + solver: non_settling_solver, + is_winner: true, + score: Default::default(), + orders: Default::default(), + price_tokens: Default::default(), + price_values: Default::default(), + }]; + save_solutions(&mut db, auction_id, &solutions) + .await + .unwrap(); + } + + // Settled 40% of won auctions + let settling_solver = ByteArray([3u8; 20]); + for auction_id in 1..=5 { + solution_uid += 1; + let solutions = vec![Solution { + uid: solution_uid, + id: auction_id.into(), + solver: settling_solver, + is_winner: true, + score: Default::default(), + orders: Default::default(), + price_tokens: Default::default(), + price_values: Default::default(), + }]; + save_solutions(&mut db, auction_id, &solutions) + .await + .unwrap(); + } + for auction_id in 2..=3 { + let event = EventIndex { + block_number: auction_id, + log_index: 0, + }; + let settlement = Settlement { + solver: settling_solver, + transaction_hash: ByteArray([u8::try_from(auction_id).unwrap(); 32]), + }; + events::insert_settlement(&mut db, &event, &settlement) + .await + .unwrap(); + settlements::update_settlement_auction(&mut db, auction_id, 0, auction_id) + .await + .unwrap(); + } + + let result = find_low_settling_solvers( + &mut db, + u32::try_from(last_auctions_count).unwrap(), + deadline_block, + min_ratio, + ) + .await + .unwrap(); + + assert_eq!(result.len(), 2); + assert!(result.contains(&low_settling_solver)); + assert!(result.contains(&non_settling_solver)); + + // Low settling solver settles another auction + let event = EventIndex { + block_number: 2, + log_index: 1, + }; + let settlement = Settlement { + solver: low_settling_solver, + transaction_hash: ByteArray([2u8; 32]), + }; + events::insert_settlement(&mut db, &event, &settlement) + .await + .unwrap(); + settlements::update_settlement_auction(&mut db, 2, 1, 2) + .await + .unwrap(); + + let result = find_low_settling_solvers( + &mut db, + u32::try_from(last_auctions_count).unwrap(), + deadline_block, + min_ratio, + ) + .await + .unwrap(); + + // Now, it is not a low-settling solver anymore + assert_eq!(result, vec![non_settling_solver]); + } } From a2710c695f7c42644fdbe94a1d2d4fd3ac420458 Mon Sep 17 00:00:00 2001 From: ilya Date: Fri, 31 Jan 2025 17:21:47 +0000 Subject: [PATCH 30/82] Refactoring --- .../competition/participation_guard/db.rs | 42 ++++++++----------- 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index 9b2731ddec..7f517debb7 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -65,41 +65,35 @@ impl Validator { .await { Ok(non_settling_solvers) => { - let non_settling_solvers = non_settling_solvers + let non_settling_drivers = non_settling_solvers .into_iter() - .map(|solver| { + .filter_map(|solver| { let address = eth::Address(solver.0.into()); - match self_.0.drivers_by_address.get(&address) { - Some(driver) => { - Metrics::get() - .non_settling_solver - .with_label_values(&[&driver.name]); - } - None => { - tracing::warn!( - ?address, - "unrecognized driver in non-settling solvers", - ); - } + if let Some(driver) = self_.0.drivers_by_address.get(&address) { + Metrics::get() + .non_settling_solver + .with_label_values(&[&driver.name]); + Some(driver.clone()) + } else { + None } - - address }) .collect::>(); - tracing::debug!(?non_settling_solvers, "found non-settling solvers"); + let non_settling_solver_names = non_settling_drivers + .iter() + .map(|driver| driver.name.clone()) + .collect::>(); + + tracing::debug!(solvers = ?non_settling_solver_names, "found non-settling solvers"); let now = Instant::now(); - non_settling_solvers + non_settling_drivers .into_iter() // Check if solver accepted this feature. This should be removed once a CIP is // approved. - .filter_map(|solver| { - self_ - .0 - .drivers_by_address - .get(&solver) - .filter(|driver| driver.accepts_unsettled_blocking).map(|_| solver) + .filter_map(|driver| { + driver.accepts_unsettled_blocking.then_some(driver.submission_address) }) .for_each(|solver| { self_.0.banned_solvers.insert(solver, now); From 7ab2a387228e7377747ae486586515f21018794c Mon Sep 17 00:00:00 2001 From: ilya Date: Fri, 31 Jan 2025 17:29:05 +0000 Subject: [PATCH 31/82] Naming --- .../src/domain/competition/participation_guard/db.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index f0976a1200..12e5aa0bdf 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -171,16 +171,16 @@ impl Validator { .with_label_values(&[&solver, reason]); } - let drivers = drivers + let non_settling_drivers = drivers .into_iter() // Notify and block only solvers that accept unsettled blocking feature. This should be removed once a CIP is approved. .filter(|driver| driver.accepts_unsettled_blocking) .collect::>(); - Self::notify_solvers(&drivers, request); + Self::notify_solvers(&non_settling_drivers, request); let now = Instant::now(); - for driver in drivers { + for driver in non_settling_drivers { self.0.banned_solvers.insert(driver.submission_address, now); } } From 43e650f18bd59826eb99190aec464698809e42e4 Mon Sep 17 00:00:00 2001 From: ilya Date: Fri, 31 Jan 2025 17:40:12 +0000 Subject: [PATCH 32/82] Banned notification --- .../infra/api/routes/notify/dto/notify_request.rs | 2 +- crates/driver/src/infra/notify/mod.rs | 9 ++++++++- crates/driver/src/infra/notify/notification.rs | 11 ++++++++--- crates/driver/src/infra/solver/dto/notification.rs | 13 ++++++++++++- crates/solvers/openapi.yml | 2 +- 5 files changed, 30 insertions(+), 7 deletions(-) diff --git a/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs b/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs index 22be6cbb00..16afa72814 100644 --- a/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs +++ b/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs @@ -12,7 +12,7 @@ impl From for notify::Kind { fn from(value: NotifyRequest) -> Self { match value { NotifyRequest::UnsettledConsecutiveAuctions => { - notify::Kind::UnsettledConsecutiveAuctions + notify::Kind::Banned(notify::BanReason::UnsettledConsecutiveAuctions) } } } diff --git a/crates/driver/src/infra/notify/mod.rs b/crates/driver/src/infra/notify/mod.rs index 72446e96bf..ae18f26aa6 100644 --- a/crates/driver/src/infra/notify/mod.rs +++ b/crates/driver/src/infra/notify/mod.rs @@ -5,7 +5,14 @@ use { mod notification; -pub use notification::{Kind, Notification, ScoreKind, Settlement, SimulationSucceededAtLeastOnce}; +pub use notification::{ + BanReason, + Kind, + Notification, + ScoreKind, + Settlement, + SimulationSucceededAtLeastOnce, +}; use { super::simulator, crate::domain::{eth, mempools::Error}, diff --git a/crates/driver/src/infra/notify/notification.rs b/crates/driver/src/infra/notify/notification.rs index ca123bb5e8..5e6323dbdf 100644 --- a/crates/driver/src/infra/notify/notification.rs +++ b/crates/driver/src/infra/notify/notification.rs @@ -45,9 +45,8 @@ pub enum Kind { DriverError(String), /// On-chain solution postprocessing timed out. PostprocessingTimedOut, - /// The solver won multiple consecutive auctions but none of the settlement - /// succeeded. - UnsettledConsecutiveAuctions, + /// The solver has been banned for a specific reason. + Banned(BanReason), } #[derive(Debug)] @@ -61,6 +60,12 @@ pub enum ScoreKind { MissingPrice(TokenAddress), } +#[derive(Debug)] +pub enum BanReason { + /// The driver won multiple consecutive auctions but never settled them. + UnsettledConsecutiveAuctions, +} + #[derive(Debug)] pub enum Settlement { /// Winning solver settled successfully transaction onchain. diff --git a/crates/driver/src/infra/solver/dto/notification.rs b/crates/driver/src/infra/solver/dto/notification.rs index 2c70f50f5a..a352c712b1 100644 --- a/crates/driver/src/infra/solver/dto/notification.rs +++ b/crates/driver/src/infra/solver/dto/notification.rs @@ -61,7 +61,11 @@ impl Notification { notify::Settlement::Expired => Kind::Expired, }, notify::Kind::PostprocessingTimedOut => Kind::PostprocessingTimedOut, - notify::Kind::UnsettledConsecutiveAuctions => Kind::UnsettledConsecutiveAuctions, + notify::Kind::Banned(reason) => Kind::Banned(match reason { + notify::BanReason::UnsettledConsecutiveAuctions => { + BanReason::UnsettledConsecutiveAuctions + } + }), }, } } @@ -145,6 +149,13 @@ pub enum Kind { Expired, Fail, PostprocessingTimedOut, + Banned(BanReason), +} + +#[serde_as] +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase", tag = "reason")] +pub enum BanReason { UnsettledConsecutiveAuctions, } diff --git a/crates/solvers/openapi.yml b/crates/solvers/openapi.yml index 135f47cfaa..0b79459413 100644 --- a/crates/solvers/openapi.yml +++ b/crates/solvers/openapi.yml @@ -89,7 +89,7 @@ paths: - cancelled - fail - postprocessingTimedOut - - unsettledConsecutiveAuctions + - banned responses: "200": description: notification successfully received. From 160e2d920559e98ce7446fb92c2226cb1386fc92 Mon Sep 17 00:00:00 2001 From: ilya Date: Fri, 31 Jan 2025 17:42:30 +0000 Subject: [PATCH 33/82] New ban reason notification --- .../driver/src/infra/api/routes/notify/dto/notify_request.rs | 5 +++++ crates/driver/src/infra/notify/notification.rs | 2 ++ crates/driver/src/infra/solver/dto/notification.rs | 2 ++ 3 files changed, 9 insertions(+) diff --git a/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs b/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs index 16afa72814..bef312eef2 100644 --- a/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs +++ b/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs @@ -6,6 +6,8 @@ use {crate::infra::notify, serde::Deserialize, serde_with::serde_as}; pub enum NotifyRequest { /// The driver won multiple consecutive auctions but never settled them. UnsettledConsecutiveAuctions, + /// Driver's success settling rate is below the threshold. + LowSettlingRate, } impl From for notify::Kind { @@ -14,6 +16,9 @@ impl From for notify::Kind { NotifyRequest::UnsettledConsecutiveAuctions => { notify::Kind::Banned(notify::BanReason::UnsettledConsecutiveAuctions) } + NotifyRequest::LowSettlingRate => { + notify::Kind::Banned(notify::BanReason::LowSettlingRate) + } } } } diff --git a/crates/driver/src/infra/notify/notification.rs b/crates/driver/src/infra/notify/notification.rs index 5e6323dbdf..6f990199f5 100644 --- a/crates/driver/src/infra/notify/notification.rs +++ b/crates/driver/src/infra/notify/notification.rs @@ -64,6 +64,8 @@ pub enum ScoreKind { pub enum BanReason { /// The driver won multiple consecutive auctions but never settled them. UnsettledConsecutiveAuctions, + /// Driver's success settling rate is below the threshold. + LowSettlingRate, } #[derive(Debug)] diff --git a/crates/driver/src/infra/solver/dto/notification.rs b/crates/driver/src/infra/solver/dto/notification.rs index a352c712b1..9408b81c17 100644 --- a/crates/driver/src/infra/solver/dto/notification.rs +++ b/crates/driver/src/infra/solver/dto/notification.rs @@ -65,6 +65,7 @@ impl Notification { notify::BanReason::UnsettledConsecutiveAuctions => { BanReason::UnsettledConsecutiveAuctions } + notify::BanReason::LowSettlingRate => BanReason::LowSettlingRate, }), }, } @@ -157,6 +158,7 @@ pub enum Kind { #[serde(rename_all = "camelCase", tag = "reason")] pub enum BanReason { UnsettledConsecutiveAuctions, + LowSettlingRate, } type BlockNo = u64; From 597d26879533a7221496cc9044ecf3ccddfd9688 Mon Sep 17 00:00:00 2001 From: ilya Date: Fri, 31 Jan 2025 17:46:37 +0000 Subject: [PATCH 34/82] min_settlement_success_rate --- crates/autopilot/src/arguments.rs | 6 ++++++ .../domain/competition/participation_guard/db.rs | 13 +++++++++---- .../domain/competition/participation_guard/mod.rs | 1 + 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/crates/autopilot/src/arguments.rs b/crates/autopilot/src/arguments.rs index e70b5ba0a3..0ad9be9d38 100644 --- a/crates/autopilot/src/arguments.rs +++ b/crates/autopilot/src/arguments.rs @@ -266,6 +266,12 @@ pub struct DbBasedSolverParticipationGuardConfig { /// The number of last auctions to check solver participation eligibility. #[clap(long, env, default_value = "3")] pub solver_last_auctions_participation_count: u32, + + /// A minimum success rate for a solver to be considered eligible for + /// participation in the competition. Otherwise, the solver will be + /// banned. + #[clap(long, env, default_value = "0.1")] + pub solver_min_settlement_success_rate: f64, } impl std::fmt::Display for Arguments { diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index 12e5aa0bdf..8a821f10fd 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -24,6 +24,7 @@ struct Inner { banned_solvers: dashmap::DashMap, ttl: Duration, last_auctions_count: u32, + min_settlement_success_rate: f64, drivers_by_address: HashMap>, } @@ -34,6 +35,7 @@ impl Validator { settlement_updates_receiver: tokio::sync::mpsc::UnboundedReceiver<()>, ttl: Duration, last_auctions_count: u32, + min_settlement_success_rate: f64, drivers_by_address: HashMap>, ) -> Self { let self_ = Self(Arc::new(Inner { @@ -41,6 +43,7 @@ impl Validator { banned_solvers: Default::default(), ttl, last_auctions_count, + min_settlement_success_rate, drivers_by_address, })); @@ -82,11 +85,10 @@ impl Validator { } async fn find_non_settling_solvers(&self, current_block: u64) -> HashSet { - let last_auctions_count = self.0.last_auctions_count; match self .0 .db - .find_non_settling_solvers(last_auctions_count, current_block) + .find_non_settling_solvers(self.0.last_auctions_count, current_block) .await { Ok(solvers) => solvers @@ -101,11 +103,14 @@ impl Validator { } async fn find_low_settling_solvers(&self, current_block: u64) -> HashSet { - let last_auctions_count = self.0.last_auctions_count; match self .0 .db - .find_low_settling_solvers(last_auctions_count, current_block, 1.0) + .find_low_settling_solvers( + self.0.last_auctions_count, + current_block, + self.0.min_settlement_success_rate, + ) .await { Ok(solvers) => solvers diff --git a/crates/autopilot/src/domain/competition/participation_guard/mod.rs b/crates/autopilot/src/domain/competition/participation_guard/mod.rs index eb252f94a1..8af060d8a6 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/mod.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/mod.rs @@ -39,6 +39,7 @@ impl SolverParticipationGuard { settlement_updates_receiver, db_based_validator_config.solver_blacklist_cache_ttl, db_based_validator_config.solver_last_auctions_participation_count, + db_based_validator_config.solver_min_settlement_success_rate, drivers_by_address, ); validators.push(Box::new(database_solver_participation_validator)); From 7c87087b494573cf29bd0e4de8e5d9db0319056e Mon Sep 17 00:00:00 2001 From: ilya Date: Fri, 31 Jan 2025 18:09:03 +0000 Subject: [PATCH 35/82] Separate configs --- crates/autopilot/src/arguments.rs | 53 +++++++++++++++---- .../competition/participation_guard/db.rs | 41 +++++++++----- .../competition/participation_guard/mod.rs | 22 ++++---- 3 files changed, 82 insertions(+), 34 deletions(-) diff --git a/crates/autopilot/src/arguments.rs b/crates/autopilot/src/arguments.rs index 0ad9be9d38..5a707ac042 100644 --- a/crates/autopilot/src/arguments.rs +++ b/crates/autopilot/src/arguments.rs @@ -250,22 +250,57 @@ pub struct Arguments { #[derive(Debug, clap::Parser)] pub struct DbBasedSolverParticipationGuardConfig { - /// Enables or disables the solver participation guard + /// The time-to-live for the solver participation blacklist cache. + #[clap(long, env, default_value = "5m", value_parser = humantime::parse_duration)] + pub solver_blacklist_cache_ttl: Duration, + + #[clap(flatten)] + pub non_settling_validator_config: NonSettlingValidatorConfig, + + #[clap(flatten)] + pub low_settling_validator_config: LowSettlingValidatorConfig, +} + +#[derive(Debug, clap::Parser)] +pub struct NonSettlingValidatorConfig { + /// Enables search of non-settling solvers. #[clap( - id = "db_enabled", - long = "db-based-solver-participation-guard-enabled", - env = "DB_BASED_SOLVER_PARTICIPATION_GUARD_ENABLED", + id = "non_settling_solvers_blacklisting_enabled", + long = "non-settling-solvers-blacklisting-enabled", + env = "NON_SETTLING_SOLVERS_BLACKLISTING_ENABLED", default_value = "true" )] pub enabled: bool, - /// The time-to-live for the solver participation blacklist cache. - #[clap(long, env, default_value = "5m", value_parser = humantime::parse_duration)] - pub solver_blacklist_cache_ttl: Duration, + /// The number of last auctions to check solver participation eligibility. + #[clap( + id = "non_settling_last_auctions_participation_count", + long = "non-settling-last-auctions-participation-count", + env = "NON_SETTLING_LAST_AUCTIONS_PARTICIPATION_COUNT", + default_value = "3" + )] + pub last_auctions_participation_count: u32, +} + +#[derive(Debug, clap::Parser)] +pub struct LowSettlingValidatorConfig { + /// Enables search of non-settling solvers. + #[clap( + id = "low_settling_solvers_blacklisting_enabled", + long = "low-settling-solvers-blacklisting-enabled", + env = "LOW_SETTLING_SOLVERS_BLACKLISTING_ENABLED", + default_value = "true" + )] + pub enabled: bool, /// The number of last auctions to check solver participation eligibility. - #[clap(long, env, default_value = "3")] - pub solver_last_auctions_participation_count: u32, + #[clap( + id = "low_settling_last_auctions_participation_count", + long = "low-settling-last-auctions-participation-count", + env = "LOW_SETTLING_LAST_AUCTIONS_PARTICIPATION_COUNT", + default_value = "100" + )] + pub last_auctions_participation_count: u32, /// A minimum success rate for a solver to be considered eligible for /// participation in the competition. Otherwise, the solver will be diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index 8a821f10fd..6c8578dbf4 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -1,5 +1,10 @@ use { crate::{ + arguments::{ + DbBasedSolverParticipationGuardConfig, + LowSettlingValidatorConfig, + NonSettlingValidatorConfig, + }, database::Postgres, domain::{eth, Metrics}, infra::{self, solvers::dto}, @@ -15,7 +20,8 @@ use { }; /// Checks the DB by searching for solvers that won N last consecutive auctions -/// but never settled any of them. +/// and either never settled any of them or their settlement success rate is +/// lower than `min_settlement_success_rate`. #[derive(Clone)] pub(super) struct Validator(Arc); @@ -23,8 +29,8 @@ struct Inner { db: Postgres, banned_solvers: dashmap::DashMap, ttl: Duration, - last_auctions_count: u32, - min_settlement_success_rate: f64, + non_settling_config: NonSettlingValidatorConfig, + low_settling_config: LowSettlingValidatorConfig, drivers_by_address: HashMap>, } @@ -33,17 +39,15 @@ impl Validator { db: Postgres, current_block: CurrentBlockWatcher, settlement_updates_receiver: tokio::sync::mpsc::UnboundedReceiver<()>, - ttl: Duration, - last_auctions_count: u32, - min_settlement_success_rate: f64, + db_based_validator_config: DbBasedSolverParticipationGuardConfig, drivers_by_address: HashMap>, ) -> Self { let self_ = Self(Arc::new(Inner { db, banned_solvers: Default::default(), - ttl, - last_auctions_count, - min_settlement_success_rate, + ttl: db_based_validator_config.solver_blacklist_cache_ttl, + non_settling_config: db_based_validator_config.non_settling_validator_config, + low_settling_config: db_based_validator_config.low_settling_validator_config, drivers_by_address, })); @@ -85,10 +89,17 @@ impl Validator { } async fn find_non_settling_solvers(&self, current_block: u64) -> HashSet { + if !self.0.non_settling_config.enabled { + return Default::default(); + } + match self .0 .db - .find_non_settling_solvers(self.0.last_auctions_count, current_block) + .find_non_settling_solvers( + self.0.non_settling_config.last_auctions_participation_count, + current_block, + ) .await { Ok(solvers) => solvers @@ -103,13 +114,19 @@ impl Validator { } async fn find_low_settling_solvers(&self, current_block: u64) -> HashSet { + if !self.0.low_settling_config.enabled { + return Default::default(); + } + match self .0 .db .find_low_settling_solvers( - self.0.last_auctions_count, + self.0.low_settling_config.last_auctions_participation_count, current_block, - self.0.min_settlement_success_rate, + self.0 + .low_settling_config + .solver_min_settlement_success_rate, ) .await { diff --git a/crates/autopilot/src/domain/competition/participation_guard/mod.rs b/crates/autopilot/src/domain/competition/participation_guard/mod.rs index 8af060d8a6..f04243ceb0 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/mod.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/mod.rs @@ -31,19 +31,15 @@ impl SolverParticipationGuard { ) -> Self { let mut validators: Vec> = Vec::new(); - if db_based_validator_config.enabled { - let current_block = eth.current_block().clone(); - let database_solver_participation_validator = db::Validator::new( - db, - current_block, - settlement_updates_receiver, - db_based_validator_config.solver_blacklist_cache_ttl, - db_based_validator_config.solver_last_auctions_participation_count, - db_based_validator_config.solver_min_settlement_success_rate, - drivers_by_address, - ); - validators.push(Box::new(database_solver_participation_validator)); - } + let current_block = eth.current_block().clone(); + let database_solver_participation_validator = db::Validator::new( + db, + current_block, + settlement_updates_receiver, + db_based_validator_config, + drivers_by_address, + ); + validators.push(Box::new(database_solver_participation_validator)); let onchain_solver_participation_validator = onchain::Validator { eth }; validators.push(Box::new(onchain_solver_participation_validator)); From 1f43009f0d9b7df05b53d5a610e25a751c33e332 Mon Sep 17 00:00:00 2001 From: ilya Date: Mon, 3 Feb 2025 11:47:18 +0000 Subject: [PATCH 36/82] Fix the tests --- crates/database/src/lib.rs | 41 +++++++++++++---------- crates/database/src/solver_competition.rs | 19 +++++++---- 2 files changed, 36 insertions(+), 24 deletions(-) diff --git a/crates/database/src/lib.rs b/crates/database/src/lib.rs index ec30cd557d..a185a90779 100644 --- a/crates/database/src/lib.rs +++ b/crates/database/src/lib.rs @@ -50,28 +50,35 @@ pub type PgTransaction<'a> = sqlx::Transaction<'a, sqlx::Postgres>; /// The names of tables we use in the db. pub const TABLES: &[&str] = &[ - "orders", - "trades", - "invalidations", - "last_indexed_blocks", - "quotes", - "settlements", - "presignature_events", - "order_quotes", - "solver_competitions", + "app_data", + "auction_orders", + "auction_participants", + "auction_prices", "auctions", "competition_auctions", - "onchain_placed_orders", "ethflow_orders", - "order_execution", - "interactions", "ethflow_refunds", - "settlement_scores", - "settlement_observations", - "auction_prices", - "auction_participants", - "app_data", + "fee_policies", + "interactions", + "invalidations", "jit_orders", + "last_indexed_blocks", + "onchain_order_invalidations", + "onchain_placed_orders", + "order_execution", + "order_quotes", + "orders", + "presignature_events", + "proposed_jit_orders", + "proposed_solutions", + "proposed_trade_executions", + "quotes", + "settlement_observations", + "settlement_scores", + "settlements", + "solver_competitions", + "surplus_capturing_jit_order_owners", + "trades", ]; /// The names of potentially big volume tables we use in the db. diff --git a/crates/database/src/solver_competition.rs b/crates/database/src/solver_competition.rs index 0139610671..9fc22fa561 100644 --- a/crates/database/src/solver_competition.rs +++ b/crates/database/src/solver_competition.rs @@ -596,6 +596,7 @@ mod tests { let non_settling_solver = ByteArray([1u8; 20]); + let mut solution_uid = 0; let deadline_block = 100u64; let last_auctions_count = 3i64; // competition_auctions @@ -628,11 +629,12 @@ mod tests { } // proposed_solutions - // Non-settling solver wins all auctions within the deadline + // Non-settling solver wins `last_auctions_count` auctions within the deadline for auction_id in 2..=4 { + solution_uid += 1; let solutions = vec![Solution { uid: auction_id, - id: auction_id.into(), + id: solution_uid.into(), solver: non_settling_solver, is_winner: true, score: Default::default(), @@ -645,11 +647,12 @@ mod tests { .unwrap(); } - // Non-settling solver wins not all the auctions within the deadline + // Another non-settling solver wins not all the auctions within the deadline for auction_id in 2..=4 { + solution_uid += 1; let solutions = vec![Solution { uid: auction_id, - id: auction_id.into(), + id: solution_uid.into(), solver: ByteArray([2u8; 20]), is_winner: auction_id != 2, score: Default::default(), @@ -662,16 +665,17 @@ mod tests { .unwrap(); } - // Another non-settling solver has `last_auctions_count` winning auctions but + // One more non-settling solver has `last_auctions_count` winning auctions but // not consecutive for auction_id in 1..=4 { // Break the sequence if auction_id == 2 { continue; } + solution_uid += 1; let solutions = vec![Solution { uid: auction_id, - id: auction_id.into(), + id: solution_uid.into(), solver: ByteArray([3u8; 20]), is_winner: true, score: Default::default(), @@ -687,9 +691,10 @@ mod tests { // One more non-settling solver has `last_auctions_count` winning auctions but // some of them are outside the deadline for auction_id in 3..=5 { + solution_uid += 1; let solutions = vec![Solution { uid: auction_id, - id: auction_id.into(), + id: solution_uid.into(), solver: ByteArray([4u8; 20]), is_winner: true, score: Default::default(), From 366611d2d417b9e941bc52025f9e3ba7ada7c742 Mon Sep 17 00:00:00 2001 From: ilya Date: Fri, 7 Feb 2025 14:54:10 +0000 Subject: [PATCH 37/82] Nits --- crates/autopilot/src/domain/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/autopilot/src/domain/mod.rs b/crates/autopilot/src/domain/mod.rs index ad7d0077af..9d8d9b36db 100644 --- a/crates/autopilot/src/domain/mod.rs +++ b/crates/autopilot/src/domain/mod.rs @@ -18,8 +18,8 @@ pub use { #[derive(prometheus_metric_storage::MetricStorage)] #[metric(subsystem = "domain")] pub struct Metrics { - /// How many times the solver marked as non-settling based on the database - /// statistics. + /// How many times the solver was marked as non-settling based on the + /// database statistics. #[metric(labels("solver"))] pub non_settling_solver: prometheus::IntCounterVec, } From e9a70f541ac47ec0c21fffa6f3d9972dc9369d14 Mon Sep 17 00:00:00 2001 From: ilya Date: Tue, 11 Feb 2025 20:16:59 +0000 Subject: [PATCH 38/82] Trigger updates on the proposed_solution table insert --- .../src/domain/settlement/observer.rs | 21 ++----------------- crates/autopilot/src/run.rs | 12 +++++------ crates/autopilot/src/run_loop.rs | 20 ++++++++++++++---- 3 files changed, 23 insertions(+), 30 deletions(-) diff --git a/crates/autopilot/src/domain/settlement/observer.rs b/crates/autopilot/src/domain/settlement/observer.rs index 0374465a1f..d142646728 100644 --- a/crates/autopilot/src/domain/settlement/observer.rs +++ b/crates/autopilot/src/domain/settlement/observer.rs @@ -19,34 +19,23 @@ use { pub struct Observer { eth: infra::Ethereum, persistence: infra::Persistence, - settlement_updates_sender: tokio::sync::mpsc::UnboundedSender<()>, } impl Observer { /// Creates a new Observer and asynchronously schedules the first update /// run. - pub fn new( - eth: infra::Ethereum, - persistence: infra::Persistence, - settlement_updates_sender: tokio::sync::mpsc::UnboundedSender<()>, - ) -> Self { - Self { - eth, - persistence, - settlement_updates_sender, - } + pub fn new(eth: infra::Ethereum, persistence: infra::Persistence) -> Self { + Self { eth, persistence } } /// Fetches all the available missing data needed for bookkeeping. /// This needs to get called after indexing a new settlement event /// since this code needs that data to already be present in the DB. pub async fn update(&self) { - let mut updated = false; loop { match self.single_update().await { Ok(true) => { tracing::debug!("on settlement event updater ran and processed event"); - updated = true; // There might be more pending updates, continue immediately. continue; } @@ -60,12 +49,6 @@ impl Observer { } } } - if updated { - // Notify the solver participation guard that a settlement has been updated. - if let Err(err) = self.settlement_updates_sender.send(()) { - tracing::error!(?err, "failed to notify solver participation guard"); - } - } } /// Update database for settlement events that have not been processed yet. diff --git a/crates/autopilot/src/run.rs b/crates/autopilot/src/run.rs index e2154a9e28..497fd19af8 100644 --- a/crates/autopilot/src/run.rs +++ b/crates/autopilot/src/run.rs @@ -365,16 +365,13 @@ pub async fn run(args: Arguments) { None }; - let (settlement_updates_sender, settlement_updates_receiver) = + let (competition_updates_sender, competition_updates_receiver) = tokio::sync::mpsc::unbounded_channel(); let persistence = infra::persistence::Persistence::new(args.s3.into().unwrap(), Arc::new(db.clone())).await; - let settlement_observer = crate::domain::settlement::Observer::new( - eth.clone(), - persistence.clone(), - settlement_updates_sender, - ); + let settlement_observer = + crate::domain::settlement::Observer::new(eth.clone(), persistence.clone()); let settlement_contract_start_index = if let Some(DeploymentInformation::BlockNumber(settlement_contract_start_index)) = eth.contracts().settlement().deployment_information() @@ -580,7 +577,7 @@ pub async fn run(args: Arguments) { let solver_participation_guard = SolverParticipationGuard::new( eth.clone(), db.clone(), - settlement_updates_receiver, + competition_updates_receiver, args.db_based_solver_participation_guard, drivers .iter() @@ -598,6 +595,7 @@ pub async fn run(args: Arguments) { trusted_tokens, liveness.clone(), Arc::new(maintenance), + competition_updates_sender, ); run.run_forever().await; } diff --git a/crates/autopilot/src/run_loop.rs b/crates/autopilot/src/run_loop.rs index 3cdb93bbd4..de5483feed 100644 --- a/crates/autopilot/src/run_loop.rs +++ b/crates/autopilot/src/run_loop.rs @@ -74,6 +74,7 @@ pub struct RunLoop { /// Maintenance tasks that should run before every runloop to have /// the most recent data available. maintenance: Arc, + competition_updates_sender: tokio::sync::mpsc::UnboundedSender<()>, } impl RunLoop { @@ -88,6 +89,7 @@ impl RunLoop { trusted_tokens: AutoUpdatingTokenList, liveness: Arc, maintenance: Arc, + competition_updates_sender: tokio::sync::mpsc::UnboundedSender<()>, ) -> Self { Self { config, @@ -100,6 +102,7 @@ impl RunLoop { in_flight_orders: Default::default(), liveness, maintenance, + competition_updates_sender, } } @@ -463,7 +466,7 @@ impl RunLoop { competition_table, }; - if let Err(err) = futures::try_join!( + match futures::try_join!( self.persistence .save_auction(auction, block_deadline) .map_err(|e| e.0.context("failed to save auction")), @@ -471,9 +474,18 @@ impl RunLoop { .save_solutions(auction.id, solutions) .map_err(|e| e.0.context("failed to save solutions")), ) { - // Don't error if saving of auction and solution fails, until stable. - // Various edge cases with JIT orders verifiable only in production. - tracing::warn!(?err, "failed to save new competition data"); + Ok(_) => { + // Notify the solver participation guard that the proposed solutions have been + // saved. + if let Err(err) = self.competition_updates_sender.send(()) { + tracing::error!(?err, "failed to notify solver participation guard"); + } + } + Err(err) => { + // Don't error if saving of auction and solution fails, until stable. + // Various edge cases with JIT orders verifiable only in production. + tracing::warn!(?err, "failed to save new competition data"); + } } tracing::trace!(?competition, "saving competition"); From e220eafa35798a74090f3181b27b647d17792f05 Mon Sep 17 00:00:00 2001 From: ilya Date: Tue, 11 Feb 2025 20:30:25 +0000 Subject: [PATCH 39/82] Nit --- .../src/domain/competition/participation_guard/db.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index 7f517debb7..e235c34722 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -69,14 +69,13 @@ impl Validator { .into_iter() .filter_map(|solver| { let address = eth::Address(solver.0.into()); - if let Some(driver) = self_.0.drivers_by_address.get(&address) { + self_.0.drivers_by_address.get(&address).map(|driver| { Metrics::get() .non_settling_solver .with_label_values(&[&driver.name]); - Some(driver.clone()) - } else { - None - } + + driver.clone() + }) }) .collect::>(); From 4bee6760a1bcc5caef96f2e28b05a463c712088f Mon Sep 17 00:00:00 2001 From: ilya Date: Tue, 11 Feb 2025 20:39:32 +0000 Subject: [PATCH 40/82] Move notification to the infra mod --- .../competition/participation_guard/db.rs | 27 ++----------------- crates/autopilot/src/infra/mod.rs | 2 +- crates/autopilot/src/infra/solvers/mod.rs | 23 +++++++++++++++- 3 files changed, 25 insertions(+), 27 deletions(-) diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index 92d7de3a4e..73683bb4ce 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -5,7 +5,6 @@ use { infra, }, ethrpc::block_stream::CurrentBlockWatcher, - futures::future::join_all, std::{ collections::HashMap, sync::Arc, @@ -74,7 +73,7 @@ impl Validator { Metrics::get() .non_settling_solver .with_label_values(&[&driver.name]); - + driver.clone() }) }) @@ -94,7 +93,7 @@ impl Validator { .filter(|driver| driver.accepts_unsettled_blocking) .collect::>(); - Self::notify_solvers(&non_settling_drivers); + infra::notify_non_settling_solvers(&non_settling_drivers); let now = Instant::now(); for driver in non_settling_drivers { @@ -111,28 +110,6 @@ impl Validator { } }); } - - /// Try to notify all the non-settling solvers in a background task. - fn notify_solvers(non_settling_drivers: &[Arc]) { - let futures = non_settling_drivers - .iter() - .cloned() - .map(|driver| { - async move { - if let Err(err) = driver - .notify(&infra::solvers::dto::notify::Request::UnsettledConsecutiveAuctions) - .await - { - tracing::debug!(solver = ?driver.name, ?err, "unable to notify external solver"); - } - } - }) - .collect::>(); - - tokio::spawn(async move { - join_all(futures).await; - }); - } } #[async_trait::async_trait] diff --git a/crates/autopilot/src/infra/mod.rs b/crates/autopilot/src/infra/mod.rs index e61a7f5dc8..b8d71918e7 100644 --- a/crates/autopilot/src/infra/mod.rs +++ b/crates/autopilot/src/infra/mod.rs @@ -7,5 +7,5 @@ pub use { blockchain::Ethereum, order_validation::banned, persistence::Persistence, - solvers::Driver, + solvers::{notify_non_settling_solvers, Driver}, }; diff --git a/crates/autopilot/src/infra/solvers/mod.rs b/crates/autopilot/src/infra/solvers/mod.rs index 2490a71b15..9e3d1ff7a5 100644 --- a/crates/autopilot/src/infra/solvers/mod.rs +++ b/crates/autopilot/src/infra/solvers/mod.rs @@ -2,8 +2,9 @@ use { self::dto::{reveal, settle, solve}, crate::{arguments::Account, domain::eth, infra::solvers::dto::notify, util}, anyhow::{anyhow, Context, Result}, + ethcontract::jsonrpc::futures_util::future::join_all, reqwest::{Client, StatusCode}, - std::time::Duration, + std::{sync::Arc, time::Duration}, thiserror::Error, url::Url, }; @@ -176,3 +177,23 @@ pub async fn response_body_with_size_limit( } Ok(bytes) } + +/// Try to notify all the non-settling solvers in a background task. +pub fn notify_non_settling_solvers(non_settling_drivers: &[Arc]) { + let futures = non_settling_drivers + .iter() + .cloned() + .map(|driver| async move { + if let Err(err) = driver + .notify(¬ify::Request::UnsettledConsecutiveAuctions) + .await + { + tracing::debug!(solver = ?driver.name, ?err, "unable to notify external solver"); + } + }) + .collect::>(); + + tokio::spawn(async move { + join_all(futures).await; + }); +} From 17f3f3a3bcbf340dd89eaf6e144ba0687f4bd103 Mon Sep 17 00:00:00 2001 From: ilya Date: Tue, 11 Feb 2025 21:03:08 +0000 Subject: [PATCH 41/82] Ban until timestamp notification --- .../competition/participation_guard/db.rs | 10 +++++++-- .../autopilot/src/infra/solvers/dto/notify.rs | 2 +- crates/autopilot/src/infra/solvers/mod.rs | 9 ++++++-- .../api/routes/notify/dto/notify_request.rs | 9 ++++---- .../driver/src/infra/api/routes/notify/mod.rs | 2 +- .../driver/src/infra/notify/notification.rs | 5 ++++- .../src/infra/solver/dto/notification.rs | 21 +++++++++++++------ 7 files changed, 41 insertions(+), 17 deletions(-) diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index 73683bb4ce..ca29326962 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -5,6 +5,7 @@ use { infra, }, ethrpc::block_stream::CurrentBlockWatcher, + model::time::now_in_epoch_seconds, std::{ collections::HashMap, sync::Arc, @@ -93,9 +94,14 @@ impl Validator { .filter(|driver| driver.accepts_unsettled_blocking) .collect::>(); - infra::notify_non_settling_solvers(&non_settling_drivers); - let now = Instant::now(); + let banned_until_timestamp = + u64::from(now_in_epoch_seconds()) + self_.0.ttl.as_secs(); + infra::notify_non_settling_solvers( + &non_settling_drivers, + banned_until_timestamp, + ); + for driver in non_settling_drivers { self_ .0 diff --git a/crates/autopilot/src/infra/solvers/dto/notify.rs b/crates/autopilot/src/infra/solvers/dto/notify.rs index 08165964fd..92a5532fa0 100644 --- a/crates/autopilot/src/infra/solvers/dto/notify.rs +++ b/crates/autopilot/src/infra/solvers/dto/notify.rs @@ -5,5 +5,5 @@ use {serde::Serialize, serde_with::serde_as}; #[serde(rename_all = "camelCase")] pub enum Request { /// The driver won multiple consecutive auctions but never settled them. - UnsettledConsecutiveAuctions, + UnsettledConsecutiveAuctions(u64), } diff --git a/crates/autopilot/src/infra/solvers/mod.rs b/crates/autopilot/src/infra/solvers/mod.rs index 9e3d1ff7a5..7123c94ba7 100644 --- a/crates/autopilot/src/infra/solvers/mod.rs +++ b/crates/autopilot/src/infra/solvers/mod.rs @@ -179,13 +179,18 @@ pub async fn response_body_with_size_limit( } /// Try to notify all the non-settling solvers in a background task. -pub fn notify_non_settling_solvers(non_settling_drivers: &[Arc]) { +pub fn notify_non_settling_solvers( + non_settling_drivers: &[Arc], + banned_until_timestamp: u64, +) { let futures = non_settling_drivers .iter() .cloned() .map(|driver| async move { if let Err(err) = driver - .notify(¬ify::Request::UnsettledConsecutiveAuctions) + .notify(¬ify::Request::UnsettledConsecutiveAuctions( + banned_until_timestamp, + )) .await { tracing::debug!(solver = ?driver.name, ?err, "unable to notify external solver"); diff --git a/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs b/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs index 16afa72814..a89a7a73e6 100644 --- a/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs +++ b/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs @@ -5,15 +5,16 @@ use {crate::infra::notify, serde::Deserialize, serde_with::serde_as}; #[serde(rename_all = "camelCase")] pub enum NotifyRequest { /// The driver won multiple consecutive auctions but never settled them. - UnsettledConsecutiveAuctions, + UnsettledConsecutiveAuctions(u64), } impl From for notify::Kind { fn from(value: NotifyRequest) -> Self { match value { - NotifyRequest::UnsettledConsecutiveAuctions => { - notify::Kind::Banned(notify::BanReason::UnsettledConsecutiveAuctions) - } + NotifyRequest::UnsettledConsecutiveAuctions(until_timestamp) => notify::Kind::Banned { + reason: notify::BanReason::UnsettledConsecutiveAuctions, + until_timestamp, + }, } } } diff --git a/crates/driver/src/infra/api/routes/notify/mod.rs b/crates/driver/src/infra/api/routes/notify/mod.rs index 7683cc499b..e6ae815b82 100644 --- a/crates/driver/src/infra/api/routes/notify/mod.rs +++ b/crates/driver/src/infra/api/routes/notify/mod.rs @@ -13,7 +13,7 @@ async fn route( req: axum::Json, ) -> Result)> { let solver = &state.solver().name().0; - tracing::trace!(?req, ?solver, "received a notification"); + tracing::debug!(?req, ?solver, "received a notification"); state .solver() .notify(None, None, req.0.into()) diff --git a/crates/driver/src/infra/notify/notification.rs b/crates/driver/src/infra/notify/notification.rs index 5e6323dbdf..2b9fe40003 100644 --- a/crates/driver/src/infra/notify/notification.rs +++ b/crates/driver/src/infra/notify/notification.rs @@ -46,7 +46,10 @@ pub enum Kind { /// On-chain solution postprocessing timed out. PostprocessingTimedOut, /// The solver has been banned for a specific reason. - Banned(BanReason), + Banned { + reason: BanReason, + until_timestamp: u64, + }, } #[derive(Debug)] diff --git a/crates/driver/src/infra/solver/dto/notification.rs b/crates/driver/src/infra/solver/dto/notification.rs index a352c712b1..49f55ac399 100644 --- a/crates/driver/src/infra/solver/dto/notification.rs +++ b/crates/driver/src/infra/solver/dto/notification.rs @@ -61,11 +61,17 @@ impl Notification { notify::Settlement::Expired => Kind::Expired, }, notify::Kind::PostprocessingTimedOut => Kind::PostprocessingTimedOut, - notify::Kind::Banned(reason) => Kind::Banned(match reason { - notify::BanReason::UnsettledConsecutiveAuctions => { - BanReason::UnsettledConsecutiveAuctions - } - }), + notify::Kind::Banned { + reason, + until_timestamp, + } => Kind::Banned { + reason: match reason { + notify::BanReason::UnsettledConsecutiveAuctions => { + BanReason::UnsettledConsecutiveAuctions + } + }, + until_timestamp, + }, }, } } @@ -149,7 +155,10 @@ pub enum Kind { Expired, Fail, PostprocessingTimedOut, - Banned(BanReason), + Banned { + reason: BanReason, + until_timestamp: u64, + }, } #[serde_as] From 51832d49c5388687a30e5f3d45201d31826c54f0 Mon Sep 17 00:00:00 2001 From: ilya Date: Tue, 11 Feb 2025 21:05:36 +0000 Subject: [PATCH 42/82] Formatting --- .../autopilot/src/domain/competition/participation_guard/db.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index e235c34722..f181265bf5 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -73,7 +73,7 @@ impl Validator { Metrics::get() .non_settling_solver .with_label_values(&[&driver.name]); - + driver.clone() }) }) From b2b4c91775f64d36d44af27501daa524c9f9ac2f Mon Sep 17 00:00:00 2001 From: ilya Date: Wed, 12 Feb 2025 08:49:22 +0000 Subject: [PATCH 43/82] Doc --- crates/autopilot/src/database/competition.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/autopilot/src/database/competition.rs b/crates/autopilot/src/database/competition.rs index ebe533e01f..d50a08c705 100644 --- a/crates/autopilot/src/database/competition.rs +++ b/crates/autopilot/src/database/competition.rs @@ -164,6 +164,10 @@ impl super::Postgres { .context("solver_competition::find_non_settling_solvers") } + /// Finds solvers that have a successful settling rate below the given + /// ratio. The current block is used to prevent + /// selecting auctions with deadline after the current block since they + /// still can be settled. pub async fn find_low_settling_solvers( &self, last_auctions_count: u32, From 7ef7db2d3040ba72d52dd065f02a2dbae677509d Mon Sep 17 00:00:00 2001 From: ilya Date: Wed, 12 Feb 2025 10:28:50 +0000 Subject: [PATCH 44/82] Switch to max failure rate --- crates/autopilot/src/arguments.rs | 6 +++--- crates/autopilot/src/database/competition.rs | 11 +++++------ .../src/domain/competition/participation_guard/db.rs | 2 +- crates/database/src/solver_competition.rs | 12 ++++++------ 4 files changed, 15 insertions(+), 16 deletions(-) diff --git a/crates/autopilot/src/arguments.rs b/crates/autopilot/src/arguments.rs index 5697170686..1dd5a0c7dc 100644 --- a/crates/autopilot/src/arguments.rs +++ b/crates/autopilot/src/arguments.rs @@ -305,11 +305,11 @@ pub struct LowSettlingValidatorConfig { )] pub last_auctions_participation_count: u32, - /// A minimum success rate for a solver to be considered eligible for + /// A max failure rate for a solver to remain eligible for /// participation in the competition. Otherwise, the solver will be /// banned. - #[clap(long, env, default_value = "0.1")] - pub solver_min_settlement_success_rate: f64, + #[clap(long, env, default_value = "0.9")] + pub solver_max_settlement_failure_rate: f64, } impl std::fmt::Display for Arguments { diff --git a/crates/autopilot/src/database/competition.rs b/crates/autopilot/src/database/competition.rs index d50a08c705..23860df1f1 100644 --- a/crates/autopilot/src/database/competition.rs +++ b/crates/autopilot/src/database/competition.rs @@ -164,15 +164,14 @@ impl super::Postgres { .context("solver_competition::find_non_settling_solvers") } - /// Finds solvers that have a successful settling rate below the given - /// ratio. The current block is used to prevent - /// selecting auctions with deadline after the current block since they - /// still can be settled. + /// Finds solvers that have a failure settling rate above the given + /// ratio. The current block is used to prevent selecting auctions with + /// deadline after the current block since they still can be settled. pub async fn find_low_settling_solvers( &self, last_auctions_count: u32, current_block: u64, - min_success_ratio: f64, + max_failure_rate: f64, ) -> anyhow::Result> { let mut ex = self.pool.acquire().await.context("acquire")?; let _timer = super::Metrics::get() @@ -184,7 +183,7 @@ impl super::Postgres { &mut ex, last_auctions_count, current_block, - min_success_ratio, + max_failure_rate, ) .await .context("solver_competition::find_low_settling_solvers") diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index 6c8578dbf4..3f3de89767 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -126,7 +126,7 @@ impl Validator { current_block, self.0 .low_settling_config - .solver_min_settlement_success_rate, + .solver_max_settlement_failure_rate, ) .await { diff --git a/crates/database/src/solver_competition.rs b/crates/database/src/solver_competition.rs index df6454f15e..a7ad747b04 100644 --- a/crates/database/src/solver_competition.rs +++ b/crates/database/src/solver_competition.rs @@ -152,7 +152,7 @@ pub async fn find_low_settling_solvers( ex: &mut PgConnection, last_auctions_count: u32, current_block: u64, - min_success_ratio: f64, + max_failure_rate: f64, ) -> Result, sqlx::Error> { const QUERY: &str = r#" WITH @@ -179,13 +179,13 @@ WITH ) SELECT solver FROM solver_settlement_counts -WHERE (total_settlements::decimal / NULLIF(total_wins, 0)) < $3; +WHERE (1 - (total_settlements::decimal / NULLIF(total_wins, 0))) > $3; "#; sqlx::query_scalar(QUERY) .bind(sqlx::types::BigDecimal::from(current_block)) .bind(i64::from(last_auctions_count)) - .bind(min_success_ratio) + .bind(max_failure_rate) .fetch_all(ex) .await } @@ -805,7 +805,7 @@ mod tests { let deadline_block = 2u64; let last_auctions_count = 100i64; - let min_ratio = 0.4; + let max_failure_ratio = 0.6; let mut solution_uid = 0; for auction_id in 1..=10 { @@ -912,7 +912,7 @@ mod tests { &mut db, u32::try_from(last_auctions_count).unwrap(), deadline_block, - min_ratio, + max_failure_ratio, ) .await .unwrap(); @@ -941,7 +941,7 @@ mod tests { &mut db, u32::try_from(last_auctions_count).unwrap(), deadline_block, - min_ratio, + max_failure_ratio, ) .await .unwrap(); From 57cb1901f7c038c8e33d56796110dfb7224714be Mon Sep 17 00:00:00 2001 From: ilya Date: Wed, 12 Feb 2025 10:58:44 +0000 Subject: [PATCH 45/82] Naming --- crates/autopilot/src/arguments.rs | 8 +++---- .../competition/participation_guard/db.rs | 24 +++++++++---------- .../competition/participation_guard/mod.rs | 8 +++---- .../participation_guard/onchain.rs | 2 +- .../autopilot/src/infra/solvers/dto/notify.rs | 4 ++-- .../api/routes/notify/dto/notify_request.rs | 6 ++--- .../driver/src/infra/notify/notification.rs | 4 ++-- .../src/infra/solver/dto/notification.rs | 2 +- 8 files changed, 29 insertions(+), 29 deletions(-) diff --git a/crates/autopilot/src/arguments.rs b/crates/autopilot/src/arguments.rs index 1dd5a0c7dc..5056e019e4 100644 --- a/crates/autopilot/src/arguments.rs +++ b/crates/autopilot/src/arguments.rs @@ -258,14 +258,14 @@ pub struct DbBasedSolverParticipationGuardConfig { pub solver_blacklist_cache_ttl: Duration, #[clap(flatten)] - pub non_settling_validator_config: NonSettlingValidatorConfig, + pub non_settling_solvers_finder_config: NonSettlingSolversFinderConfig, #[clap(flatten)] - pub low_settling_validator_config: LowSettlingValidatorConfig, + pub low_settling_solvers_finder_config: LowSettlingSolversFinderConfig, } #[derive(Debug, clap::Parser)] -pub struct NonSettlingValidatorConfig { +pub struct NonSettlingSolversFinderConfig { /// Enables search of non-settling solvers. #[clap( id = "non_settling_solvers_blacklisting_enabled", @@ -286,7 +286,7 @@ pub struct NonSettlingValidatorConfig { } #[derive(Debug, clap::Parser)] -pub struct LowSettlingValidatorConfig { +pub struct LowSettlingSolversFinderConfig { /// Enables search of non-settling solvers. #[clap( id = "low_settling_solvers_blacklisting_enabled", diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index 3f3de89767..70218a038d 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -2,8 +2,8 @@ use { crate::{ arguments::{ DbBasedSolverParticipationGuardConfig, - LowSettlingValidatorConfig, - NonSettlingValidatorConfig, + LowSettlingSolversFinderConfig, + NonSettlingSolversFinderConfig, }, database::Postgres, domain::{eth, Metrics}, @@ -23,18 +23,18 @@ use { /// and either never settled any of them or their settlement success rate is /// lower than `min_settlement_success_rate`. #[derive(Clone)] -pub(super) struct Validator(Arc); +pub(super) struct SolverValidator(Arc); struct Inner { db: Postgres, banned_solvers: dashmap::DashMap, ttl: Duration, - non_settling_config: NonSettlingValidatorConfig, - low_settling_config: LowSettlingValidatorConfig, + non_settling_config: NonSettlingSolversFinderConfig, + low_settling_config: LowSettlingSolversFinderConfig, drivers_by_address: HashMap>, } -impl Validator { +impl SolverValidator { pub fn new( db: Postgres, current_block: CurrentBlockWatcher, @@ -46,8 +46,8 @@ impl Validator { db, banned_solvers: Default::default(), ttl: db_based_validator_config.solver_blacklist_cache_ttl, - non_settling_config: db_based_validator_config.non_settling_validator_config, - low_settling_config: db_based_validator_config.low_settling_validator_config, + non_settling_config: db_based_validator_config.non_settling_solvers_finder_config, + low_settling_config: db_based_validator_config.low_settling_solvers_finder_config, drivers_by_address, })); @@ -82,7 +82,7 @@ impl Validator { ); self_.post_process( &low_settling_solvers, - &dto::notify::Request::LowSettlingRate, + &dto::notify::Request::HighSettleFailureRate, ); } }); @@ -174,7 +174,7 @@ impl Validator { let log_message = match request { dto::notify::Request::UnsettledConsecutiveAuctions => "found non-settling solvers", - dto::notify::Request::LowSettlingRate => "found low-settling solvers", + dto::notify::Request::HighSettleFailureRate => "found high-failure-settlement solvers", }; let solver_names = drivers .iter() @@ -184,7 +184,7 @@ impl Validator { let reason = match request { dto::notify::Request::UnsettledConsecutiveAuctions => "non_settling", - dto::notify::Request::LowSettlingRate => "low_settling", + dto::notify::Request::HighSettleFailureRate => "high_settle_failure_rate", }; for solver in solver_names { @@ -209,7 +209,7 @@ impl Validator { } #[async_trait::async_trait] -impl super::Validator for Validator { +impl super::SolverValidator for SolverValidator { async fn is_allowed(&self, solver: ð::Address) -> anyhow::Result { if let Some(entry) = self.0.banned_solvers.get(solver) { if Instant::now().duration_since(*entry.value()) < self.0.ttl { diff --git a/crates/autopilot/src/domain/competition/participation_guard/mod.rs b/crates/autopilot/src/domain/competition/participation_guard/mod.rs index f04243ceb0..c1a583909b 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/mod.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/mod.rs @@ -18,7 +18,7 @@ pub struct SolverParticipationGuard(Arc); struct Inner { /// Stores the validators in order they will be called. - validators: Vec>, + validators: Vec>, } impl SolverParticipationGuard { @@ -29,10 +29,10 @@ impl SolverParticipationGuard { db_based_validator_config: DbBasedSolverParticipationGuardConfig, drivers_by_address: HashMap>, ) -> Self { - let mut validators: Vec> = Vec::new(); + let mut validators: Vec> = Vec::new(); let current_block = eth.current_block().clone(); - let database_solver_participation_validator = db::Validator::new( + let database_solver_participation_validator = db::SolverValidator::new( db, current_block, settlement_updates_receiver, @@ -64,6 +64,6 @@ impl SolverParticipationGuard { } #[async_trait::async_trait] -trait Validator: Send + Sync { +trait SolverValidator: Send + Sync { async fn is_allowed(&self, solver: ð::Address) -> anyhow::Result; } diff --git a/crates/autopilot/src/domain/competition/participation_guard/onchain.rs b/crates/autopilot/src/domain/competition/participation_guard/onchain.rs index 598da9cf26..a23cd7e483 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/onchain.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/onchain.rs @@ -7,7 +7,7 @@ pub(super) struct Validator { } #[async_trait::async_trait] -impl super::Validator for Validator { +impl super::SolverValidator for Validator { async fn is_allowed(&self, solver: ð::Address) -> anyhow::Result { Ok(self .eth diff --git a/crates/autopilot/src/infra/solvers/dto/notify.rs b/crates/autopilot/src/infra/solvers/dto/notify.rs index e72ae90a12..8bb5b6e0c0 100644 --- a/crates/autopilot/src/infra/solvers/dto/notify.rs +++ b/crates/autopilot/src/infra/solvers/dto/notify.rs @@ -6,6 +6,6 @@ use {serde::Serialize, serde_with::serde_as}; pub enum Request { /// The driver won multiple consecutive auctions but never settled them. UnsettledConsecutiveAuctions, - /// Driver's success settling rate is below the threshold. - LowSettlingRate, + /// Driver's settle failure rate is above the threshold. + HighSettleFailureRate, } diff --git a/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs b/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs index bef312eef2..019683191c 100644 --- a/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs +++ b/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs @@ -7,7 +7,7 @@ pub enum NotifyRequest { /// The driver won multiple consecutive auctions but never settled them. UnsettledConsecutiveAuctions, /// Driver's success settling rate is below the threshold. - LowSettlingRate, + HighSettleFailureRate, } impl From for notify::Kind { @@ -16,8 +16,8 @@ impl From for notify::Kind { NotifyRequest::UnsettledConsecutiveAuctions => { notify::Kind::Banned(notify::BanReason::UnsettledConsecutiveAuctions) } - NotifyRequest::LowSettlingRate => { - notify::Kind::Banned(notify::BanReason::LowSettlingRate) + NotifyRequest::HighSettleFailureRate => { + notify::Kind::Banned(notify::BanReason::HighSettleFailureRate) } } } diff --git a/crates/driver/src/infra/notify/notification.rs b/crates/driver/src/infra/notify/notification.rs index 6f990199f5..97aba00390 100644 --- a/crates/driver/src/infra/notify/notification.rs +++ b/crates/driver/src/infra/notify/notification.rs @@ -64,8 +64,8 @@ pub enum ScoreKind { pub enum BanReason { /// The driver won multiple consecutive auctions but never settled them. UnsettledConsecutiveAuctions, - /// Driver's success settling rate is below the threshold. - LowSettlingRate, + /// Driver's settle failure rate is above the threshold. + HighSettleFailureRate, } #[derive(Debug)] diff --git a/crates/driver/src/infra/solver/dto/notification.rs b/crates/driver/src/infra/solver/dto/notification.rs index 9408b81c17..f17d532e26 100644 --- a/crates/driver/src/infra/solver/dto/notification.rs +++ b/crates/driver/src/infra/solver/dto/notification.rs @@ -65,7 +65,7 @@ impl Notification { notify::BanReason::UnsettledConsecutiveAuctions => { BanReason::UnsettledConsecutiveAuctions } - notify::BanReason::LowSettlingRate => BanReason::LowSettlingRate, + notify::BanReason::HighSettleFailureRate => BanReason::LowSettlingRate, }), }, } From 8af1e4bb90b1947869dd5fe6f9b0aa5f406db25b Mon Sep 17 00:00:00 2001 From: ilya Date: Wed, 12 Feb 2025 11:37:24 +0000 Subject: [PATCH 46/82] Fix after merge --- .../competition/participation_guard/db.rs | 34 +++++++++++++------ .../autopilot/src/infra/solvers/dto/notify.rs | 2 +- crates/autopilot/src/infra/solvers/mod.rs | 16 ++++----- .../api/routes/notify/dto/notify_request.rs | 4 +++ 4 files changed, 36 insertions(+), 20 deletions(-) diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index eea4c4fbc3..a86820336b 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -76,13 +76,19 @@ impl SolverValidator { // solvers. low_settling_solvers.retain(|solver| !non_settling_solvers.contains(solver)); + let found_at = Instant::now(); + let banned_until_timestamp = + u64::from(now_in_epoch_seconds()) + self_.0.ttl.as_secs(); + self_.post_process( &non_settling_solvers, - &dto::notify::Request::UnsettledConsecutiveAuctions, + &dto::notify::Request::UnsettledConsecutiveAuctions(banned_until_timestamp), + found_at, ); self_.post_process( &low_settling_solvers, - &dto::notify::Request::HighSettleFailureRate, + &dto::notify::Request::HighSettleFailureRate(banned_until_timestamp), + found_at, ); } }); @@ -142,7 +148,12 @@ impl SolverValidator { } /// Updates the cache and notifies the solvers. - fn post_process(&self, solvers: &HashSet, request: &dto::notify::Request) { + fn post_process( + &self, + solvers: &HashSet, + request: &dto::notify::Request, + found_at: Instant, + ) { if solvers.is_empty() { return; } @@ -153,8 +164,10 @@ impl SolverValidator { .collect::>(); let log_message = match request { - dto::notify::Request::UnsettledConsecutiveAuctions => "found non-settling solvers", - dto::notify::Request::HighSettleFailureRate => "found high-failure-settlement solvers", + dto::notify::Request::UnsettledConsecutiveAuctions(_) => "found non-settling solvers", + dto::notify::Request::HighSettleFailureRate(_) => { + "found high-failure-settlement solvers" + } }; let solver_names = drivers .iter() @@ -163,8 +176,8 @@ impl SolverValidator { tracing::debug!(solvers = ?solver_names, log_message); let reason = match request { - dto::notify::Request::UnsettledConsecutiveAuctions => "non_settling", - dto::notify::Request::HighSettleFailureRate => "high_settle_failure_rate", + dto::notify::Request::UnsettledConsecutiveAuctions(_) => "non_settling", + dto::notify::Request::HighSettleFailureRate(_) => "high_settle_failure_rate", }; for solver in solver_names { @@ -179,11 +192,12 @@ impl SolverValidator { .filter(|driver| driver.accepts_unsettled_blocking) .collect::>(); - Self::notify_solvers(&non_settling_drivers, request); + infra::notify_non_settling_solvers(&non_settling_drivers, request); - let now = Instant::now(); for driver in non_settling_drivers { - self.0.banned_solvers.insert(driver.submission_address, now); + self.0 + .banned_solvers + .insert(driver.submission_address, found_at); } } } diff --git a/crates/autopilot/src/infra/solvers/dto/notify.rs b/crates/autopilot/src/infra/solvers/dto/notify.rs index 1282ac6bd1..c28dd63962 100644 --- a/crates/autopilot/src/infra/solvers/dto/notify.rs +++ b/crates/autopilot/src/infra/solvers/dto/notify.rs @@ -7,5 +7,5 @@ pub enum Request { /// The driver won multiple consecutive auctions but never settled them. UnsettledConsecutiveAuctions(u64), /// Driver's settle failure rate is above the threshold. - HighSettleFailureRate, + HighSettleFailureRate(u64), } diff --git a/crates/autopilot/src/infra/solvers/mod.rs b/crates/autopilot/src/infra/solvers/mod.rs index 7123c94ba7..3e8efd90c4 100644 --- a/crates/autopilot/src/infra/solvers/mod.rs +++ b/crates/autopilot/src/infra/solvers/mod.rs @@ -181,19 +181,17 @@ pub async fn response_body_with_size_limit( /// Try to notify all the non-settling solvers in a background task. pub fn notify_non_settling_solvers( non_settling_drivers: &[Arc], - banned_until_timestamp: u64, + request: ¬ify::Request, ) { let futures = non_settling_drivers .iter() .cloned() - .map(|driver| async move { - if let Err(err) = driver - .notify(¬ify::Request::UnsettledConsecutiveAuctions( - banned_until_timestamp, - )) - .await - { - tracing::debug!(solver = ?driver.name, ?err, "unable to notify external solver"); + .map(|driver| { + let request = request.clone(); + async move { + if let Err(err) = driver.notify(&request).await { + tracing::debug!(solver = ?driver.name, ?err, "unable to notify external solver"); + } } }) .collect::>(); diff --git a/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs b/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs index cdfe2dca76..cbd4b9a71b 100644 --- a/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs +++ b/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs @@ -17,6 +17,10 @@ impl From for notify::Kind { reason: notify::BanReason::UnsettledConsecutiveAuctions, until_timestamp, }, + NotifyRequest::HighSettleFailureRate(until_timestamp) => notify::Kind::Banned { + reason: notify::BanReason::HighSettleFailureRate, + until_timestamp, + }, } } } From e7ad14f91f153d1b6ca8df10e7851ff3ebd86450 Mon Sep 17 00:00:00 2001 From: ilya Date: Wed, 12 Feb 2025 11:54:20 +0000 Subject: [PATCH 47/82] Notifications refactoring --- .../competition/participation_guard/db.rs | 27 ++++++++------- crates/autopilot/src/infra/mod.rs | 2 +- .../autopilot/src/infra/solvers/dto/notify.rs | 14 ++++++-- crates/autopilot/src/infra/solvers/mod.rs | 20 ++++++----- .../api/routes/notify/dto/notify_request.rs | 34 +++++++++++++------ 5 files changed, 64 insertions(+), 33 deletions(-) diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index a86820336b..5cd3ff324f 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -82,13 +82,15 @@ impl SolverValidator { self_.post_process( &non_settling_solvers, - &dto::notify::Request::UnsettledConsecutiveAuctions(banned_until_timestamp), + &dto::notify::BanReason::UnsettledConsecutiveAuctions, found_at, + banned_until_timestamp, ); self_.post_process( &low_settling_solvers, - &dto::notify::Request::HighSettleFailureRate(banned_until_timestamp), + &dto::notify::BanReason::HighSettleFailureRate, found_at, + banned_until_timestamp, ); } }); @@ -151,8 +153,9 @@ impl SolverValidator { fn post_process( &self, solvers: &HashSet, - request: &dto::notify::Request, + ban_reason: &dto::notify::BanReason, found_at: Instant, + banned_until_timestamp: u64, ) { if solvers.is_empty() { return; @@ -163,9 +166,9 @@ impl SolverValidator { .filter_map(|solver| self.0.drivers_by_address.get(solver).cloned()) .collect::>(); - let log_message = match request { - dto::notify::Request::UnsettledConsecutiveAuctions(_) => "found non-settling solvers", - dto::notify::Request::HighSettleFailureRate(_) => { + let log_message = match ban_reason { + dto::notify::BanReason::UnsettledConsecutiveAuctions => "found non-settling solvers", + dto::notify::BanReason::HighSettleFailureRate => { "found high-failure-settlement solvers" } }; @@ -175,9 +178,9 @@ impl SolverValidator { .collect::>(); tracing::debug!(solvers = ?solver_names, log_message); - let reason = match request { - dto::notify::Request::UnsettledConsecutiveAuctions(_) => "non_settling", - dto::notify::Request::HighSettleFailureRate(_) => "high_settle_failure_rate", + let reason = match ban_reason { + dto::notify::BanReason::UnsettledConsecutiveAuctions => "non_settling", + dto::notify::BanReason::HighSettleFailureRate => "high_settle_failure_rate", }; for solver in solver_names { @@ -186,15 +189,15 @@ impl SolverValidator { .with_label_values(&[&solver, reason]); } - let non_settling_drivers = drivers + let banned_drivers = drivers .into_iter() // Notify and block only solvers that accept unsettled blocking feature. This should be removed once a CIP is approved. .filter(|driver| driver.accepts_unsettled_blocking) .collect::>(); - infra::notify_non_settling_solvers(&non_settling_drivers, request); + infra::notify_banned_solvers(&banned_drivers, ban_reason, banned_until_timestamp); - for driver in non_settling_drivers { + for driver in banned_drivers { self.0 .banned_solvers .insert(driver.submission_address, found_at); diff --git a/crates/autopilot/src/infra/mod.rs b/crates/autopilot/src/infra/mod.rs index b8d71918e7..d960038283 100644 --- a/crates/autopilot/src/infra/mod.rs +++ b/crates/autopilot/src/infra/mod.rs @@ -7,5 +7,5 @@ pub use { blockchain::Ethereum, order_validation::banned, persistence::Persistence, - solvers::{notify_non_settling_solvers, Driver}, + solvers::{notify_banned_solvers, Driver}, }; diff --git a/crates/autopilot/src/infra/solvers/dto/notify.rs b/crates/autopilot/src/infra/solvers/dto/notify.rs index c28dd63962..4af2449343 100644 --- a/crates/autopilot/src/infra/solvers/dto/notify.rs +++ b/crates/autopilot/src/infra/solvers/dto/notify.rs @@ -4,8 +4,18 @@ use {serde::Serialize, serde_with::serde_as}; #[derive(Clone, Debug, Serialize)] #[serde(rename_all = "camelCase")] pub enum Request { + Banned { + reason: BanReason, + banned_until_timestamp: u64, + }, +} + +#[serde_as] +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum BanReason { /// The driver won multiple consecutive auctions but never settled them. - UnsettledConsecutiveAuctions(u64), + UnsettledConsecutiveAuctions, /// Driver's settle failure rate is above the threshold. - HighSettleFailureRate(u64), + HighSettleFailureRate, } diff --git a/crates/autopilot/src/infra/solvers/mod.rs b/crates/autopilot/src/infra/solvers/mod.rs index 3e8efd90c4..21be4150f8 100644 --- a/crates/autopilot/src/infra/solvers/mod.rs +++ b/crates/autopilot/src/infra/solvers/mod.rs @@ -179,21 +179,25 @@ pub async fn response_body_with_size_limit( } /// Try to notify all the non-settling solvers in a background task. -pub fn notify_non_settling_solvers( - non_settling_drivers: &[Arc], - request: ¬ify::Request, +pub fn notify_banned_solvers( + drivers: &[Arc], + reason: ¬ify::BanReason, + banned_until_timestamp: u64, ) { - let futures = non_settling_drivers + let request = notify::Request::Banned { + reason: reason.clone(), + banned_until_timestamp, + }; + let futures = drivers .iter() .cloned() .map(|driver| { let request = request.clone(); async move { - if let Err(err) = driver.notify(&request).await { - tracing::debug!(solver = ?driver.name, ?err, "unable to notify external solver"); - } + if let Err(err) = driver.notify(&request).await { + tracing::debug!(solver = ?driver.name, ?err, "unable to notify external solver"); } - }) + }}) .collect::>(); tokio::spawn(async move { diff --git a/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs b/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs index cbd4b9a71b..04e2f30268 100644 --- a/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs +++ b/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs @@ -4,22 +4,36 @@ use {crate::infra::notify, serde::Deserialize, serde_with::serde_as}; #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub enum NotifyRequest { + Banned { + reason: BanReason, + banned_until_timestamp: u64, + }, +} + +#[serde_as] +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum BanReason { /// The driver won multiple consecutive auctions but never settled them. - UnsettledConsecutiveAuctions(u64), - /// Driver's success settling rate is below the threshold. - HighSettleFailureRate(u64), + UnsettledConsecutiveAuctions, + /// Driver's settle failure rate is above the threshold. + HighSettleFailureRate, } impl From for notify::Kind { fn from(value: NotifyRequest) -> Self { match value { - NotifyRequest::UnsettledConsecutiveAuctions(until_timestamp) => notify::Kind::Banned { - reason: notify::BanReason::UnsettledConsecutiveAuctions, - until_timestamp, - }, - NotifyRequest::HighSettleFailureRate(until_timestamp) => notify::Kind::Banned { - reason: notify::BanReason::HighSettleFailureRate, - until_timestamp, + NotifyRequest::Banned { + reason, + banned_until_timestamp, + } => notify::Kind::Banned { + reason: match reason { + BanReason::UnsettledConsecutiveAuctions => { + notify::BanReason::UnsettledConsecutiveAuctions + } + BanReason::HighSettleFailureRate => notify::BanReason::HighSettleFailureRate, + }, + until_timestamp: banned_until_timestamp, }, } } From cca70c889a136515c457320fda52f37498a3a7bd Mon Sep 17 00:00:00 2001 From: ilya Date: Wed, 12 Feb 2025 11:58:00 +0000 Subject: [PATCH 48/82] Refactoring --- .../autopilot/src/infra/solvers/dto/notify.rs | 13 +++++++++-- crates/autopilot/src/infra/solvers/mod.rs | 7 +++--- .../api/routes/notify/dto/notify_request.rs | 23 ++++++++++++++++--- 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/crates/autopilot/src/infra/solvers/dto/notify.rs b/crates/autopilot/src/infra/solvers/dto/notify.rs index 92a5532fa0..0ac2e786f9 100644 --- a/crates/autopilot/src/infra/solvers/dto/notify.rs +++ b/crates/autopilot/src/infra/solvers/dto/notify.rs @@ -4,6 +4,15 @@ use {serde::Serialize, serde_with::serde_as}; #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] pub enum Request { - /// The driver won multiple consecutive auctions but never settled them. - UnsettledConsecutiveAuctions(u64), + Banned { + reason: BanReason, + until_timestamp: u64, + }, +} + +#[serde_as] +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum BanReason { + UnsettledConsecutiveAuctions, } diff --git a/crates/autopilot/src/infra/solvers/mod.rs b/crates/autopilot/src/infra/solvers/mod.rs index 7123c94ba7..264ab0931b 100644 --- a/crates/autopilot/src/infra/solvers/mod.rs +++ b/crates/autopilot/src/infra/solvers/mod.rs @@ -188,9 +188,10 @@ pub fn notify_non_settling_solvers( .cloned() .map(|driver| async move { if let Err(err) = driver - .notify(¬ify::Request::UnsettledConsecutiveAuctions( - banned_until_timestamp, - )) + .notify(¬ify::Request::Banned { + reason: notify::BanReason::UnsettledConsecutiveAuctions, + until_timestamp: banned_until_timestamp, + }) .await { tracing::debug!(solver = ?driver.name, ?err, "unable to notify external solver"); diff --git a/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs b/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs index a89a7a73e6..4eb36c20ef 100644 --- a/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs +++ b/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs @@ -4,15 +4,32 @@ use {crate::infra::notify, serde::Deserialize, serde_with::serde_as}; #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub enum NotifyRequest { + Banned { + reason: BanReason, + until_timestamp: u64, + }, +} + +#[serde_as] +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum BanReason { /// The driver won multiple consecutive auctions but never settled them. - UnsettledConsecutiveAuctions(u64), + UnsettledConsecutiveAuctions, } impl From for notify::Kind { fn from(value: NotifyRequest) -> Self { match value { - NotifyRequest::UnsettledConsecutiveAuctions(until_timestamp) => notify::Kind::Banned { - reason: notify::BanReason::UnsettledConsecutiveAuctions, + NotifyRequest::Banned { + reason, + until_timestamp, + } => notify::Kind::Banned { + reason: match reason { + BanReason::UnsettledConsecutiveAuctions => { + notify::BanReason::UnsettledConsecutiveAuctions + } + }, until_timestamp, }, } From 7de4dc15ec749ed8fa80e143c055e6649baa4440 Mon Sep 17 00:00:00 2001 From: ilya Date: Wed, 12 Feb 2025 11:59:45 +0000 Subject: [PATCH 49/82] OpenAPI --- crates/driver/openapi.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/driver/openapi.yml b/crates/driver/openapi.yml index 0383a35130..4a43152472 100644 --- a/crates/driver/openapi.yml +++ b/crates/driver/openapi.yml @@ -149,9 +149,9 @@ paths: schema: type: string enum: - - unsettledConsecutiveAuctions + - banned description: |- - A notification that informs about a reasoning why the driver was banned. + The reason for the notification with optional additional context. responses: "200": description: notification successfully received. From ee7f492762ad4324ea36543248ee3c723724e78d Mon Sep 17 00:00:00 2001 From: ilya Date: Wed, 12 Feb 2025 12:04:05 +0000 Subject: [PATCH 50/82] Naming --- crates/autopilot/src/infra/solvers/dto/notify.rs | 2 +- crates/autopilot/src/infra/solvers/mod.rs | 2 +- .../src/infra/api/routes/notify/dto/notify_request.rs | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/autopilot/src/infra/solvers/dto/notify.rs b/crates/autopilot/src/infra/solvers/dto/notify.rs index 4af2449343..4a92306ec9 100644 --- a/crates/autopilot/src/infra/solvers/dto/notify.rs +++ b/crates/autopilot/src/infra/solvers/dto/notify.rs @@ -6,7 +6,7 @@ use {serde::Serialize, serde_with::serde_as}; pub enum Request { Banned { reason: BanReason, - banned_until_timestamp: u64, + until_timestamp: u64, }, } diff --git a/crates/autopilot/src/infra/solvers/mod.rs b/crates/autopilot/src/infra/solvers/mod.rs index 21be4150f8..16eedb0a41 100644 --- a/crates/autopilot/src/infra/solvers/mod.rs +++ b/crates/autopilot/src/infra/solvers/mod.rs @@ -186,7 +186,7 @@ pub fn notify_banned_solvers( ) { let request = notify::Request::Banned { reason: reason.clone(), - banned_until_timestamp, + until_timestamp: banned_until_timestamp, }; let futures = drivers .iter() diff --git a/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs b/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs index 04e2f30268..f60a5bcaca 100644 --- a/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs +++ b/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs @@ -6,7 +6,7 @@ use {crate::infra::notify, serde::Deserialize, serde_with::serde_as}; pub enum NotifyRequest { Banned { reason: BanReason, - banned_until_timestamp: u64, + until_timestamp: u64, }, } @@ -25,7 +25,7 @@ impl From for notify::Kind { match value { NotifyRequest::Banned { reason, - banned_until_timestamp, + until_timestamp, } => notify::Kind::Banned { reason: match reason { BanReason::UnsettledConsecutiveAuctions => { @@ -33,7 +33,7 @@ impl From for notify::Kind { } BanReason::HighSettleFailureRate => notify::BanReason::HighSettleFailureRate, }, - until_timestamp: banned_until_timestamp, + until_timestamp, }, } } From 17ee52c373d76e498c7efaa41bf3eff389aba873 Mon Sep 17 00:00:00 2001 From: ilya Date: Wed, 12 Feb 2025 14:20:07 +0000 Subject: [PATCH 51/82] infra::Persistence --- crates/autopilot/src/database/competition.rs | 24 ----------------- .../competition/participation_guard/db.rs | 12 ++++----- .../competition/participation_guard/mod.rs | 5 ++-- crates/autopilot/src/infra/persistence/mod.rs | 27 +++++++++++++++++++ crates/autopilot/src/run.rs | 2 +- 5 files changed, 35 insertions(+), 35 deletions(-) diff --git a/crates/autopilot/src/database/competition.rs b/crates/autopilot/src/database/competition.rs index 682ac81d1d..81cc5e63ef 100644 --- a/crates/autopilot/src/database/competition.rs +++ b/crates/autopilot/src/database/competition.rs @@ -139,28 +139,4 @@ impl super::Postgres { Ok(()) } - - /// Finds solvers that won `last_auctions_count` consecutive auctions but - /// never settled any of them. The current block is used to prevent - /// selecting auctions with deadline after the current block since they - /// still can be settled. - pub async fn find_non_settling_solvers( - &self, - last_auctions_count: u32, - current_block: u64, - ) -> anyhow::Result> { - let mut ex = self.pool.acquire().await.context("acquire")?; - let _timer = super::Metrics::get() - .database_queries - .with_label_values(&["find_non_settling_solvers"]) - .start_timer(); - - database::solver_competition::find_non_settling_solvers( - &mut ex, - last_auctions_count, - current_block, - ) - .await - .context("solver_competition::find_non_settling_solvers") - } } diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index f181265bf5..ef8280e3bc 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -1,6 +1,5 @@ use { crate::{ - database::Postgres, domain::{eth, Metrics}, infra, }, @@ -18,7 +17,7 @@ use { pub(super) struct Validator(Arc); struct Inner { - db: Postgres, + persistence: infra::Persistence, banned_solvers: dashmap::DashMap, ttl: Duration, last_auctions_count: u32, @@ -27,7 +26,7 @@ struct Inner { impl Validator { pub fn new( - db: Postgres, + persistence: infra::Persistence, current_block: CurrentBlockWatcher, settlement_updates_receiver: tokio::sync::mpsc::UnboundedReceiver<()>, ttl: Duration, @@ -35,7 +34,7 @@ impl Validator { drivers_by_address: HashMap>, ) -> Self { let self_ = Self(Arc::new(Inner { - db, + persistence, banned_solvers: Default::default(), ttl, last_auctions_count, @@ -60,7 +59,7 @@ impl Validator { let current_block = current_block.borrow().number; match self_ .0 - .db + .persistence .find_non_settling_solvers(self_.0.last_auctions_count, current_block) .await { @@ -68,8 +67,7 @@ impl Validator { let non_settling_drivers = non_settling_solvers .into_iter() .filter_map(|solver| { - let address = eth::Address(solver.0.into()); - self_.0.drivers_by_address.get(&address).map(|driver| { + self_.0.drivers_by_address.get(&solver).map(|driver| { Metrics::get() .non_settling_solver .with_label_values(&[&driver.name]); diff --git a/crates/autopilot/src/domain/competition/participation_guard/mod.rs b/crates/autopilot/src/domain/competition/participation_guard/mod.rs index 925cf9e52e..aae595091d 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/mod.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/mod.rs @@ -4,7 +4,6 @@ mod onchain; use { crate::{ arguments::DbBasedSolverParticipationGuardConfig, - database::Postgres, domain::eth, infra::{self, Ethereum}, }, @@ -24,7 +23,7 @@ struct Inner { impl SolverParticipationGuard { pub fn new( eth: Ethereum, - db: Postgres, + persistence: infra::Persistence, settlement_updates_receiver: tokio::sync::mpsc::UnboundedReceiver<()>, db_based_validator_config: DbBasedSolverParticipationGuardConfig, drivers_by_address: HashMap>, @@ -34,7 +33,7 @@ impl SolverParticipationGuard { if db_based_validator_config.enabled { let current_block = eth.current_block().clone(); let database_solver_participation_validator = db::Validator::new( - db, + persistence, current_block, settlement_updates_receiver, db_based_validator_config.solver_blacklist_cache_ttl, diff --git a/crates/autopilot/src/infra/persistence/mod.rs b/crates/autopilot/src/infra/persistence/mod.rs index 5a46e35a7e..813ef4fb4c 100644 --- a/crates/autopilot/src/infra/persistence/mod.rs +++ b/crates/autopilot/src/infra/persistence/mod.rs @@ -797,6 +797,33 @@ impl Persistence { ex.commit().await?; Ok(()) } + + /// Finds solvers that won `last_auctions_count` consecutive auctions but + /// never settled any of them. The current block is used to prevent + /// selecting auctions with deadline after the current block since they + /// still can be settled. + pub async fn find_non_settling_solvers( + &self, + last_auctions_count: u32, + current_block: u64, + ) -> anyhow::Result> { + let mut ex = self.postgres.pool.acquire().await.context("acquire")?; + let _timer = Metrics::get() + .database_queries + .with_label_values(&["find_non_settling_solvers"]) + .start_timer(); + + Ok(database::solver_competition::find_non_settling_solvers( + &mut ex, + last_auctions_count, + current_block, + ) + .await + .context("failed to fetch non-settling solvers")? + .into_iter() + .map(|solver| eth::Address(solver.0.into())) + .collect()) + } } #[derive(prometheus_metric_storage::MetricStorage)] diff --git a/crates/autopilot/src/run.rs b/crates/autopilot/src/run.rs index 497fd19af8..1320dfcbe3 100644 --- a/crates/autopilot/src/run.rs +++ b/crates/autopilot/src/run.rs @@ -576,7 +576,7 @@ pub async fn run(args: Arguments) { let solver_participation_guard = SolverParticipationGuard::new( eth.clone(), - db.clone(), + persistence.clone(), competition_updates_receiver, args.db_based_solver_participation_guard, drivers From cba693a38622f2d6a5fc42f50c9c23c855351e17 Mon Sep 17 00:00:00 2001 From: ilya Date: Wed, 12 Feb 2025 14:21:22 +0000 Subject: [PATCH 52/82] Naming --- .../src/domain/competition/participation_guard/db.rs | 8 ++++---- .../src/domain/competition/participation_guard/mod.rs | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index ef8280e3bc..dc00916646 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -28,7 +28,7 @@ impl Validator { pub fn new( persistence: infra::Persistence, current_block: CurrentBlockWatcher, - settlement_updates_receiver: tokio::sync::mpsc::UnboundedReceiver<()>, + competition_updates_receiver: tokio::sync::mpsc::UnboundedReceiver<()>, ttl: Duration, last_auctions_count: u32, drivers_by_address: HashMap>, @@ -41,7 +41,7 @@ impl Validator { drivers_by_address, })); - self_.start_maintenance(settlement_updates_receiver, current_block); + self_.start_maintenance(competition_updates_receiver, current_block); self_ } @@ -50,12 +50,12 @@ impl Validator { /// avoid redundant DB queries. fn start_maintenance( &self, - mut settlement_updates_receiver: tokio::sync::mpsc::UnboundedReceiver<()>, + mut competition_updates_receiver: tokio::sync::mpsc::UnboundedReceiver<()>, current_block: CurrentBlockWatcher, ) { let self_ = self.clone(); tokio::spawn(async move { - while settlement_updates_receiver.recv().await.is_some() { + while competition_updates_receiver.recv().await.is_some() { let current_block = current_block.borrow().number; match self_ .0 diff --git a/crates/autopilot/src/domain/competition/participation_guard/mod.rs b/crates/autopilot/src/domain/competition/participation_guard/mod.rs index aae595091d..75e89d0679 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/mod.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/mod.rs @@ -24,7 +24,7 @@ impl SolverParticipationGuard { pub fn new( eth: Ethereum, persistence: infra::Persistence, - settlement_updates_receiver: tokio::sync::mpsc::UnboundedReceiver<()>, + competition_updates_receiver: tokio::sync::mpsc::UnboundedReceiver<()>, db_based_validator_config: DbBasedSolverParticipationGuardConfig, drivers_by_address: HashMap>, ) -> Self { @@ -35,7 +35,7 @@ impl SolverParticipationGuard { let database_solver_participation_validator = db::Validator::new( persistence, current_block, - settlement_updates_receiver, + competition_updates_receiver, db_based_validator_config.solver_blacklist_cache_ttl, db_based_validator_config.solver_last_auctions_participation_count, drivers_by_address, From 4523b52141ec98ac68f1189882776b84752ab727 Mon Sep 17 00:00:00 2001 From: ilya Date: Wed, 12 Feb 2025 14:26:32 +0000 Subject: [PATCH 53/82] Formatting --- .../src/domain/competition/participation_guard/mod.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/crates/autopilot/src/domain/competition/participation_guard/mod.rs b/crates/autopilot/src/domain/competition/participation_guard/mod.rs index 54e9a72ecf..63eb301e3a 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/mod.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/mod.rs @@ -2,11 +2,7 @@ mod db; mod onchain; use { - crate::{ - arguments::DbBasedSolverParticipationGuardConfig, - domain::eth, - infra, - }, + crate::{arguments::DbBasedSolverParticipationGuardConfig, domain::eth, infra}, std::{collections::HashMap, sync::Arc}, }; From 0d57971a8baabda4ec342d69b8c39c80434160cc Mon Sep 17 00:00:00 2001 From: ilya Date: Wed, 12 Feb 2025 14:35:10 +0000 Subject: [PATCH 54/82] Fix after merge --- .../competition/participation_guard/db.rs | 11 ++------ crates/autopilot/src/infra/persistence/mod.rs | 28 +++++++++++++++++++ 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index c2a42ba82a..be572fbc1d 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -5,7 +5,6 @@ use { LowSettlingSolversFinderConfig, NonSettlingSolversFinderConfig, }, - database::Postgres, domain::{eth, Metrics}, infra::{self, solvers::dto}, }, @@ -110,10 +109,7 @@ impl SolverValidator { ) .await { - Ok(solvers) => solvers - .into_iter() - .map(|solver| eth::Address(solver.0.into())) - .collect(), + Ok(solvers) => solvers.into_iter().collect(), Err(err) => { tracing::warn!(?err, "error while searching for non-settling solvers"); Default::default() @@ -138,10 +134,7 @@ impl SolverValidator { ) .await { - Ok(solvers) => solvers - .into_iter() - .map(|solver| eth::Address(solver.0.into())) - .collect(), + Ok(solvers) => solvers.into_iter().collect(), Err(err) => { tracing::warn!(?err, "error while searching for low-settling solvers"); Default::default() diff --git a/crates/autopilot/src/infra/persistence/mod.rs b/crates/autopilot/src/infra/persistence/mod.rs index 813ef4fb4c..72129eebac 100644 --- a/crates/autopilot/src/infra/persistence/mod.rs +++ b/crates/autopilot/src/infra/persistence/mod.rs @@ -824,6 +824,34 @@ impl Persistence { .map(|solver| eth::Address(solver.0.into())) .collect()) } + + /// Finds solvers that have a failure settling rate above the given + /// ratio. The current block is used to prevent selecting auctions with + /// deadline after the current block since they still can be settled. + pub async fn find_low_settling_solvers( + &self, + last_auctions_count: u32, + current_block: u64, + max_failure_rate: f64, + ) -> anyhow::Result> { + let mut ex = self.postgres.pool.acquire().await.context("acquire")?; + let _timer = Metrics::get() + .database_queries + .with_label_values(&["find_low_settling_solvers"]) + .start_timer(); + + Ok(database::solver_competition::find_low_settling_solvers( + &mut ex, + last_auctions_count, + current_block, + max_failure_rate, + ) + .await + .context("solver_competition::find_low_settling_solvers")? + .into_iter() + .map(|solver| eth::Address(solver.0.into())) + .collect()) + } } #[derive(prometheus_metric_storage::MetricStorage)] From c3c9433f7bb38c61633ee2437f7877da6fd684d6 Mon Sep 17 00:00:00 2001 From: ilya Date: Fri, 14 Feb 2025 09:41:00 +0000 Subject: [PATCH 55/82] Comment --- .../src/domain/competition/participation_guard/db.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index dc00916646..c843c2fb7e 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -46,8 +46,9 @@ impl Validator { self_ } - /// Update the internal cache only once the settlement table is updated to - /// avoid redundant DB queries. + /// Update the internal cache only once the competition auctions table is + /// updated to avoid redundant DB queries on each block or any other + /// timeout. fn start_maintenance( &self, mut competition_updates_receiver: tokio::sync::mpsc::UnboundedReceiver<()>, From b773f66744017b4d7145e3eb9f49a3b102732f95 Mon Sep 17 00:00:00 2001 From: ilya Date: Fri, 14 Feb 2025 10:38:16 +0000 Subject: [PATCH 56/82] Api description --- crates/driver/openapi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/driver/openapi.yml b/crates/driver/openapi.yml index 4a43152472..2cea19933f 100644 --- a/crates/driver/openapi.yml +++ b/crates/driver/openapi.yml @@ -141,7 +141,7 @@ paths: /notify: post: description: | - Receive a notification in case the driver was banned for a certain reason. + Receive a notification with a specific reason. requestBody: required: true content: From ab5c6165b3e34f1a7b5697355b14cb5887034827 Mon Sep 17 00:00:00 2001 From: ilya Date: Fri, 14 Feb 2025 12:15:58 +0000 Subject: [PATCH 57/82] Drop error response --- crates/driver/openapi.yml | 10 ---------- crates/driver/src/infra/api/error.rs | 9 --------- crates/driver/src/infra/api/routes/mod.rs | 2 +- .../driver/src/infra/api/routes/notify/dto/mod.rs | 2 +- .../infra/api/routes/notify/dto/notify_request.rs | 6 ------ crates/driver/src/infra/api/routes/notify/mod.rs | 14 ++++++-------- 6 files changed, 8 insertions(+), 35 deletions(-) diff --git a/crates/driver/openapi.yml b/crates/driver/openapi.yml index 2cea19933f..3e810c4ac6 100644 --- a/crates/driver/openapi.yml +++ b/crates/driver/openapi.yml @@ -155,16 +155,6 @@ paths: responses: "200": description: notification successfully received. - "500": - description: Unable to notify solver. - content: - application/json: - schema: - type: object - properties: - error: - type: string - example: "Unable to notify solver" components: schemas: Address: diff --git a/crates/driver/src/infra/api/error.rs b/crates/driver/src/infra/api/error.rs index e95768c616..e788bbe5bf 100644 --- a/crates/driver/src/infra/api/error.rs +++ b/crates/driver/src/infra/api/error.rs @@ -112,12 +112,3 @@ impl From for (hyper::StatusCode, axum::Json) { error.into() } } - -impl From for (hyper::StatusCode, axum::Json) { - fn from(value: api::routes::NotifyError) -> Self { - let error = match value { - api::routes::NotifyError::UnableToNotify => Kind::Unknown, - }; - error.into() - } -} diff --git a/crates/driver/src/infra/api/routes/mod.rs b/crates/driver/src/infra/api/routes/mod.rs index d4ed5e12e3..962dd5587e 100644 --- a/crates/driver/src/infra/api/routes/mod.rs +++ b/crates/driver/src/infra/api/routes/mod.rs @@ -11,7 +11,7 @@ pub(super) use { healthz::healthz, info::info, metrics::metrics, - notify::{notify, NotifyError}, + notify::notify, quote::{quote, OrderError}, reveal::reveal, settle::settle, diff --git a/crates/driver/src/infra/api/routes/notify/dto/mod.rs b/crates/driver/src/infra/api/routes/notify/dto/mod.rs index 3f99a96ff7..9a24eedbc1 100644 --- a/crates/driver/src/infra/api/routes/notify/dto/mod.rs +++ b/crates/driver/src/infra/api/routes/notify/dto/mod.rs @@ -1,3 +1,3 @@ mod notify_request; -pub use notify_request::{Error as NotifyError, NotifyRequest}; +pub use notify_request::NotifyRequest; diff --git a/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs b/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs index 4eb36c20ef..4dbb8210d5 100644 --- a/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs +++ b/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs @@ -35,9 +35,3 @@ impl From for notify::Kind { } } } - -#[derive(Debug, thiserror::Error)] -pub enum Error { - #[error("Unable to notify solver")] - UnableToNotify, -} diff --git a/crates/driver/src/infra/api/routes/notify/mod.rs b/crates/driver/src/infra/api/routes/notify/mod.rs index e6ae815b82..d8922b7631 100644 --- a/crates/driver/src/infra/api/routes/notify/mod.rs +++ b/crates/driver/src/infra/api/routes/notify/mod.rs @@ -1,7 +1,5 @@ mod dto; -pub use dto::NotifyError; - use crate::infra::api::{Error, State}; pub(in crate::infra::api) fn notify(router: axum::Router) -> axum::Router { @@ -14,10 +12,10 @@ async fn route( ) -> Result)> { let solver = &state.solver().name().0; tracing::debug!(?req, ?solver, "received a notification"); - state - .solver() - .notify(None, None, req.0.into()) - .await - .map(|_| hyper::StatusCode::OK) - .map_err(|_| NotifyError::UnableToNotify.into()) + + if let Err(err) = state.solver().notify(None, None, req.0.into()).await { + tracing::debug!(?err, "failed to notify solver"); + } + + Ok(hyper::StatusCode::OK) } From 680e8065930898450937b1c4346e13d99b5ea107 Mon Sep 17 00:00:00 2001 From: ilya Date: Fri, 14 Feb 2025 12:25:37 +0000 Subject: [PATCH 58/82] Naming --- crates/driver/src/infra/solver/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/driver/src/infra/solver/mod.rs b/crates/driver/src/infra/solver/mod.rs index 6a6b3f8386..bda080e0b1 100644 --- a/crates/driver/src/infra/solver/mod.rs +++ b/crates/driver/src/infra/solver/mod.rs @@ -270,7 +270,7 @@ impl Solver { let client = self.client.clone(); let response_limit = self.config.response_size_limit_max_bytes; let future = async move { - if let Err(error) = Self::notify_( + if let Err(error) = Self::notify_inner( base_url, client, response_limit, @@ -296,7 +296,7 @@ impl Solver { let base_url = self.config.endpoint.clone(); let client = self.client.clone(); - Self::notify_( + Self::notify_inner( base_url, client, self.config.response_size_limit_max_bytes, @@ -307,7 +307,7 @@ impl Solver { .await } - async fn notify_( + async fn notify_inner( base_url: url::Url, client: reqwest::Client, response_limit: usize, From 4f6cd1d0433e56dbf9d44511db5b9fd2d5892a81 Mon Sep 17 00:00:00 2001 From: ilya Date: Mon, 17 Feb 2025 11:58:20 +0000 Subject: [PATCH 59/82] Comments --- crates/autopilot/src/arguments.rs | 4 +++- .../src/domain/competition/participation_guard/db.rs | 11 ++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/crates/autopilot/src/arguments.rs b/crates/autopilot/src/arguments.rs index b581c504d0..4d2b9f4a46 100644 --- a/crates/autopilot/src/arguments.rs +++ b/crates/autopilot/src/arguments.rs @@ -262,7 +262,9 @@ pub struct DbBasedSolverParticipationGuardConfig { )] pub enabled: bool, - /// The time-to-live for the solver participation blacklist cache. + /// Sets the duration for which the solver remains blacklisted. + /// Technically, the time-to-live for the solver participation blacklist + /// cache. #[clap(long, env, default_value = "5m", value_parser = humantime::parse_duration)] pub solver_blacklist_cache_ttl: Duration, diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index c843c2fb7e..e0f254a9fe 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -88,8 +88,8 @@ impl Validator { let now = Instant::now(); non_settling_drivers .into_iter() - // Check if solver accepted this feature. This should be removed once a CIP is - // approved. + // Check if solver accepted this feature. This should be removed once the CIP + // making this mandatory has been approved. .filter_map(|driver| { driver.accepts_unsettled_blocking.then_some(driver.submission_address) }) @@ -102,6 +102,7 @@ impl Validator { } } } + tracing::error!("competition_updates_receiver closed"); }); } } @@ -110,11 +111,7 @@ impl Validator { impl super::Validator for Validator { async fn is_allowed(&self, solver: ð::Address) -> anyhow::Result { if let Some(entry) = self.0.banned_solvers.get(solver) { - if Instant::now().duration_since(*entry.value()) < self.0.ttl { - return Ok(false); - } else { - self.0.banned_solvers.remove(solver); - } + return Ok(entry.elapsed() >= self.0.ttl); } Ok(true) From bdd33d0b381844fa79f948de2487357c1ba26e17 Mon Sep 17 00:00:00 2001 From: ilya Date: Mon, 17 Feb 2025 12:03:17 +0000 Subject: [PATCH 60/82] Simplify the code --- .../competition/participation_guard/db.rs | 57 +++++++------------ 1 file changed, 22 insertions(+), 35 deletions(-) diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index e0f254a9fe..2304302715 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -58,51 +58,38 @@ impl Validator { tokio::spawn(async move { while competition_updates_receiver.recv().await.is_some() { let current_block = current_block.borrow().number; - match self_ + let non_settling_solvers = match self_ .0 .persistence .find_non_settling_solvers(self_.0.last_auctions_count, current_block) .await { - Ok(non_settling_solvers) => { - let non_settling_drivers = non_settling_solvers - .into_iter() - .filter_map(|solver| { - self_.0.drivers_by_address.get(&solver).map(|driver| { - Metrics::get() - .non_settling_solver - .with_label_values(&[&driver.name]); - - driver.clone() - }) - }) - .collect::>(); - - let non_settling_solver_names = non_settling_drivers - .iter() - .map(|driver| driver.name.clone()) - .collect::>(); + Ok(non_settling_solvers) => non_settling_solvers, + Err(err) => { + tracing::warn!(?err, "error while searching for non-settling solvers"); + continue; + } + }; - tracing::debug!(solvers = ?non_settling_solver_names, "found non-settling solvers"); + tracing::debug!(solvers = ?non_settling_solvers, "found non-settling solvers"); - let now = Instant::now(); - non_settling_drivers - .into_iter() - // Check if solver accepted this feature. This should be removed once the CIP - // making this mandatory has been approved. - .filter_map(|driver| { - driver.accepts_unsettled_blocking.then_some(driver.submission_address) - }) - .for_each(|solver| { - self_.0.banned_solvers.insert(solver, now); - }); - } - Err(err) => { - tracing::warn!(?err, "error while searching for non-settling solvers") + let now = Instant::now(); + for solver in non_settling_solvers { + let Some(driver) = self_.0.drivers_by_address.get(&solver) else { + continue; + }; + Metrics::get() + .non_settling_solver + .with_label_values(&[&driver.name]); + // Check if solver accepted this feature. This should be removed once the CIP + // making this mandatory has been approved. + if driver.accepts_unsettled_blocking { + tracing::debug!(?solver, "disabling solver temporarily"); + self_.0.banned_solvers.insert(solver, now); } } } - tracing::error!("competition_updates_receiver closed"); + tracing::error!("stream of settlement updates terminated unexpectedly"); }); } } From fd0fc273c8bd223b6088f08c228049abcc32a339 Mon Sep 17 00:00:00 2001 From: ilya Date: Mon, 17 Feb 2025 12:15:54 +0000 Subject: [PATCH 61/82] Nits --- .../domain/competition/participation_guard/mod.rs | 9 ++++++--- crates/autopilot/src/run.rs | 5 +---- crates/database/src/solver_competition.rs | 12 ++++++++++++ 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/crates/autopilot/src/domain/competition/participation_guard/mod.rs b/crates/autopilot/src/domain/competition/participation_guard/mod.rs index 75e89d0679..00a2759f21 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/mod.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/mod.rs @@ -7,7 +7,7 @@ use { domain::eth, infra::{self, Ethereum}, }, - std::{collections::HashMap, sync::Arc}, + std::sync::Arc, }; /// This struct checks whether a solver can participate in the competition by @@ -26,7 +26,7 @@ impl SolverParticipationGuard { persistence: infra::Persistence, competition_updates_receiver: tokio::sync::mpsc::UnboundedReceiver<()>, db_based_validator_config: DbBasedSolverParticipationGuardConfig, - drivers_by_address: HashMap>, + drivers: impl IntoIterator>, ) -> Self { let mut validators: Vec> = Vec::new(); @@ -38,7 +38,10 @@ impl SolverParticipationGuard { competition_updates_receiver, db_based_validator_config.solver_blacklist_cache_ttl, db_based_validator_config.solver_last_auctions_participation_count, - drivers_by_address, + drivers + .into_iter() + .map(|driver| (driver.submission_address, driver.clone())) + .collect(), ); validators.push(Box::new(database_solver_participation_validator)); } diff --git a/crates/autopilot/src/run.rs b/crates/autopilot/src/run.rs index 1320dfcbe3..24faf50c72 100644 --- a/crates/autopilot/src/run.rs +++ b/crates/autopilot/src/run.rs @@ -579,10 +579,7 @@ pub async fn run(args: Arguments) { persistence.clone(), competition_updates_receiver, args.db_based_solver_participation_guard, - drivers - .iter() - .map(|driver| (driver.submission_address, driver.clone())) - .collect(), + drivers.iter().cloned(), ); let run = RunLoop::new( diff --git a/crates/database/src/solver_competition.rs b/crates/database/src/solver_competition.rs index 9fc22fa561..f778c4315a 100644 --- a/crates/database/src/solver_competition.rs +++ b/crates/database/src/solver_competition.rs @@ -97,6 +97,18 @@ GROUP BY sc.id sqlx::query_as(QUERY).bind(tx_hash).fetch_optional(ex).await } +/// Identifies solvers that have consistently failed to settle solutions in +/// recent N auctions. +/// +/// 1. Retrieves `last_auctions_count` most recent auctions already ended +/// auctions by filtering them by their deadlines. +/// 2. Identifies solvers who won these auctions but did not submit a successful +/// settlement. +/// 3. Counts how often each solver appears in these unsuccessful cases. +/// 4. Determines the total number of auctions considered. +/// 5. Flags solvers who failed to settle in all of these auctions. +/// 6. Returns a list of solvers that have consistently failed to settle +/// solutions. pub async fn find_non_settling_solvers( ex: &mut PgConnection, last_auctions_count: u32, From 38ad5369fc2361c40171a1a435ceee4626d5bd7b Mon Sep 17 00:00:00 2001 From: ilya Date: Mon, 17 Feb 2025 12:57:36 +0000 Subject: [PATCH 62/82] ISO timestamp --- .../competition/participation_guard/db.rs | 10 ++----- .../autopilot/src/infra/solvers/dto/notify.rs | 8 +++-- crates/autopilot/src/infra/solvers/mod.rs | 30 +++++++------------ .../api/routes/notify/dto/notify_request.rs | 16 +++++----- .../driver/src/infra/notify/notification.rs | 3 +- .../src/infra/solver/dto/notification.rs | 10 +++---- 6 files changed, 35 insertions(+), 42 deletions(-) diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index bf0e5fff84..0467c41b3f 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -3,8 +3,8 @@ use { domain::{eth, Metrics}, infra, }, + chrono::Utc, ethrpc::block_stream::CurrentBlockWatcher, - model::time::now_in_epoch_seconds, std::{ collections::HashMap, sync::Arc, @@ -94,12 +94,8 @@ impl Validator { .collect::>(); let now = Instant::now(); - let banned_until_timestamp = - u64::from(now_in_epoch_seconds()) + self_.0.ttl.as_secs(); - infra::notify_non_settling_solvers( - &non_settling_drivers, - banned_until_timestamp, - ); + let banned_until = Utc::now() + self_.0.ttl; + infra::notify_non_settling_solvers(&non_settling_drivers, banned_until); for driver in non_settling_drivers { self_ diff --git a/crates/autopilot/src/infra/solvers/dto/notify.rs b/crates/autopilot/src/infra/solvers/dto/notify.rs index 0ac2e786f9..1584b60e87 100644 --- a/crates/autopilot/src/infra/solvers/dto/notify.rs +++ b/crates/autopilot/src/infra/solvers/dto/notify.rs @@ -1,4 +1,8 @@ -use {serde::Serialize, serde_with::serde_as}; +use { + chrono::{DateTime, Utc}, + serde::Serialize, + serde_with::serde_as, +}; #[serde_as] #[derive(Debug, Serialize)] @@ -6,7 +10,7 @@ use {serde::Serialize, serde_with::serde_as}; pub enum Request { Banned { reason: BanReason, - until_timestamp: u64, + until: DateTime, }, } diff --git a/crates/autopilot/src/infra/solvers/mod.rs b/crates/autopilot/src/infra/solvers/mod.rs index 264ab0931b..334439cdf7 100644 --- a/crates/autopilot/src/infra/solvers/mod.rs +++ b/crates/autopilot/src/infra/solvers/mod.rs @@ -2,7 +2,7 @@ use { self::dto::{reveal, settle, solve}, crate::{arguments::Account, domain::eth, infra::solvers::dto::notify, util}, anyhow::{anyhow, Context, Result}, - ethcontract::jsonrpc::futures_util::future::join_all, + chrono::{DateTime, Utc}, reqwest::{Client, StatusCode}, std::{sync::Arc, time::Duration}, thiserror::Error, @@ -178,28 +178,20 @@ pub async fn response_body_with_size_limit( Ok(bytes) } -/// Try to notify all the non-settling solvers in a background task. +/// Notifies all non-settling solvers in a fire-and-forget manner. pub fn notify_non_settling_solvers( non_settling_drivers: &[Arc], - banned_until_timestamp: u64, + banned_until: DateTime, ) { - let futures = non_settling_drivers - .iter() - .cloned() - .map(|driver| async move { - if let Err(err) = driver + for driver in non_settling_drivers { + let driver = Arc::clone(driver); + tokio::spawn(async move { + let _ = driver .notify(¬ify::Request::Banned { reason: notify::BanReason::UnsettledConsecutiveAuctions, - until_timestamp: banned_until_timestamp, + until: banned_until, }) - .await - { - tracing::debug!(solver = ?driver.name, ?err, "unable to notify external solver"); - } - }) - .collect::>(); - - tokio::spawn(async move { - join_all(futures).await; - }); + .await; + }); + } } diff --git a/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs b/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs index 4dbb8210d5..38ad8d943d 100644 --- a/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs +++ b/crates/driver/src/infra/api/routes/notify/dto/notify_request.rs @@ -1,4 +1,9 @@ -use {crate::infra::notify, serde::Deserialize, serde_with::serde_as}; +use { + crate::infra::notify, + chrono::{DateTime, Utc}, + serde::Deserialize, + serde_with::serde_as, +}; #[serde_as] #[derive(Debug, Deserialize)] @@ -6,7 +11,7 @@ use {crate::infra::notify, serde::Deserialize, serde_with::serde_as}; pub enum NotifyRequest { Banned { reason: BanReason, - until_timestamp: u64, + until: DateTime, }, } @@ -21,16 +26,13 @@ pub enum BanReason { impl From for notify::Kind { fn from(value: NotifyRequest) -> Self { match value { - NotifyRequest::Banned { - reason, - until_timestamp, - } => notify::Kind::Banned { + NotifyRequest::Banned { reason, until } => notify::Kind::Banned { reason: match reason { BanReason::UnsettledConsecutiveAuctions => { notify::BanReason::UnsettledConsecutiveAuctions } }, - until_timestamp, + until, }, } } diff --git a/crates/driver/src/infra/notify/notification.rs b/crates/driver/src/infra/notify/notification.rs index 2b9fe40003..5e2bcb1e62 100644 --- a/crates/driver/src/infra/notify/notification.rs +++ b/crates/driver/src/infra/notify/notification.rs @@ -3,6 +3,7 @@ use { competition::{auction, solution}, eth::{self, Ether, TokenAddress}, }, + chrono::{DateTime, Utc}, std::collections::BTreeSet, }; @@ -48,7 +49,7 @@ pub enum Kind { /// The solver has been banned for a specific reason. Banned { reason: BanReason, - until_timestamp: u64, + until: DateTime, }, } diff --git a/crates/driver/src/infra/solver/dto/notification.rs b/crates/driver/src/infra/solver/dto/notification.rs index 49f55ac399..fc2f91e263 100644 --- a/crates/driver/src/infra/solver/dto/notification.rs +++ b/crates/driver/src/infra/solver/dto/notification.rs @@ -7,6 +7,7 @@ use { infra::notify, util::serialize, }, + chrono::{DateTime, Utc}, serde::Serialize, serde_with::serde_as, std::collections::BTreeSet, @@ -61,16 +62,13 @@ impl Notification { notify::Settlement::Expired => Kind::Expired, }, notify::Kind::PostprocessingTimedOut => Kind::PostprocessingTimedOut, - notify::Kind::Banned { - reason, - until_timestamp, - } => Kind::Banned { + notify::Kind::Banned { reason, until } => Kind::Banned { reason: match reason { notify::BanReason::UnsettledConsecutiveAuctions => { BanReason::UnsettledConsecutiveAuctions } }, - until_timestamp, + until, }, }, } @@ -157,7 +155,7 @@ pub enum Kind { PostprocessingTimedOut, Banned { reason: BanReason, - until_timestamp: u64, + until: DateTime, }, } From c76761a05f88db2699a8fdff481e6da616ef2431 Mon Sep 17 00:00:00 2001 From: ilya Date: Mon, 17 Feb 2025 14:07:11 +0000 Subject: [PATCH 63/82] As str --- .../src/domain/competition/participation_guard/db.rs | 7 +------ crates/autopilot/src/infra/solvers/dto/notify.rs | 9 +++++++++ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index 9bfbb0cb7e..fb562bc6b1 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -172,15 +172,10 @@ impl SolverValidator { .collect::>(); tracing::debug!(solvers = ?solver_names, log_message); - let reason = match ban_reason { - dto::notify::BanReason::UnsettledConsecutiveAuctions => "non_settling", - dto::notify::BanReason::HighSettleFailureRate => "high_settle_failure_rate", - }; - for solver in solver_names { Metrics::get() .banned_solver - .with_label_values(&[&solver, reason]); + .with_label_values(&[&solver, ban_reason.as_str()]); } let banned_drivers = drivers diff --git a/crates/autopilot/src/infra/solvers/dto/notify.rs b/crates/autopilot/src/infra/solvers/dto/notify.rs index 4a92306ec9..2a71dc9454 100644 --- a/crates/autopilot/src/infra/solvers/dto/notify.rs +++ b/crates/autopilot/src/infra/solvers/dto/notify.rs @@ -19,3 +19,12 @@ pub enum BanReason { /// Driver's settle failure rate is above the threshold. HighSettleFailureRate, } + +impl BanReason { + pub fn as_str(&self) -> &'static str { + match self { + BanReason::UnsettledConsecutiveAuctions => "non_settling", + BanReason::HighSettleFailureRate => "high_settle_failure_rate", + } + } +} From 1a35e9dacda468a65461d73b0e242e68601f3eab Mon Sep 17 00:00:00 2001 From: ilya Date: Mon, 17 Feb 2025 14:10:36 +0000 Subject: [PATCH 64/82] Nit --- .../src/domain/competition/participation_guard/db.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index fb562bc6b1..65229f3c04 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -168,14 +168,14 @@ impl SolverValidator { }; let solver_names = drivers .iter() - .map(|driver| driver.name.clone()) + .map(|driver| driver.name.as_ref()) .collect::>(); tracing::debug!(solvers = ?solver_names, log_message); for solver in solver_names { Metrics::get() .banned_solver - .with_label_values(&[&solver, ban_reason.as_str()]); + .with_label_values(&[solver, ban_reason.as_str()]); } let banned_drivers = drivers From d4b87bd09afe17d43e7eb88d8aff7f7ed873cecd Mon Sep 17 00:00:00 2001 From: ilya Date: Mon, 17 Feb 2025 14:22:40 +0000 Subject: [PATCH 65/82] Fix after merge --- .../competition/participation_guard/db.rs | 2 +- .../competition/participation_guard/mod.rs | 6 +---- crates/autopilot/src/infra/mod.rs | 2 +- crates/autopilot/src/infra/solvers/mod.rs | 26 +++++++------------ 4 files changed, 13 insertions(+), 23 deletions(-) diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index 81c580ae57..d32e026505 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -87,7 +87,7 @@ impl Validator { // making this mandatory has been approved. if driver.accepts_unsettled_blocking { tracing::debug!(?solver, "disabling solver temporarily"); - infra::notify_non_settling_solver(&solver, banned_until); + infra::notify_non_settling_solver(driver.clone(), banned_until); self_.0.banned_solvers.insert(solver, now); } } diff --git a/crates/autopilot/src/domain/competition/participation_guard/mod.rs b/crates/autopilot/src/domain/competition/participation_guard/mod.rs index c8e7dc7f4c..7c0dd7db33 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/mod.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/mod.rs @@ -2,11 +2,7 @@ mod db; mod onchain; use { - crate::{ - arguments::DbBasedSolverParticipationGuardConfig, - domain::eth, - infra::self, - }, + crate::{arguments::DbBasedSolverParticipationGuardConfig, domain::eth, infra}, std::sync::Arc, }; diff --git a/crates/autopilot/src/infra/mod.rs b/crates/autopilot/src/infra/mod.rs index b8d71918e7..28046fa333 100644 --- a/crates/autopilot/src/infra/mod.rs +++ b/crates/autopilot/src/infra/mod.rs @@ -7,5 +7,5 @@ pub use { blockchain::Ethereum, order_validation::banned, persistence::Persistence, - solvers::{notify_non_settling_solvers, Driver}, + solvers::{notify_non_settling_solver, Driver}, }; diff --git a/crates/autopilot/src/infra/solvers/mod.rs b/crates/autopilot/src/infra/solvers/mod.rs index 334439cdf7..bc0b3361e6 100644 --- a/crates/autopilot/src/infra/solvers/mod.rs +++ b/crates/autopilot/src/infra/solvers/mod.rs @@ -178,20 +178,14 @@ pub async fn response_body_with_size_limit( Ok(bytes) } -/// Notifies all non-settling solvers in a fire-and-forget manner. -pub fn notify_non_settling_solvers( - non_settling_drivers: &[Arc], - banned_until: DateTime, -) { - for driver in non_settling_drivers { - let driver = Arc::clone(driver); - tokio::spawn(async move { - let _ = driver - .notify(¬ify::Request::Banned { - reason: notify::BanReason::UnsettledConsecutiveAuctions, - until: banned_until, - }) - .await; - }); - } +/// Notifies the non-settling driver in a fire-and-forget manner. +pub fn notify_non_settling_solver(non_settling_driver: Arc, banned_until: DateTime) { + tokio::spawn(async move { + let _ = non_settling_driver + .notify(¬ify::Request::Banned { + reason: notify::BanReason::UnsettledConsecutiveAuctions, + until: banned_until, + }) + .await; + }); } From e5250a56a54a9d41b989b71e028b887768613575 Mon Sep 17 00:00:00 2001 From: ilya Date: Mon, 17 Feb 2025 14:58:12 +0000 Subject: [PATCH 66/82] Solver names in the log --- .../src/domain/competition/participation_guard/db.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index 2304302715..49f61d79ac 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -71,23 +71,25 @@ impl Validator { } }; - tracing::debug!(solvers = ?non_settling_solvers, "found non-settling solvers"); - + let mut non_settling_solver_names: Vec<&str> = Vec::new(); let now = Instant::now(); for solver in non_settling_solvers { let Some(driver) = self_.0.drivers_by_address.get(&solver) else { continue; }; + non_settling_solver_names.push(driver.name.as_ref()); Metrics::get() .non_settling_solver .with_label_values(&[&driver.name]); // Check if solver accepted this feature. This should be removed once the CIP // making this mandatory has been approved. if driver.accepts_unsettled_blocking { - tracing::debug!(?solver, "disabling solver temporarily"); + tracing::debug!(solver = ?driver.name, "disabling solver temporarily"); self_.0.banned_solvers.insert(solver, now); } } + + tracing::debug!(solvers = ?non_settling_solver_names, "found non-settling solvers"); } tracing::error!("stream of settlement updates terminated unexpectedly"); }); From 204984f1b6882b9a458ea863cac9c15bf6ca7bcd Mon Sep 17 00:00:00 2001 From: ilya Date: Mon, 17 Feb 2025 15:46:48 +0000 Subject: [PATCH 67/82] Fix after merge --- .../src/domain/competition/participation_guard/db.rs | 9 ++++----- crates/autopilot/src/infra/mod.rs | 2 +- crates/autopilot/src/infra/solvers/mod.rs | 10 ++++++---- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index afc9e1431e..199cc95c9a 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -1,4 +1,3 @@ -use chrono::DateTime; use { crate::{ arguments::{ @@ -9,7 +8,7 @@ use { domain::{eth, Metrics}, infra::{self, solvers::dto}, }, - chrono::Utc, + chrono::{DateTime, Utc}, ethrpc::block_stream::CurrentBlockWatcher, std::{ collections::{HashMap, HashSet}, @@ -154,7 +153,7 @@ impl SolverValidator { ) { let mut non_settling_solver_names: Vec<&str> = Vec::new(); for solver in solvers { - let Some(driver) = self.0.drivers_by_address.get(&solver) else { + let Some(driver) = self.0.drivers_by_address.get(solver) else { continue; }; non_settling_solver_names.push(driver.name.as_ref()); @@ -165,8 +164,8 @@ impl SolverValidator { // making this mandatory has been approved. if driver.accepts_unsettled_blocking { tracing::debug!(solver = ?driver.name, "disabling solver temporarily"); - infra::notify_non_settling_solver(driver.clone(), banned_until); - self.0.banned_solvers.insert(solver.clone(), found_at); + infra::notify_banned_solver(driver.clone(), ban_reason, banned_until); + self.0.banned_solvers.insert(*solver, found_at); } } diff --git a/crates/autopilot/src/infra/mod.rs b/crates/autopilot/src/infra/mod.rs index d960038283..77a76f95b4 100644 --- a/crates/autopilot/src/infra/mod.rs +++ b/crates/autopilot/src/infra/mod.rs @@ -7,5 +7,5 @@ pub use { blockchain::Ethereum, order_validation::banned, persistence::Persistence, - solvers::{notify_banned_solvers, Driver}, + solvers::{notify_banned_solver, Driver}, }; diff --git a/crates/autopilot/src/infra/solvers/mod.rs b/crates/autopilot/src/infra/solvers/mod.rs index f26b42b257..73cdad52ab 100644 --- a/crates/autopilot/src/infra/solvers/mod.rs +++ b/crates/autopilot/src/infra/solvers/mod.rs @@ -179,14 +179,16 @@ pub async fn response_body_with_size_limit( } /// Notifies the non-settling driver in a fire-and-forget manner. -pub fn notify_non_settling_solver(non_settling_driver: Arc,reason: ¬ify::BanReason, banned_until: DateTime) { +pub fn notify_banned_solver( + non_settling_driver: Arc, + reason: ¬ify::BanReason, + banned_until: DateTime, +) { let request = notify::Request::Banned { reason: reason.clone(), until: banned_until, }; tokio::spawn(async move { - let _ = non_settling_driver - .notify(&request) - .await; + let _ = non_settling_driver.notify(&request).await; }); } From 051bd50822c0e02f9a3f6b30d15c75ca3d8d67b6 Mon Sep 17 00:00:00 2001 From: ilya Date: Mon, 17 Feb 2025 17:37:21 +0000 Subject: [PATCH 68/82] Naming --- crates/autopilot/src/arguments.rs | 25 ++++++++----------- .../competition/participation_guard/db.rs | 2 +- crates/autopilot/src/infra/solvers/mod.rs | 6 ++--- crates/autopilot/src/run.rs | 4 +-- 4 files changed, 17 insertions(+), 20 deletions(-) diff --git a/crates/autopilot/src/arguments.rs b/crates/autopilot/src/arguments.rs index 4d2b9f4a46..63a23b6a36 100644 --- a/crates/autopilot/src/arguments.rs +++ b/crates/autopilot/src/arguments.rs @@ -416,7 +416,7 @@ pub struct Solver { pub url: Url, pub submission_account: Account, pub fairness_threshold: Option, - pub accepts_unsettled_blocking: bool, + pub requested_timeout_on_problems: bool, } #[derive(Debug, Clone, PartialEq, Eq, Hash)] @@ -466,7 +466,7 @@ impl FromStr for Solver { }; let mut fairness_threshold: Option = Default::default(); - let mut accepts_unsettled_blocking = false; + let mut requested_timeout_on_problems = false; if let Some(value) = parts.get(3) { match U256::from_dec_str(value) { @@ -474,17 +474,14 @@ impl FromStr for Solver { fairness_threshold = Some(parsed_fairness_threshold); } Err(_) => { - accepts_unsettled_blocking = value - .parse() - .context("failed to parse solver's third arg param")? + requested_timeout_on_problems = + value.to_lowercase() == "requested_timeout_on_problems"; } } }; if let Some(value) = parts.get(4) { - accepts_unsettled_blocking = value - .parse() - .context("failed to parse `accepts_unsettled_blocking` flag")?; + requested_timeout_on_problems = value.to_lowercase() == "requested_timeout_on_problems"; } Ok(Self { @@ -492,7 +489,7 @@ impl FromStr for Solver { url, fairness_threshold, submission_account, - accepts_unsettled_blocking, + requested_timeout_on_problems, }) } } @@ -689,7 +686,7 @@ mod test { name: "name1".into(), url: Url::parse("http://localhost:8080").unwrap(), fairness_threshold: None, - accepts_unsettled_blocking: false, + requested_timeout_on_problems: false, submission_account: Account::Address(H160::from_slice(&hex!( "C02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2" ))), @@ -705,7 +702,7 @@ mod test { name: "name1".into(), url: Url::parse("http://localhost:8080").unwrap(), fairness_threshold: None, - accepts_unsettled_blocking: false, + requested_timeout_on_problems: false, submission_account: Account::Kms( Arn::from_str("arn:aws:kms:supersecretstuff").unwrap(), ), @@ -724,7 +721,7 @@ mod test { "C02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2" ))), fairness_threshold: Some(U256::exp10(18)), - accepts_unsettled_blocking: false, + requested_timeout_on_problems: false, }; assert_eq!(driver, expected); } @@ -741,7 +738,7 @@ mod test { "C02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2" ))), fairness_threshold: None, - accepts_unsettled_blocking: true, + requested_timeout_on_problems: true, }; assert_eq!(driver, expected); } @@ -757,7 +754,7 @@ mod test { "C02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2" ))), fairness_threshold: Some(U256::exp10(18)), - accepts_unsettled_blocking: true, + requested_timeout_on_problems: true, }; assert_eq!(driver, expected); } diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index 49f61d79ac..d84be45405 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -83,7 +83,7 @@ impl Validator { .with_label_values(&[&driver.name]); // Check if solver accepted this feature. This should be removed once the CIP // making this mandatory has been approved. - if driver.accepts_unsettled_blocking { + if driver.requested_timeout_on_problems { tracing::debug!(solver = ?driver.name, "disabling solver temporarily"); self_.0.banned_solvers.insert(solver, now); } diff --git a/crates/autopilot/src/infra/solvers/mod.rs b/crates/autopilot/src/infra/solvers/mod.rs index aae623ebaa..0d8a28d026 100644 --- a/crates/autopilot/src/infra/solvers/mod.rs +++ b/crates/autopilot/src/infra/solvers/mod.rs @@ -21,7 +21,7 @@ pub struct Driver { // another driver solved with surplus exceeding this driver's surplus by `threshold` pub fairness_threshold: Option, pub submission_address: eth::Address, - pub accepts_unsettled_blocking: bool, + pub requested_timeout_on_problems: bool, client: Client, } @@ -39,7 +39,7 @@ impl Driver { name: String, fairness_threshold: Option, submission_account: Account, - accepts_unsettled_blocking: bool, + requested_timeout_on_problems: bool, ) -> Result { let submission_address = match submission_account { Account::Kms(key_id) => { @@ -72,7 +72,7 @@ impl Driver { .build() .map_err(Error::FailedToBuildClient)?, submission_address: submission_address.into(), - accepts_unsettled_blocking, + requested_timeout_on_problems, }) } diff --git a/crates/autopilot/src/run.rs b/crates/autopilot/src/run.rs index 24faf50c72..9d9b2b6887 100644 --- a/crates/autopilot/src/run.rs +++ b/crates/autopilot/src/run.rs @@ -561,7 +561,7 @@ pub async fn run(args: Arguments) { driver.name.clone(), driver.fairness_threshold.map(Into::into), driver.submission_account, - driver.accepts_unsettled_blocking, + driver.requested_timeout_on_problems, ) .await .map(Arc::new) @@ -614,7 +614,7 @@ async fn shadow_mode(args: Arguments) -> ! { driver.name.clone(), driver.fairness_threshold.map(Into::into), driver.submission_account, - driver.accepts_unsettled_blocking, + driver.requested_timeout_on_problems, ) .await .map(Arc::new) From 4bb86404e058ff23799ba856b1d85e1a95fa3959 Mon Sep 17 00:00:00 2001 From: ilya Date: Mon, 17 Feb 2025 17:41:21 +0000 Subject: [PATCH 69/82] Fixes unit tests --- crates/autopilot/src/arguments.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/autopilot/src/arguments.rs b/crates/autopilot/src/arguments.rs index 63a23b6a36..44715b3090 100644 --- a/crates/autopilot/src/arguments.rs +++ b/crates/autopilot/src/arguments.rs @@ -729,7 +729,7 @@ mod test { #[test] fn parse_driver_with_accepts_unsettled_blocking_flag() { let argument = - "name1|http://localhost:8080|0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2|true"; + "name1|http://localhost:8080|0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2|requested_timeout_on_problems"; let driver = Solver::from_str(argument).unwrap(); let expected = Solver { name: "name1".into(), @@ -745,7 +745,7 @@ mod test { #[test] fn parse_driver_with_threshold_and_accepts_unsettled_blocking_flag() { - let argument = "name1|http://localhost:8080|0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2|1000000000000000000|true"; + let argument = "name1|http://localhost:8080|0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2|1000000000000000000|requested_timeout_on_problems"; let driver = Solver::from_str(argument).unwrap(); let expected = Solver { name: "name1".into(), From de31f1ebe8b7142302df3e08628d74751429155b Mon Sep 17 00:00:00 2001 From: ilya Date: Tue, 18 Feb 2025 10:58:04 +0000 Subject: [PATCH 70/82] Nit --- .../competition/participation_guard/db.rs | 35 ++++++++++--------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index d84be45405..7ef9fddb67 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -71,23 +71,26 @@ impl Validator { } }; - let mut non_settling_solver_names: Vec<&str> = Vec::new(); let now = Instant::now(); - for solver in non_settling_solvers { - let Some(driver) = self_.0.drivers_by_address.get(&solver) else { - continue; - }; - non_settling_solver_names.push(driver.name.as_ref()); - Metrics::get() - .non_settling_solver - .with_label_values(&[&driver.name]); - // Check if solver accepted this feature. This should be removed once the CIP - // making this mandatory has been approved. - if driver.requested_timeout_on_problems { - tracing::debug!(solver = ?driver.name, "disabling solver temporarily"); - self_.0.banned_solvers.insert(solver, now); - } - } + let non_settling_solver_names: Vec<&str> = non_settling_solvers + .iter() + .filter_map(|solver| self_.0.drivers_by_address.get(solver)) + .map(|driver| { + Metrics::get() + .non_settling_solver + .with_label_values(&[&driver.name]); + // Check if solver accepted this feature. This should be removed once the + // CIP making this mandatory has been approved. + if driver.requested_timeout_on_problems { + tracing::debug!(solver = ?driver.name, "disabling solver temporarily"); + self_ + .0 + .banned_solvers + .insert(driver.submission_address, now); + } + driver.name.as_ref() + }) + .collect(); tracing::debug!(solvers = ?non_settling_solver_names, "found non-settling solvers"); } From 2985000b21ca502f4996843e05b0dad1dd957b17 Mon Sep 17 00:00:00 2001 From: ilya Date: Tue, 18 Feb 2025 11:07:55 +0000 Subject: [PATCH 71/82] Fix after merge --- .../src/domain/competition/participation_guard/db.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index 28a634d44d..cf548569cb 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -151,9 +151,9 @@ impl SolverValidator { found_at: Instant, banned_until: DateTime, ) { - let non_settling_solver_names: Vec<&str> = non_settling_solvers + let non_settling_solver_names: Vec<&str> = solvers .iter() - .filter_map(|solver| self_.0.drivers_by_address.get(solver)) + .filter_map(|solver| self.0.drivers_by_address.get(solver)) .map(|driver| { Metrics::get() .banned_solver @@ -162,9 +162,8 @@ impl SolverValidator { // CIP making this mandatory has been approved. if driver.requested_timeout_on_problems { tracing::debug!(solver = ?driver.name, "disabling solver temporarily"); - infra::notify_banned_solver(driver.clone(), banned_until); - self_ - .0 + infra::notify_banned_solver(driver.clone(), ban_reason, banned_until); + self.0 .banned_solvers .insert(driver.submission_address, found_at); } From a4a43e5bc595a8aff8c06267e4d220b47187bc49 Mon Sep 17 00:00:00 2001 From: ilya Date: Tue, 18 Feb 2025 18:35:45 +0000 Subject: [PATCH 72/82] Fire notify and forget --- .../driver/src/infra/api/routes/notify/mod.rs | 6 +- crates/driver/src/infra/notify/mod.rs | 16 ++--- crates/driver/src/infra/solver/mod.rs | 64 +++---------------- 3 files changed, 19 insertions(+), 67 deletions(-) diff --git a/crates/driver/src/infra/api/routes/notify/mod.rs b/crates/driver/src/infra/api/routes/notify/mod.rs index d8922b7631..3e952a6c6c 100644 --- a/crates/driver/src/infra/api/routes/notify/mod.rs +++ b/crates/driver/src/infra/api/routes/notify/mod.rs @@ -12,10 +12,6 @@ async fn route( ) -> Result)> { let solver = &state.solver().name().0; tracing::debug!(?req, ?solver, "received a notification"); - - if let Err(err) = state.solver().notify(None, None, req.0.into()).await { - tracing::debug!(?err, "failed to notify solver"); - } - + state.solver().notify(None, None, req.0.into()); Ok(hyper::StatusCode::OK) } diff --git a/crates/driver/src/infra/notify/mod.rs b/crates/driver/src/infra/notify/mod.rs index 4baf6f0266..0e6f060513 100644 --- a/crates/driver/src/infra/notify/mod.rs +++ b/crates/driver/src/infra/notify/mod.rs @@ -19,11 +19,11 @@ use { }; pub fn solver_timeout(solver: &Solver, auction_id: Option) { - solver.notify_and_forget(auction_id, None, notification::Kind::Timeout); + solver.notify(auction_id, None, notification::Kind::Timeout); } pub fn empty_solution(solver: &Solver, auction_id: Option, solution: solution::Id) { - solver.notify_and_forget( + solver.notify( auction_id, Some(solution), notification::Kind::EmptySolution, @@ -52,7 +52,7 @@ pub fn scoring_failed( } }; - solver.notify_and_forget(auction_id, Some(solution_id.clone()), notification); + solver.notify(auction_id, Some(solution_id.clone()), notification); } pub fn encoding_failed( @@ -83,7 +83,7 @@ pub fn encoding_failed( solution::Error::Encoding(_) => return, }; - solver.notify_and_forget(auction_id, Some(solution_id.clone()), notification); + solver.notify(auction_id, Some(solution_id.clone()), notification); } pub fn simulation_failed( @@ -101,7 +101,7 @@ pub fn simulation_failed( ), simulator::Error::Other(error) => notification::Kind::DriverError(error.to_string()), }; - solver.notify_and_forget(auction_id, Some(solution_id.clone()), kind); + solver.notify(auction_id, Some(solution_id.clone()), kind); } pub fn executed( @@ -118,7 +118,7 @@ pub fn executed( Err(Error::Other(_) | Error::Disabled) => notification::Settlement::Fail, }; - solver.notify_and_forget( + solver.notify( Some(auction_id), Some(solution_id.clone()), notification::Kind::Settled(kind), @@ -130,7 +130,7 @@ pub fn duplicated_solution_id( auction_id: Option, solution_id: &solution::Id, ) { - solver.notify_and_forget( + solver.notify( auction_id, Some(solution_id.clone()), notification::Kind::DuplicatedSolutionId, @@ -138,5 +138,5 @@ pub fn duplicated_solution_id( } pub fn postprocessing_timed_out(solver: &Solver, auction_id: Option) { - solver.notify_and_forget(auction_id, None, notification::Kind::PostprocessingTimedOut); + solver.notify(auction_id, None, notification::Kind::PostprocessingTimedOut); } diff --git a/crates/driver/src/infra/solver/mod.rs b/crates/driver/src/infra/solver/mod.rs index bda080e0b1..3e63af9e86 100644 --- a/crates/driver/src/infra/solver/mod.rs +++ b/crates/driver/src/infra/solver/mod.rs @@ -260,71 +260,27 @@ impl Solver { } /// Make a fire and forget POST request to notify the solver about an event. - pub fn notify_and_forget( + pub fn notify( &self, auction_id: Option, solution_id: Option, kind: notify::Kind, ) { - let base_url = self.config.endpoint.clone(); - let client = self.client.clone(); - let response_limit = self.config.response_size_limit_max_bytes; - let future = async move { - if let Err(error) = Self::notify_inner( - base_url, - client, - response_limit, - auction_id, - solution_id, - kind, - ) - .await - { - tracing::warn!(?error, "failed to notify solver"); - } - }; - - tokio::task::spawn(future.in_current_span()); - } - - pub async fn notify( - &self, - auction_id: Option, - solution_id: Option, - kind: notify::Kind, - ) -> Result<(), crate::util::http::Error> { - let base_url = self.config.endpoint.clone(); - let client = self.client.clone(); - - Self::notify_inner( - base_url, - client, - self.config.response_size_limit_max_bytes, - auction_id, - solution_id, - kind, - ) - .await - } - - async fn notify_inner( - base_url: url::Url, - client: reqwest::Client, - response_limit: usize, - auction_id: Option, - solution_id: Option, - kind: notify::Kind, - ) -> Result<(), crate::util::http::Error> { let body = serde_json::to_string(&dto::Notification::new(auction_id, solution_id, kind)).unwrap(); - let url = shared::url::join(&base_url, "notify"); + let url = shared::url::join(&self.config.endpoint, "notify"); super::observe::solver_request(&url, &body); - let mut req = client.post(url).body(body); + let mut req = self.client.post(url).body(body); if let Some(id) = observe::request_id::from_current_span() { req = req.header("X-REQUEST-ID", id); } - - util::http::send(response_limit, req).await.map(|_| ()) + let response_size = self.config.response_size_limit_max_bytes; + let future = async move { + if let Err(error) = util::http::send(response_size, req).await { + tracing::warn!(?error, "failed to notify solver"); + } + }; + tokio::task::spawn(future.in_current_span()); } } From 4a814541b5bb3d0de9142a658878d9bc274ba073 Mon Sep 17 00:00:00 2001 From: ilya Date: Tue, 18 Feb 2025 19:47:53 +0000 Subject: [PATCH 73/82] Copy enum --- .../src/domain/competition/participation_guard/db.rs | 6 +++--- crates/autopilot/src/infra/solvers/dto/notify.rs | 2 +- crates/autopilot/src/infra/solvers/mod.rs | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index cf548569cb..46bf974e7e 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -81,13 +81,13 @@ impl SolverValidator { self_.post_process( &non_settling_solvers, - &dto::notify::BanReason::UnsettledConsecutiveAuctions, + dto::notify::BanReason::UnsettledConsecutiveAuctions, found_at, banned_until, ); self_.post_process( &low_settling_solvers, - &dto::notify::BanReason::HighSettleFailureRate, + dto::notify::BanReason::HighSettleFailureRate, found_at, banned_until, ); @@ -147,7 +147,7 @@ impl SolverValidator { fn post_process( &self, solvers: &HashSet, - ban_reason: &dto::notify::BanReason, + ban_reason: dto::notify::BanReason, found_at: Instant, banned_until: DateTime, ) { diff --git a/crates/autopilot/src/infra/solvers/dto/notify.rs b/crates/autopilot/src/infra/solvers/dto/notify.rs index 0b96526595..7a63a0bc4a 100644 --- a/crates/autopilot/src/infra/solvers/dto/notify.rs +++ b/crates/autopilot/src/infra/solvers/dto/notify.rs @@ -15,7 +15,7 @@ pub enum Request { } #[serde_as] -#[derive(Clone, Debug, Serialize)] +#[derive(Clone, Copy, Debug, Serialize)] #[serde(rename_all = "camelCase")] pub enum BanReason { /// The driver won multiple consecutive auctions but never settled them. diff --git a/crates/autopilot/src/infra/solvers/mod.rs b/crates/autopilot/src/infra/solvers/mod.rs index 73bec84274..4dcdbbfc97 100644 --- a/crates/autopilot/src/infra/solvers/mod.rs +++ b/crates/autopilot/src/infra/solvers/mod.rs @@ -181,11 +181,11 @@ pub async fn response_body_with_size_limit( /// Notifies the non-settling driver in a fire-and-forget manner. pub fn notify_banned_solver( non_settling_driver: Arc, - reason: ¬ify::BanReason, + reason: notify::BanReason, banned_until: DateTime, ) { let request = notify::Request::Banned { - reason: reason.clone(), + reason, until: banned_until, }; tokio::spawn(async move { From b4c31f31571bfefe257ec7c48e32b2ea81686ba4 Mon Sep 17 00:00:00 2001 From: ilya Date: Wed, 19 Feb 2025 13:07:57 +0000 Subject: [PATCH 74/82] Redundant serde_as --- crates/driver/src/infra/solver/dto/notification.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/driver/src/infra/solver/dto/notification.rs b/crates/driver/src/infra/solver/dto/notification.rs index fc2f91e263..e13470e158 100644 --- a/crates/driver/src/infra/solver/dto/notification.rs +++ b/crates/driver/src/infra/solver/dto/notification.rs @@ -159,7 +159,6 @@ pub enum Kind { }, } -#[serde_as] #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase", tag = "reason")] pub enum BanReason { From ab9889014de67722f4bd242a107e57aa5bd6fb10 Mon Sep 17 00:00:00 2001 From: ilya Date: Tue, 18 Feb 2025 19:56:23 +0000 Subject: [PATCH 75/82] Log the found block --- .../src/domain/competition/participation_guard/db.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index 46bf974e7e..c23d45543e 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -83,12 +83,14 @@ impl SolverValidator { &non_settling_solvers, dto::notify::BanReason::UnsettledConsecutiveAuctions, found_at, + current_block, banned_until, ); self_.post_process( &low_settling_solvers, dto::notify::BanReason::HighSettleFailureRate, found_at, + current_block, banned_until, ); } @@ -148,7 +150,8 @@ impl SolverValidator { &self, solvers: &HashSet, ban_reason: dto::notify::BanReason, - found_at: Instant, + found_at_timestamp: Instant, + found_at_block: u64, banned_until: DateTime, ) { let non_settling_solver_names: Vec<&str> = solvers @@ -165,7 +168,7 @@ impl SolverValidator { infra::notify_banned_solver(driver.clone(), ban_reason, banned_until); self.0 .banned_solvers - .insert(driver.submission_address, found_at); + .insert(driver.submission_address, found_at_timestamp); } driver.name.as_ref() }) @@ -177,7 +180,7 @@ impl SolverValidator { "found high-failure-settlement solvers" } }; - tracing::debug!(solvers = ?non_settling_solver_names, log_message); + tracing::debug!(solvers = ?non_settling_solver_names, ?found_at_block, log_message); } } From d362f9b1690fcab82cdff4ca41fd1f5fc7119b25 Mon Sep 17 00:00:00 2001 From: ilya Date: Mon, 24 Feb 2025 12:21:33 +0000 Subject: [PATCH 76/82] Redundant ordering --- crates/database/src/solver_competition.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/database/src/solver_competition.rs b/crates/database/src/solver_competition.rs index 42cfe3b76e..e532645e0f 100644 --- a/crates/database/src/solver_competition.rs +++ b/crates/database/src/solver_competition.rs @@ -174,7 +174,6 @@ WITH SELECT DISTINCT ca.id AS auction_id FROM competition_auctions ca WHERE ca.deadline <= $1 - ORDER BY ca.id DESC LIMIT $2 ) latest_auctions JOIN proposed_solutions ps ON ps.auction_id = latest_auctions.auction_id From f66d2570bb35fcf4930d3ccceaec040874063386 Mon Sep 17 00:00:00 2001 From: ilya Date: Wed, 26 Feb 2025 10:41:36 +0000 Subject: [PATCH 77/82] Minor --- .../autopilot/src/domain/competition/participation_guard/db.rs | 2 +- crates/autopilot/src/infra/mod.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index fe8f6b1f63..eb4981f185 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -1,6 +1,6 @@ use { crate::{ - domain::{eth, Metrics}, + domain::{Metrics, eth}, infra, }, chrono::Utc, diff --git a/crates/autopilot/src/infra/mod.rs b/crates/autopilot/src/infra/mod.rs index 28046fa333..5d00d20960 100644 --- a/crates/autopilot/src/infra/mod.rs +++ b/crates/autopilot/src/infra/mod.rs @@ -7,5 +7,5 @@ pub use { blockchain::Ethereum, order_validation::banned, persistence::Persistence, - solvers::{notify_non_settling_solver, Driver}, + solvers::{Driver, notify_non_settling_solver}, }; From eba672c5e1fb25109e1afda17e88b5736abd1f2f Mon Sep 17 00:00:00 2001 From: ilya Date: Wed, 26 Feb 2025 10:44:31 +0000 Subject: [PATCH 78/82] Minor --- .../autopilot/src/domain/competition/participation_guard/db.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/autopilot/src/domain/competition/participation_guard/db.rs b/crates/autopilot/src/domain/competition/participation_guard/db.rs index 7ef9fddb67..1c5be28de3 100644 --- a/crates/autopilot/src/domain/competition/participation_guard/db.rs +++ b/crates/autopilot/src/domain/competition/participation_guard/db.rs @@ -1,6 +1,6 @@ use { crate::{ - domain::{eth, Metrics}, + domain::{Metrics, eth}, infra, }, ethrpc::block_stream::CurrentBlockWatcher, From 2677d2661160fa544f79407bbf0250ea80be87e6 Mon Sep 17 00:00:00 2001 From: ilya Date: Wed, 26 Feb 2025 10:47:32 +0000 Subject: [PATCH 79/82] Formatting --- crates/autopilot/src/infra/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/autopilot/src/infra/mod.rs b/crates/autopilot/src/infra/mod.rs index 77a76f95b4..52809890c3 100644 --- a/crates/autopilot/src/infra/mod.rs +++ b/crates/autopilot/src/infra/mod.rs @@ -7,5 +7,5 @@ pub use { blockchain::Ethereum, order_validation::banned, persistence::Persistence, - solvers::{notify_banned_solver, Driver}, + solvers::{Driver, notify_banned_solver}, }; From 88d9c7140e8d0cb595b916db03cb5b5aac428919 Mon Sep 17 00:00:00 2001 From: ilya Date: Wed, 26 Feb 2025 11:40:57 +0000 Subject: [PATCH 80/82] Solvers-dto --- crates/solvers-dto/src/notification.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/crates/solvers-dto/src/notification.rs b/crates/solvers-dto/src/notification.rs index e99f73028e..ff308bfb15 100644 --- a/crates/solvers-dto/src/notification.rs +++ b/crates/solvers-dto/src/notification.rs @@ -1,5 +1,6 @@ use { super::serialize, + chrono::{DateTime, Utc}, number::serialization::HexOrDecimalU256, serde::Deserialize, serde_with::{DisplayFromStr, serde_as}, @@ -64,6 +65,10 @@ pub enum Kind { Cancelled, Fail, PostprocessingTimedOut, + Banned { + reason: BanReason, + until: DateTime, + }, } type BlockNo = u64; @@ -80,3 +85,9 @@ pub struct Tx { pub value: U256, pub access_list: AccessList, } + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", tag = "reason")] +pub enum BanReason { + UnsettledConsecutiveAuctions, +} From 06ee6b0e673361a9c5703ce8f5f23a40a7a65205 Mon Sep 17 00:00:00 2001 From: ilya Date: Wed, 26 Feb 2025 11:43:46 +0000 Subject: [PATCH 81/82] Solvers-dto --- crates/solvers-dto/src/notification.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/solvers-dto/src/notification.rs b/crates/solvers-dto/src/notification.rs index ff308bfb15..d776dae2c2 100644 --- a/crates/solvers-dto/src/notification.rs +++ b/crates/solvers-dto/src/notification.rs @@ -90,4 +90,5 @@ pub struct Tx { #[serde(rename_all = "camelCase", tag = "reason")] pub enum BanReason { UnsettledConsecutiveAuctions, + LowSettlingRate, } From 92047e77059cb4b7282bdb9466441b4ce6872310 Mon Sep 17 00:00:00 2001 From: ilya Date: Wed, 26 Feb 2025 11:53:33 +0000 Subject: [PATCH 82/82] Fix after merge --- crates/driver/src/infra/solver/dto/notification.rs | 6 ++++-- crates/solvers-dto/src/notification.rs | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/driver/src/infra/solver/dto/notification.rs b/crates/driver/src/infra/solver/dto/notification.rs index 36ac086a0a..c2faafc12a 100644 --- a/crates/driver/src/infra/solver/dto/notification.rs +++ b/crates/driver/src/infra/solver/dto/notification.rs @@ -67,7 +67,9 @@ impl Notification { notify::BanReason::UnsettledConsecutiveAuctions => { BanReason::UnsettledConsecutiveAuctions } - notify::BanReason::HighSettleFailureRate => BanReason::LowSettlingRate, + notify::BanReason::HighSettleFailureRate => { + BanReason::HighSettleFailureRate + } }, until, }, @@ -164,7 +166,7 @@ pub enum Kind { #[serde(rename_all = "camelCase", tag = "reason")] pub enum BanReason { UnsettledConsecutiveAuctions, - LowSettlingRate, + HighSettleFailureRate, } type BlockNo = u64; diff --git a/crates/solvers-dto/src/notification.rs b/crates/solvers-dto/src/notification.rs index d776dae2c2..c5b70d299f 100644 --- a/crates/solvers-dto/src/notification.rs +++ b/crates/solvers-dto/src/notification.rs @@ -90,5 +90,5 @@ pub struct Tx { #[serde(rename_all = "camelCase", tag = "reason")] pub enum BanReason { UnsettledConsecutiveAuctions, - LowSettlingRate, + HighSettleFailureRate, }