From e0f006a03f73d626d9b39d905bf3edc63281147e Mon Sep 17 00:00:00 2001 From: sunce86 Date: Mon, 15 Jan 2024 12:02:04 +0100 Subject: [PATCH 01/23] Remove auction_transaction --- crates/autopilot/src/database.rs | 1 - .../src/database/auction_transaction.rs | 50 ---- .../database/on_settlement_event_updater.rs | 55 ++--- .../src/on_settlement_event_updater.rs | 216 +++++------------- crates/database/src/auction_transaction.rs | 205 ----------------- crates/database/src/lib.rs | 1 - crates/database/src/settlements.rs | 49 +++- .../sql/V057__drop_auction_transaction.sql | 9 + 8 files changed, 136 insertions(+), 450 deletions(-) delete mode 100644 crates/autopilot/src/database/auction_transaction.rs delete mode 100644 crates/database/src/auction_transaction.rs create mode 100644 database/sql/V057__drop_auction_transaction.sql diff --git a/crates/autopilot/src/database.rs b/crates/autopilot/src/database.rs index 319bd6aa11..a08feca42c 100644 --- a/crates/autopilot/src/database.rs +++ b/crates/autopilot/src/database.rs @@ -6,7 +6,6 @@ use { mod auction; pub mod auction_prices; -pub mod auction_transaction; pub mod competition; pub mod ethflow_events; mod events; diff --git a/crates/autopilot/src/database/auction_transaction.rs b/crates/autopilot/src/database/auction_transaction.rs deleted file mode 100644 index 96cccc3edf..0000000000 --- a/crates/autopilot/src/database/auction_transaction.rs +++ /dev/null @@ -1,50 +0,0 @@ -use { - anyhow::Context, - database::{auction_transaction::SettlementEvent, byte_array::ByteArray}, - primitive_types::H160, -}; - -impl super::Postgres { - pub async fn update_settlement_tx_info( - &self, - block_number: i64, - log_index: i64, - tx_from: H160, - tx_nonce: i64, - ) -> anyhow::Result<()> { - let _timer = super::Metrics::get() - .database_queries - .with_label_values(&["update_settlement_tx_info"]) - .start_timer(); - - let mut ex = self.pool.acquire().await.context("acquire")?; - database::auction_transaction::insert_settlement_tx_info( - &mut ex, - block_number, - log_index, - &ByteArray(tx_from.0), - tx_nonce, - ) - .await - .context("insert_settlement_tx_info")?; - - Ok(()) - } - - pub async fn get_settlement_event_without_tx_info( - &self, - max_block_number: i64, - ) -> Result, sqlx::Error> { - let _timer = super::Metrics::get() - .database_queries - .with_label_values(&["get_settlement_event_without_tx_info"]) - .start_timer(); - - let mut ex = self.pool.acquire().await?; - database::auction_transaction::get_settlement_event_without_tx_info( - &mut ex, - max_block_number, - ) - .await - } -} diff --git a/crates/autopilot/src/database/on_settlement_event_updater.rs b/crates/autopilot/src/database/on_settlement_event_updater.rs index ff1760c179..d4d7ee42b5 100644 --- a/crates/autopilot/src/database/on_settlement_event_updater.rs +++ b/crates/autopilot/src/database/on_settlement_event_updater.rs @@ -1,7 +1,8 @@ use { + crate::on_settlement_event_updater::AuctionKind, anyhow::{Context, Result}, database::{byte_array::ByteArray, settlement_observations::Observation}, - ethcontract::{H160, U256}, + ethcontract::U256, model::order::OrderUid, number::conversions::u256_to_big_decimal, sqlx::PgConnection, @@ -21,12 +22,11 @@ pub struct AuctionData { pub order_executions: Vec<(OrderUid, ExecutedFee)>, } -#[derive(Debug, Default, Clone)] +#[derive(Debug, Clone)] pub struct SettlementUpdate { pub block_number: i64, pub log_index: i64, - pub tx_from: H160, - pub tx_nonce: i64, + pub auction_kind: AuctionKind, pub auction_data: Option, } @@ -40,32 +40,25 @@ impl super::Postgres { .with_label_values(&["update_settlement_details"]) .start_timer(); + let (auction_id, auction_kind) = match settlement_update.auction_kind { + AuctionKind::Valid { auction_id } => { + (Some(auction_id), database::settlements::AuctionKind::Valid) + } + AuctionKind::Invalid => (None, database::settlements::AuctionKind::Invalid), + }; + // update settlements - database::auction_transaction::insert_settlement_tx_info( + database::settlements::update_settlement_auction( ex, settlement_update.block_number, settlement_update.log_index, - &ByteArray(settlement_update.tx_from.0), - settlement_update.tx_nonce, + auction_id, + auction_kind, ) .await .context("insert_settlement_tx_info")?; if let Some(auction_data) = settlement_update.auction_data { - // Link the `auction_id` to the settlement tx. This is needed for - // colocated solutions and is a no-op for centralized - // solutions. - let insert_succesful = database::auction_transaction::try_insert_auction_transaction( - ex, - auction_data.auction_id, - &ByteArray(settlement_update.tx_from.0), - settlement_update.tx_nonce, - ) - .await - .context("failed to insert auction_transaction")?; - - // in case of deep reindexing we might already have the observation, so just - // overwrite it database::settlement_observations::upsert( ex, Observation { @@ -80,17 +73,15 @@ impl super::Postgres { .await .context("insert_settlement_observations")?; - if insert_succesful { - for (order, executed_fee) in auction_data.order_executions { - database::order_execution::save( - ex, - &ByteArray(order.0), - auction_data.auction_id, - &u256_to_big_decimal(&executed_fee), - ) - .await - .context("save_order_executions")?; - } + for (order, executed_fee) in auction_data.order_executions { + database::order_execution::save( + ex, + &ByteArray(order.0), + auction_data.auction_id, + &u256_to_big_decimal(&executed_fee), + ) + .await + .context("save_order_executions")?; } } Ok(()) diff --git a/crates/autopilot/src/on_settlement_event_updater.rs b/crates/autopilot/src/on_settlement_event_updater.rs index 120cec41dc..fce5d13584 100644 --- a/crates/autopilot/src/on_settlement_event_updater.rs +++ b/crates/autopilot/src/on_settlement_event_updater.rs @@ -38,7 +38,7 @@ use { }, decoded_settlement::{DecodedSettlement, DecodingError}, }, - anyhow::{anyhow, Context, Result}, + anyhow::{Context, Result}, contracts::GPv2Settlement, ethrpc::{ current_block::{into_stream, CurrentBlockStream}, @@ -47,11 +47,37 @@ use { futures::StreamExt, model::DomainSeparator, primitive_types::{H160, H256}, - shared::{event_handling::MAX_REORG_BLOCK_COUNT, external_prices::ExternalPrices}, + shared::external_prices::ExternalPrices, sqlx::PgConnection, web3::types::{Transaction, TransactionId}, }; +#[derive(Debug, Copy, Clone)] +pub enum AuctionKind { + /// This auction is regular and all the auction dependent data should be + /// updated. + Valid { auction_id: i64 }, + /// Some possible reasons to have invalid auction are: + /// - This auction was created by another environment (e.g. + /// production/staging) + /// - Failed to decode settlement calldata + /// - Failed to recover auction id from calldata + /// - Settlement transaction was submitted by solver other than the winner + /// + /// In this case, settlement event should be market as invalid and no + /// auction dependent data is updated. + Invalid, +} + +impl AuctionKind { + pub fn auction_id(&self) -> Option { + match self { + AuctionKind::Valid { auction_id } => Some(*auction_id), + AuctionKind::Invalid => None, + } + } +} + pub struct OnSettlementEventUpdater { pub web3: Web3, pub contract: GPv2Settlement, @@ -64,7 +90,7 @@ impl OnSettlementEventUpdater { let mut current_block = *block_stream.borrow(); let mut block_stream = into_stream(block_stream); loop { - match self.update(current_block.number).await { + match self.update().await { Ok(true) => { tracing::debug!( block = current_block.number, @@ -90,25 +116,16 @@ impl OnSettlementEventUpdater { /// Update database for settlement events that have not been processed yet. /// /// Returns whether an update was performed. - async fn update(&self, current_block: u64) -> Result { - let reorg_safe_block: i64 = current_block - .checked_sub(MAX_REORG_BLOCK_COUNT) - .context("no reorg safe block")? - .try_into() - .context("convert block")?; - + async fn update(&self) -> Result { let mut ex = self .db .pool .begin() .await .context("acquire DB connection")?; - let event = match database::auction_transaction::get_settlement_event_without_tx_info( - &mut ex, - reorg_safe_block, - ) - .await - .context("get_settlement_event_without_tx_info")? + let event = match database::settlements::get_settlement_without_auction(&mut ex) + .await + .context("get_settlement_without_auction")? { Some(event) => event, None => return Ok(false), @@ -124,25 +141,14 @@ impl OnSettlementEventUpdater { .await .with_context(|| format!("get tx {hash:?}"))? .with_context(|| format!("no tx {hash:?}"))?; - let tx_from = transaction - .from - .with_context(|| format!("no from {hash:?}"))?; - let tx_nonce: i64 = transaction - .nonce - .try_into() - .map_err(|err| anyhow!("{}", err)) - .with_context(|| format!("convert nonce {hash:?}"))?; let domain_separator = DomainSeparator(self.contract.domain_separator().call().await?.0); - let auction_id = - Self::recover_auction_id_from_calldata(&mut ex, &transaction, &domain_separator) - .await?; + let auction_kind = Self::get_auction_kind(&mut ex, &transaction, &domain_separator).await?; let mut update = SettlementUpdate { block_number: event.block_number, log_index: event.log_index, - tx_from, - tx_nonce, + auction_kind, auction_data: None, }; @@ -152,7 +158,7 @@ impl OnSettlementEventUpdater { // // If auction_id exists, we expect all other relevant information to exist as // well. - if let Some(auction_id) = auction_id { + if let AuctionKind::Valid { auction_id } = auction_kind { let receipt = self .web3 .eth() @@ -248,12 +254,12 @@ impl OnSettlementEventUpdater { /// `auction_id` to the settlement calldata. This function tries to /// recover that `auction_id`. This function only returns an error if /// retrying the operation makes sense. If all went well and there - /// simply is no sensible `auction_id` `Ok(None)` will be returned. - async fn recover_auction_id_from_calldata( + /// simply is no sensible `auction_id` `AuctionKind::Invalid` will be returned. + async fn get_auction_kind( ex: &mut PgConnection, tx: &Transaction, domain_separator: &DomainSeparator, - ) -> Result> { + ) -> Result { let tx_from = tx.from.context("tx is missing sender")?; let metadata = match DecodedSettlement::new(&tx.input.0, domain_separator) { Ok(settlement) => settlement.metadata, @@ -263,149 +269,39 @@ impl OnSettlementEventUpdater { ?err, "could not decode settlement tx, unclear which auction it belongs to" ); - return Ok(None); + return Ok(AuctionKind::Invalid); } }; let auction_id = match metadata { Some(bytes) => i64::from_be_bytes(bytes.0), None => { tracing::warn!(?tx, "could not recover the auction_id from the calldata"); - return Ok(None); + return Ok(AuctionKind::Invalid); } }; let score = database::settlement_scores::fetch(ex, auction_id).await?; - let data_already_recorded = - database::auction_transaction::data_exists(ex, auction_id).await?; - match (score, data_already_recorded) { - (None, _) => { + match score { + None => { tracing::debug!( auction_id, "calldata claims to settle auction that has no competition" ); - Ok(None) + Ok(AuctionKind::Invalid) } - (Some(score), _) if score.winner.0 != tx_from.0 => { - tracing::warn!( - auction_id, - ?tx_from, - winner = ?score.winner, - "solution submitted by solver other than the winner" - ); - Ok(None) - } - (Some(_), true) => { - tracing::warn!( - auction_id, - "settlement data already recorded for this auction" - ); - Ok(None) + Some(score) => { + if score.winner.0 != tx_from.0 { + tracing::warn!( + auction_id, + ?tx_from, + winner = ?score.winner, + "solution submitted by solver other than the winner" + ); + Ok(AuctionKind::Invalid) + } else { + Ok(AuctionKind::Valid { auction_id }) + } } - (Some(_), false) => Ok(Some(auction_id)), } } } - -#[cfg(test)] -mod tests { - use { - super::*, - database::{auction_prices::AuctionPrice, settlement_observations::Observation}, - sqlx::Executor, - }; - - #[tokio::test] - #[ignore] - async fn manual_node_test() { - // TODO update test - observe::tracing::initialize_reentrant("autopilot=trace"); - let db = Postgres::with_defaults().await.unwrap(); - database::clear_DANGER(&db.pool).await.unwrap(); - let transport = shared::ethrpc::create_env_test_transport(); - let web3 = Web3::new(transport); - - let contract = contracts::GPv2Settlement::deployed(&web3).await.unwrap(); - let native_token = contracts::WETH9::deployed(&web3).await.unwrap().address(); - let updater = OnSettlementEventUpdater { - web3, - db, - native_token, - contract, - }; - - assert!(!updater.update(15875900).await.unwrap()); - - let query = r" -INSERT INTO settlements (block_number, log_index, solver, tx_hash, tx_from, tx_nonce) -VALUES (15875801, 405, '\x', '\x0e9d0f4ea243ac0f02e1d3ecab3fea78108d83bfca632b30e9bc4acb22289c5a', NULL, NULL) - ;"; - updater.db.pool.execute(query).await.unwrap(); - - let query = r" -INSERT INTO auction_transaction (auction_id, tx_from, tx_nonce) -VALUES (0, '\xa21740833858985e4d801533a808786d3647fb83', 4701) - ;"; - updater.db.pool.execute(query).await.unwrap(); - - let query = r" -INSERT INTO auction_prices (auction_id, token, price) -VALUES (0, '\x056fd409e1d7a124bd7017459dfea2f387b6d5cd', 6347795727933475088343330979840), - (0, '\x6b175474e89094c44da98b954eedeac495271d0f', 634671683530053), - (0, '\xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48', 634553336916241343152390144) - ;"; - - updater.db.pool.execute(query).await.unwrap(); - - assert!(updater.update(15875900).await.unwrap()); - - let query = r" -SELECT tx_from, tx_nonce -FROM settlements -WHERE block_number = 15875801 AND log_index = 405 - ;"; - let (tx_from, tx_nonce): (Vec, i64) = sqlx::query_as(query) - .fetch_one(&updater.db.pool) - .await - .unwrap(); - assert_eq!( - tx_from, - hex_literal::hex!("a21740833858985e4d801533a808786d3647fb83") - ); - assert_eq!(tx_nonce, 4701); - - let query = r" -SELECT auction_id, tx_from, tx_nonce -FROM auction_transaction - ;"; - let (auction_id, tx_from, tx_nonce): (i64, Vec, i64) = sqlx::query_as(query) - .fetch_one(&updater.db.pool) - .await - .unwrap(); - assert_eq!(auction_id, 0); - assert_eq!( - tx_from, - hex_literal::hex!("a21740833858985e4d801533a808786d3647fb83") - ); - assert_eq!(tx_nonce, 4701); - - // assert that the prices are updated - let query = r#"SELECT * FROM auction_prices;"#; - let prices: Vec = sqlx::query_as(query) - .fetch_all(&updater.db.pool) - .await - .unwrap(); - assert_eq!(prices.len(), 2); - - // assert that the observations are updated - let query = r#"SELECT * FROM settlement_observations;"#; - let observation: Observation = sqlx::query_as(query) - .fetch_one(&updater.db.pool) - .await - .unwrap(); - assert_eq!(observation.gas_used, 179155.into()); - assert_eq!(observation.effective_gas_price, 19789368758u64.into()); - assert_eq!(observation.surplus, 5150444803867862u64.into()); - - assert!(!updater.update(15875900).await.unwrap()); - } -} diff --git a/crates/database/src/auction_transaction.rs b/crates/database/src/auction_transaction.rs deleted file mode 100644 index 5555cf66dd..0000000000 --- a/crates/database/src/auction_transaction.rs +++ /dev/null @@ -1,205 +0,0 @@ -use { - crate::{auction::AuctionId, Address, TransactionHash}, - sqlx::{postgres::PgQueryResult, PgConnection}, -}; - -/// Inserts a row **iff** we don't have an entry for the given `auction_id` yet. -/// This is useful to associate a settlement transaction coming from a colocated -/// driver with an auction. -/// In that case anybody could claim to settle the given auction but we only -/// ever want to store the first claim. -pub async fn try_insert_auction_transaction( - ex: &mut PgConnection, - auction_id: AuctionId, - tx_from: &Address, - tx_nonce: i64, -) -> Result { - const QUERY: &str = r#" - INSERT INTO auction_transaction (auction_id, tx_from, tx_nonce) - VALUES ($1, $2, $3) - ON CONFLICT (auction_id) DO NOTHING - "#; - - let result = sqlx::query(QUERY) - .bind(auction_id) - .bind(tx_from) - .bind(tx_nonce) - .execute(ex) - .await?; - - Ok(result.rows_affected() == 1) -} - -pub async fn insert_settlement_tx_info( - ex: &mut PgConnection, - block_number: i64, - log_index: i64, - tx_from: &Address, - tx_nonce: i64, -) -> Result { - const QUERY: &str = r#" -UPDATE settlements -SET tx_from = $1, tx_nonce = $2 -WHERE block_number = $3 AND log_index = $4 - ;"#; - sqlx::query(QUERY) - .bind(tx_from) - .bind(tx_nonce) - .bind(block_number) - .bind(log_index) - .execute(ex) - .await -} - -#[derive(Debug, sqlx::FromRow)] -pub struct SettlementEvent { - pub block_number: i64, - pub log_index: i64, - pub tx_hash: TransactionHash, -} - -pub async fn get_settlement_event_without_tx_info( - ex: &mut PgConnection, - max_block_number: i64, -) -> Result, sqlx::Error> { - const QUERY: &str = r#" -SELECT block_number, log_index, tx_hash -FROM settlements -WHERE tx_from IS NULL AND block_number <= $1 -LIMIT 1 - "#; - sqlx::query_as(QUERY) - .bind(max_block_number) - .fetch_optional(ex) - .await -} - -pub async fn data_exists(ex: &mut PgConnection, auction_id: i64) -> Result { - const QUERY: &str = r#"SELECT COUNT(*) FROM auction_transaction WHERE auction_id = $1;"#; - let count: i64 = sqlx::query_scalar(QUERY) - .bind(auction_id) - .fetch_one(ex) - .await?; - Ok(count >= 1) -} - -#[cfg(test)] -mod tests { - use { - super::*, - crate::events::{Event, EventIndex, Settlement}, - sqlx::Connection, - std::ops::DerefMut, - }; - - #[tokio::test] - #[ignore] - async fn insert_settlement_tx_info_() { - let mut db = PgConnection::connect("postgresql://").await.unwrap(); - let mut db = db.begin().await.unwrap(); - crate::clear_DANGER_(&mut db).await.unwrap(); - - let index = EventIndex::default(); - let event = Event::Settlement(Settlement { - solver: Default::default(), - transaction_hash: Default::default(), - }); - crate::events::append(&mut db, &[(index, event)]) - .await - .unwrap(); - - let auction_id: Option = sqlx::query_scalar("SELECT tx_nonce FROM settlements") - .fetch_one(db.deref_mut()) - .await - .unwrap(); - assert_eq!(auction_id, None); - - insert_settlement_tx_info(&mut db, 0, 0, &Default::default(), 1) - .await - .unwrap(); - - let auction_id: Option = sqlx::query_scalar("SELECT tx_nonce FROM settlements") - .fetch_one(db.deref_mut()) - .await - .unwrap(); - assert_eq!(auction_id, Some(1)); - } - - #[tokio::test] - #[ignore] - async fn get_settlement_event_without_tx_info_() { - let mut db = PgConnection::connect("postgresql://").await.unwrap(); - let mut db = db.begin().await.unwrap(); - crate::clear_DANGER_(&mut db).await.unwrap(); - - let event = get_settlement_event_without_tx_info(&mut db, 10) - .await - .unwrap(); - assert!(event.is_none()); - - // event at block 0 - let index = EventIndex::default(); - let event = Event::Settlement(Settlement { - solver: Default::default(), - transaction_hash: Default::default(), - }); - crate::events::append(&mut db, &[(index, event)]) - .await - .unwrap(); - - // is found - let event = get_settlement_event_without_tx_info(&mut db, 10) - .await - .unwrap() - .unwrap(); - assert_eq!(event.block_number, 0); - - // gets tx info - insert_settlement_tx_info(&mut db, 0, 0, &Default::default(), 1) - .await - .unwrap(); - - // no longer found - let event = get_settlement_event_without_tx_info(&mut db, 10) - .await - .unwrap(); - assert!(event.is_none()); - - // event at 11 - let index = EventIndex { - block_number: 11, - log_index: 0, - }; - let event = Event::Settlement(Settlement { - solver: Default::default(), - transaction_hash: Default::default(), - }); - crate::events::append(&mut db, &[(index, event)]) - .await - .unwrap(); - - // not found because block number too large - let event = get_settlement_event_without_tx_info(&mut db, 10) - .await - .unwrap(); - assert!(event.is_none()); - } - - #[tokio::test] - #[ignore] - async fn try_insert_auction_transaction_test() { - let mut db = PgConnection::connect("postgresql://").await.unwrap(); - let mut db = db.begin().await.unwrap(); - crate::clear_DANGER_(&mut db).await.unwrap(); - - let inserted = try_insert_auction_transaction(&mut db, 3, &Default::default(), 1) - .await - .unwrap(); - assert!(inserted); - - let inserted = try_insert_auction_transaction(&mut db, 3, &Default::default(), 1) - .await - .unwrap(); - assert!(!inserted); - } -} diff --git a/crates/database/src/lib.rs b/crates/database/src/lib.rs index 77046a684e..0b7475d6b0 100644 --- a/crates/database/src/lib.rs +++ b/crates/database/src/lib.rs @@ -2,7 +2,6 @@ pub mod app_data; pub mod auction; pub mod auction_participants; pub mod auction_prices; -pub mod auction_transaction; pub mod byte_array; pub mod ethflow_orders; pub mod events; diff --git a/crates/database/src/settlements.rs b/crates/database/src/settlements.rs index c8cc07fa50..e97726f0f8 100644 --- a/crates/database/src/settlements.rs +++ b/crates/database/src/settlements.rs @@ -1,6 +1,6 @@ use { crate::{events::EventIndex, TransactionHash}, - sqlx::PgConnection, + sqlx::{postgres::PgQueryResult, PgConnection}, std::ops::Range, }; @@ -40,6 +40,53 @@ WHERE .await } +#[derive(Debug, sqlx::FromRow)] +pub struct SettlementEvent { + pub block_number: i64, + pub log_index: i64, + pub tx_hash: TransactionHash, +} + +#[derive(Debug, Clone, PartialEq, sqlx::Type)] +#[sqlx(type_name = "AuctionKind", rename_all = "lowercase")] +pub enum AuctionKind { + Valid, + Invalid, +} + +pub async fn get_settlement_without_auction( + ex: &mut PgConnection, +) -> Result, sqlx::Error> { + const QUERY: &str = r#" +SELECT block_number, log_index, tx_hash +FROM settlements +WHERE auction_kind IS unprocessed +LIMIT 1 + "#; + sqlx::query_as(QUERY).fetch_optional(ex).await +} + +pub async fn update_settlement_auction( + ex: &mut PgConnection, + block_number: i64, + log_index: i64, + auction_id: Option, + auction_kind: AuctionKind, +) -> Result { + const QUERY: &str = r#" +UPDATE settlements +SET auction_kind = $1 and auction_id = $2 +WHERE block_number = $3 AND log_index = $4 + ;"#; + sqlx::query(QUERY) + .bind(auction_kind) + .bind(auction_id) + .bind(block_number) + .bind(log_index) + .execute(ex) + .await +} + #[cfg(test)] mod tests { use { diff --git a/database/sql/V057__drop_auction_transaction.sql b/database/sql/V057__drop_auction_transaction.sql new file mode 100644 index 0000000000..3d2b66c9a8 --- /dev/null +++ b/database/sql/V057__drop_auction_transaction.sql @@ -0,0 +1,9 @@ +CREATE TYPE AuctionKind AS ENUM ('valid', 'invalid', `unprocessed`); + +ALTER TABLE settlements + DROP COLUMN from, + DROP COLUMN nonce, + ADD COLUMN auction_kind AuctionKind NOT NULL DEFAULT 'unprocessed', + ADD COLUMN auction_id bigint; + +DROP TABLE auction_transaction; \ No newline at end of file From 4522a1be1dade863ca64fb5f765198588a1a42e6 Mon Sep 17 00:00:00 2001 From: sunce86 Date: Mon, 15 Jan 2024 12:15:01 +0100 Subject: [PATCH 02/23] fix issues --- crates/autopilot/src/on_settlement_event_updater.rs | 2 +- database/sql/V057__drop_auction_transaction.sql | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/autopilot/src/on_settlement_event_updater.rs b/crates/autopilot/src/on_settlement_event_updater.rs index fce5d13584..a7d2f14b1f 100644 --- a/crates/autopilot/src/on_settlement_event_updater.rs +++ b/crates/autopilot/src/on_settlement_event_updater.rs @@ -64,7 +64,7 @@ pub enum AuctionKind { /// - Failed to recover auction id from calldata /// - Settlement transaction was submitted by solver other than the winner /// - /// In this case, settlement event should be market as invalid and no + /// In this case, settlement event should be marked as invalid and no /// auction dependent data is updated. Invalid, } diff --git a/database/sql/V057__drop_auction_transaction.sql b/database/sql/V057__drop_auction_transaction.sql index 3d2b66c9a8..45fd6bf89a 100644 --- a/database/sql/V057__drop_auction_transaction.sql +++ b/database/sql/V057__drop_auction_transaction.sql @@ -1,8 +1,8 @@ CREATE TYPE AuctionKind AS ENUM ('valid', 'invalid', `unprocessed`); ALTER TABLE settlements - DROP COLUMN from, - DROP COLUMN nonce, + DROP COLUMN tx_from, + DROP COLUMN tx_nonce, ADD COLUMN auction_kind AuctionKind NOT NULL DEFAULT 'unprocessed', ADD COLUMN auction_id bigint; From eabc1870850b44214b5ace4714bb81d807ad36c6 Mon Sep 17 00:00:00 2001 From: sunce86 Date: Thu, 18 Jan 2024 23:45:34 +0100 Subject: [PATCH 03/23] fix solver competition test --- crates/database/src/solver_competition.rs | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/crates/database/src/solver_competition.rs b/crates/database/src/solver_competition.rs index 4c2ee98fc4..4c24ae28d2 100644 --- a/crates/database/src/solver_competition.rs +++ b/crates/database/src/solver_competition.rs @@ -74,7 +74,6 @@ mod tests { crate::{ byte_array::ByteArray, events::{Event, EventIndex, Settlement}, - Address, }, sqlx::Connection, }; @@ -127,22 +126,16 @@ mod tests { .await .unwrap(); - let tx_from: Address = ByteArray([0x01; 20]); - let tx_nonce: i64 = 2; - crate::auction_transaction::insert_settlement_tx_info( + crate::settlements::update_settlement_auction( &mut db, index.block_number, index.log_index, - &tx_from, - tx_nonce, + Some(id), + crate::settlements::AuctionKind::Valid, ) .await .unwrap(); - crate::auction_transaction::try_insert_auction_transaction(&mut db, id, &tx_from, tx_nonce) - .await - .unwrap(); - // Now succeeds. let value_by_hash = load_by_tx_hash(&mut db, &hash).await.unwrap().unwrap(); assert_eq!(value, value_by_hash.json); From 5171b2ca5460abada893d77a4467a1cbd9aced71 Mon Sep 17 00:00:00 2001 From: sunce86 Date: Thu, 18 Jan 2024 23:49:33 +0100 Subject: [PATCH 04/23] rename file --- ...auction_transaction.sql => V058__drop_auction_transaction.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename database/sql/{V057__drop_auction_transaction.sql => V058__drop_auction_transaction.sql} (100%) diff --git a/database/sql/V057__drop_auction_transaction.sql b/database/sql/V058__drop_auction_transaction.sql similarity index 100% rename from database/sql/V057__drop_auction_transaction.sql rename to database/sql/V058__drop_auction_transaction.sql From f6eabc70dd2205e2b631d79f3013a713ea0386ef Mon Sep 17 00:00:00 2001 From: sunce86 Date: Fri, 19 Jan 2024 00:10:35 +0100 Subject: [PATCH 05/23] fix migr --- database/sql/V058__drop_auction_transaction.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/sql/V058__drop_auction_transaction.sql b/database/sql/V058__drop_auction_transaction.sql index 45fd6bf89a..2ef914584a 100644 --- a/database/sql/V058__drop_auction_transaction.sql +++ b/database/sql/V058__drop_auction_transaction.sql @@ -1,4 +1,4 @@ -CREATE TYPE AuctionKind AS ENUM ('valid', 'invalid', `unprocessed`); +CREATE TYPE AuctionKind AS ENUM ('valid', 'invalid', 'unprocessed'); ALTER TABLE settlements DROP COLUMN tx_from, From a039a1ae39139adb274d2dbc29530c57d799aeef Mon Sep 17 00:00:00 2001 From: sunce86 Date: Fri, 19 Jan 2024 00:23:36 +0100 Subject: [PATCH 06/23] various fixes --- crates/database/src/lib.rs | 1 - crates/database/src/solver_competition.rs | 9 +++------ crates/e2e/tests/e2e/database.rs | 11 +++-------- 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/crates/database/src/lib.rs b/crates/database/src/lib.rs index 0b7475d6b0..016bded60d 100644 --- a/crates/database/src/lib.rs +++ b/crates/database/src/lib.rs @@ -58,7 +58,6 @@ pub const TABLES: &[&str] = &[ "ethflow_orders", "order_execution", "interactions", - "auction_transaction", "ethflow_refunds", "settlement_scores", "settlement_observations", diff --git a/crates/database/src/solver_competition.rs b/crates/database/src/solver_competition.rs index 4c24ae28d2..ce6f80e83f 100644 --- a/crates/database/src/solver_competition.rs +++ b/crates/database/src/solver_competition.rs @@ -31,8 +31,7 @@ pub async fn load_by_id( SELECT sc.json, sc.id, s.tx_hash FROM solver_competitions sc -- outer joins because the data might not have been indexed yet -LEFT OUTER JOIN auction_transaction at ON sc.id = at.auction_id -LEFT OUTER JOIN settlements s ON (at.tx_from, at.tx_nonce) = (s.tx_from, s.tx_nonce) +LEFT OUTER JOIN settlements s ON sc.id = s.auction_id WHERE sc.id = $1 ;"#; sqlx::query_as(QUERY).bind(id).fetch_optional(ex).await @@ -45,8 +44,7 @@ pub async fn load_latest_competition( SELECT sc.json, sc.id, s.tx_hash FROM solver_competitions sc -- outer joins because the data might not have been indexed yet -LEFT OUTER JOIN auction_transaction at ON sc.id = at.auction_id -LEFT OUTER JOIN settlements s ON (at.tx_from, at.tx_nonce) = (s.tx_from, s.tx_nonce) +LEFT OUTER JOIN settlements s ON sc.id = s.auction_id ORDER BY sc.id DESC LIMIT 1 ;"#; @@ -60,8 +58,7 @@ pub async fn load_by_tx_hash( const QUERY: &str = r#" SELECT sc.json, sc.id, s.tx_hash FROM solver_competitions sc -JOIN auction_transaction at ON sc.id = at.auction_id -JOIN settlements s ON (at.tx_from, at.tx_nonce) = (s.tx_from, s.tx_nonce) +JOIN settlements s ON sc.id = s.auction_id WHERE s.tx_hash = $1 ;"#; sqlx::query_as(QUERY).bind(tx_hash).fetch_optional(ex).await diff --git a/crates/e2e/tests/e2e/database.rs b/crates/e2e/tests/e2e/database.rs index d18364e146..e4ca9a41c7 100644 --- a/crates/e2e/tests/e2e/database.rs +++ b/crates/e2e/tests/e2e/database.rs @@ -21,8 +21,6 @@ pub async fn events_of_order(db: &Db, uid: &OrderUid) -> Vec Option { let mut db = db.acquire().await.unwrap(); const LAST_AUCTION_ID: &str = - "SELECT auction_id FROM auction_transaction ORDER BY auction_id DESC LIMIT 1"; + "SELECT auction_id FROM settlements ORDER BY auction_id DESC LIMIT 1"; let auction_id: i64 = sqlx::query_scalar(LAST_AUCTION_ID) .fetch_optional(db.deref_mut()) .await .unwrap()?; const TX_QUERY: &str = r" -SELECT s.* -FROM auction_transaction at -JOIN settlements s ON s.tx_from = at.tx_from AND s.tx_nonce = at.tx_nonce -WHERE at.auction_id = $1 - "; +SELECT * FROM settlements WHERE auction_id = $1"; + let tx: AuctionTransaction = sqlx::query_as(TX_QUERY) .bind(auction_id) .fetch_optional(db.deref_mut()) From 53ce32e81c7cf76f608892eee647db3c27e9dc7c Mon Sep 17 00:00:00 2001 From: sunce86 Date: Fri, 19 Jan 2024 00:40:16 +0100 Subject: [PATCH 07/23] fix cip_20_data_updated --- crates/e2e/src/setup/services.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/e2e/src/setup/services.rs b/crates/e2e/src/setup/services.rs index a8892a4f1c..13f86660ef 100644 --- a/crates/e2e/src/setup/services.rs +++ b/crates/e2e/src/setup/services.rs @@ -429,7 +429,7 @@ pub async fn clear_database() { tracing::info!("Clearing database."); let mut db = sqlx::PgConnection::connect(LOCAL_DB_URL).await.unwrap(); let mut db = db.begin().await.unwrap(); - database::clear_DANGER_(&mut db).await.unwrap(); + database::clear_DANGER_(&mut db).await.unwrap(); db.commit().await.unwrap(); } From 757e76bb0f60920748f3bccea585b17d6a11cb17 Mon Sep 17 00:00:00 2001 From: sunce86 Date: Fri, 19 Jan 2024 00:40:25 +0100 Subject: [PATCH 08/23] fix cip_20_data_updated --- crates/e2e/tests/e2e/univ2.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/crates/e2e/tests/e2e/univ2.rs b/crates/e2e/tests/e2e/univ2.rs index 58335da635..bbb56b3a8b 100644 --- a/crates/e2e/tests/e2e/univ2.rs +++ b/crates/e2e/tests/e2e/univ2.rs @@ -104,8 +104,6 @@ async fn test(web3: Web3) { && data.participants.iter().any(|p| p.participant.0 == solver.address().0) // and won the auction && data.score.winner.0 == solver.address().0 - // and submitted the solution - && data.tx.tx_from.0 == solver.address().0 // and calldata is present && !data.call_data.call_data.is_empty() }; From b94835ecb60445fc992645b2f5fc06f8b0d86c92 Mon Sep 17 00:00:00 2001 From: sunce86 Date: Fri, 19 Jan 2024 00:41:15 +0100 Subject: [PATCH 09/23] revent nonsense --- crates/e2e/src/setup/services.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/e2e/src/setup/services.rs b/crates/e2e/src/setup/services.rs index 13f86660ef..a8892a4f1c 100644 --- a/crates/e2e/src/setup/services.rs +++ b/crates/e2e/src/setup/services.rs @@ -429,7 +429,7 @@ pub async fn clear_database() { tracing::info!("Clearing database."); let mut db = sqlx::PgConnection::connect(LOCAL_DB_URL).await.unwrap(); let mut db = db.begin().await.unwrap(); - database::clear_DANGER_(&mut db).await.unwrap(); + database::clear_DANGER_(&mut db).await.unwrap(); db.commit().await.unwrap(); } From 94beabcda739a47b4c751d0707b26e40cb469c7f Mon Sep 17 00:00:00 2001 From: sunce86 Date: Fri, 19 Jan 2024 00:46:54 +0100 Subject: [PATCH 10/23] fix sql --- crates/database/src/settlements.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/database/src/settlements.rs b/crates/database/src/settlements.rs index e97726f0f8..ab56a59045 100644 --- a/crates/database/src/settlements.rs +++ b/crates/database/src/settlements.rs @@ -75,7 +75,7 @@ pub async fn update_settlement_auction( ) -> Result { const QUERY: &str = r#" UPDATE settlements -SET auction_kind = $1 and auction_id = $2 +SET auction_kind = $1, auction_id = $2 WHERE block_number = $3 AND log_index = $4 ;"#; sqlx::query(QUERY) From 2f8a0703f6999ece8142844b98c518ada5f14f7b Mon Sep 17 00:00:00 2001 From: sunce86 Date: Fri, 19 Jan 2024 10:17:19 +0100 Subject: [PATCH 11/23] fix query --- crates/database/src/settlements.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/database/src/settlements.rs b/crates/database/src/settlements.rs index ab56a59045..c3ba6d7648 100644 --- a/crates/database/src/settlements.rs +++ b/crates/database/src/settlements.rs @@ -60,7 +60,7 @@ pub async fn get_settlement_without_auction( const QUERY: &str = r#" SELECT block_number, log_index, tx_hash FROM settlements -WHERE auction_kind IS unprocessed +WHERE auction_kind = 'unprocessed' LIMIT 1 "#; sqlx::query_as(QUERY).fetch_optional(ex).await From c2e9478be1121e9a2593a9fb9b91e9ea3f8c68fa Mon Sep 17 00:00:00 2001 From: sunce86 Date: Fri, 19 Jan 2024 11:05:07 +0100 Subject: [PATCH 12/23] migration fix --- .../sql/V058__drop_auction_transaction.sql | 36 +++++++++++++++++-- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/database/sql/V058__drop_auction_transaction.sql b/database/sql/V058__drop_auction_transaction.sql index 2ef914584a..ed31b25606 100644 --- a/database/sql/V058__drop_auction_transaction.sql +++ b/database/sql/V058__drop_auction_transaction.sql @@ -1,9 +1,39 @@ +-- Step 1: Add the new columns to the settlements table CREATE TYPE AuctionKind AS ENUM ('valid', 'invalid', 'unprocessed'); ALTER TABLE settlements - DROP COLUMN tx_from, - DROP COLUMN tx_nonce, ADD COLUMN auction_kind AuctionKind NOT NULL DEFAULT 'unprocessed', ADD COLUMN auction_id bigint; -DROP TABLE auction_transaction; \ No newline at end of file +-- Step 2a: Populate auction_kind and auction_id columns +-- For all settlements that already have an auction_transaction record, set auction_kind to 'valid' and auction_id to the auction_transaction's auction_id +UPDATE settlements +SET auction_kind = 'valid', + auction_id = auction_transaction.auction_id +FROM auction_transaction +WHERE settlements.from = auction_transaction.from AND settlements.nonce = auction_transaction.nonce; + +-- Step 2b: Populate auction_kind and auction_id columns +-- For all settlements that have auction_id = NULL, set auction_kind to 'invalid' +UPDATE settlements +SET auction_kind = 'invalid' +WHERE auction_id IS NULL; + +-- Step 2c: Populate auction_kind and auction_id columns +-- For all settlements, going from the most recent to the oldest, set auction_kind to 'unprocessed' until the first settlement with auction_kind = 'valid' is found +UPDATE settlements +SET + auction_kind = 'unprocessed' +WHERE + auction_kind = 'invalid' + AND block_number > ( + SELECT MAX(block_number) + FROM settlements + WHERE auction_kind = 'valid'; + ); + +-- Step 4: Drop the auction_transaction table, and the tx_from and tx_nonce columns from the settlements table +DROP TABLE auction_transaction; +ALTER TABLE settlements + DROP COLUMN tx_from, + DROP COLUMN tx_nonce; From 9ee0a4f68770d5dc14c8240ad2d00cc36402ccd5 Mon Sep 17 00:00:00 2001 From: sunce86 Date: Fri, 19 Jan 2024 11:54:05 +0100 Subject: [PATCH 13/23] fix syntax errors --- database/sql/V058__drop_auction_transaction.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/database/sql/V058__drop_auction_transaction.sql b/database/sql/V058__drop_auction_transaction.sql index ed31b25606..27d463ffd0 100644 --- a/database/sql/V058__drop_auction_transaction.sql +++ b/database/sql/V058__drop_auction_transaction.sql @@ -11,7 +11,7 @@ UPDATE settlements SET auction_kind = 'valid', auction_id = auction_transaction.auction_id FROM auction_transaction -WHERE settlements.from = auction_transaction.from AND settlements.nonce = auction_transaction.nonce; +WHERE settlements.tx_from = auction_transaction.tx_from AND settlements.tx_nonce = auction_transaction.tx_nonce; -- Step 2b: Populate auction_kind and auction_id columns -- For all settlements that have auction_id = NULL, set auction_kind to 'invalid' @@ -29,7 +29,7 @@ WHERE AND block_number > ( SELECT MAX(block_number) FROM settlements - WHERE auction_kind = 'valid'; + WHERE auction_kind = 'valid' ); -- Step 4: Drop the auction_transaction table, and the tx_from and tx_nonce columns from the settlements table From 0346f8047a5b2dddff087dc44f9ce892d8a2cf38 Mon Sep 17 00:00:00 2001 From: sunce86 Date: Fri, 19 Jan 2024 13:21:24 +0100 Subject: [PATCH 14/23] tests --- .../src/on_settlement_event_updater.rs | 2 ++ crates/database/src/settlements.rs | 31 +++++++++++++++++++ crates/e2e/tests/e2e/database.rs | 4 +-- crates/e2e/tests/e2e/univ2.rs | 3 +- 4 files changed, 37 insertions(+), 3 deletions(-) diff --git a/crates/autopilot/src/on_settlement_event_updater.rs b/crates/autopilot/src/on_settlement_event_updater.rs index f97b5ae197..52898f76e5 100644 --- a/crates/autopilot/src/on_settlement_event_updater.rs +++ b/crates/autopilot/src/on_settlement_event_updater.rs @@ -103,6 +103,8 @@ impl OnSettlementEventUpdater { } } current_block = block_stream.next().await.expect("blockchains never end"); + // Wait a bit more to not race with the event indexer. + tokio::time::sleep(std::time::Duration::from_secs(1)).await; } } diff --git a/crates/database/src/settlements.rs b/crates/database/src/settlements.rs index c3ba6d7648..b137030ccc 100644 --- a/crates/database/src/settlements.rs +++ b/crates/database/src/settlements.rs @@ -155,4 +155,35 @@ mod tests { let results = recent_settlement_tx_hashes(&mut db, 3..5).await.unwrap(); assert_eq!(results, &[]); } + + #[tokio::test] + #[ignore] + async fn postgres_settlement_auction() { + let mut db = PgConnection::connect("postgresql://").await.unwrap(); + let mut db = db.begin().await.unwrap(); + crate::clear_DANGER_(&mut db).await.unwrap(); + + let event = Default::default(); + crate::events::insert_settlement(&mut db, &event, &Default::default()) + .await + .unwrap(); + + let settlement = get_settlement_without_auction(&mut db) + .await + .unwrap() + .unwrap(); + + assert_eq!(settlement.block_number, event.block_number); + assert_eq!(settlement.log_index, event.log_index); + + update_settlement_auction(&mut db, 0, 0, Some(1), AuctionKind::Valid) + .await + .unwrap(); + + let settlement = get_settlement_without_auction(&mut db) + .await + .unwrap(); + + assert!(settlement.is_none()); + } } diff --git a/crates/e2e/tests/e2e/database.rs b/crates/e2e/tests/e2e/database.rs index e4ca9a41c7..5c566c35b7 100644 --- a/crates/e2e/tests/e2e/database.rs +++ b/crates/e2e/tests/e2e/database.rs @@ -40,12 +40,12 @@ pub struct Cip20Data { } /// Returns `Some(data)` if the all the expected CIP-20 data has been indexed -/// for the most recent `auction_transaction`. +/// for the most recent `auction_id` from `settlements` table. pub async fn most_recent_cip_20_data(db: &Db) -> Option { let mut db = db.acquire().await.unwrap(); const LAST_AUCTION_ID: &str = - "SELECT auction_id FROM settlements ORDER BY auction_id DESC LIMIT 1"; + "SELECT auction_id FROM settlements WHERE auction_id IS NOT NULL ORDER BY auction_id DESC LIMIT 1"; let auction_id: i64 = sqlx::query_scalar(LAST_AUCTION_ID) .fetch_optional(db.deref_mut()) .await diff --git a/crates/e2e/tests/e2e/univ2.rs b/crates/e2e/tests/e2e/univ2.rs index bbb56b3a8b..d0ff762a75 100644 --- a/crates/e2e/tests/e2e/univ2.rs +++ b/crates/e2e/tests/e2e/univ2.rs @@ -89,7 +89,8 @@ async fn test(web3: Web3) { .await .unwrap(); - onchain.mint_blocks_past_reorg_threshold().await; + // mint block to re-trigger OnSettlementEventUpdater update loop + onchain.mint_block().await; let cip_20_data_updated = || async { let data = match crate::database::most_recent_cip_20_data(services.db()).await { From b221e8cbe7991c8c160fcaa66128bb37ba46fc52 Mon Sep 17 00:00:00 2001 From: sunce86 Date: Fri, 19 Jan 2024 13:28:44 +0100 Subject: [PATCH 15/23] e2e tests --- crates/database/src/settlements.rs | 4 +--- crates/e2e/src/setup/onchain_components.rs | 17 ----------------- crates/e2e/tests/e2e/database.rs | 4 ++-- crates/e2e/tests/e2e/partial_fill.rs | 2 -- crates/e2e/tests/e2e/partially_fillable_pool.rs | 1 - crates/e2e/tests/e2e/protocol_fee.rs | 1 - crates/e2e/tests/e2e/solver_competition.rs | 2 -- crates/e2e/tests/e2e/univ2.rs | 7 +++---- 8 files changed, 6 insertions(+), 32 deletions(-) diff --git a/crates/database/src/settlements.rs b/crates/database/src/settlements.rs index b137030ccc..b645611c81 100644 --- a/crates/database/src/settlements.rs +++ b/crates/database/src/settlements.rs @@ -180,9 +180,7 @@ mod tests { .await .unwrap(); - let settlement = get_settlement_without_auction(&mut db) - .await - .unwrap(); + let settlement = get_settlement_without_auction(&mut db).await.unwrap(); assert!(settlement.is_none()); } diff --git a/crates/e2e/src/setup/onchain_components.rs b/crates/e2e/src/setup/onchain_components.rs index a52a776aee..d902d91a35 100644 --- a/crates/e2e/src/setup/onchain_components.rs +++ b/crates/e2e/src/setup/onchain_components.rs @@ -475,23 +475,6 @@ impl OnchainComponents { .unwrap(); } - /// We will only index events when they are 64 blocks old so we don't have - /// to throw out indexed data on reorgs. - /// This function executes enough transactions to ensure that all events - /// before calling this function are old enough to be indexed. - pub async fn mint_blocks_past_reorg_threshold(&self) { - tracing::info!("mining blocks to get past the reorg safety threshold for indexing events"); - - let current_block = || async { self.web3.eth().block_number().await.unwrap() }; - let target = current_block().await + 65; - // Simply issuing n transactions may not result in n blocks being mined. - // Therefore we continually call `evm_mine` until we can confirm the block - // number increased the expected number of times. - while current_block().await <= target { - let _ = self.web3.transport().execute("evm_mine", vec![]).await; - } - } - pub async fn mint_block(&self) { tracing::info!("mining block"); self.web3 diff --git a/crates/e2e/tests/e2e/database.rs b/crates/e2e/tests/e2e/database.rs index 5c566c35b7..58a89c00ec 100644 --- a/crates/e2e/tests/e2e/database.rs +++ b/crates/e2e/tests/e2e/database.rs @@ -44,8 +44,8 @@ pub struct Cip20Data { pub async fn most_recent_cip_20_data(db: &Db) -> Option { let mut db = db.acquire().await.unwrap(); - const LAST_AUCTION_ID: &str = - "SELECT auction_id FROM settlements WHERE auction_id IS NOT NULL ORDER BY auction_id DESC LIMIT 1"; + const LAST_AUCTION_ID: &str = "SELECT auction_id FROM settlements WHERE auction_id IS NOT \ + NULL ORDER BY auction_id DESC LIMIT 1"; let auction_id: i64 = sqlx::query_scalar(LAST_AUCTION_ID) .fetch_optional(db.deref_mut()) .await diff --git a/crates/e2e/tests/e2e/partial_fill.rs b/crates/e2e/tests/e2e/partial_fill.rs index 58c396fee2..f9b6aa8613 100644 --- a/crates/e2e/tests/e2e/partial_fill.rs +++ b/crates/e2e/tests/e2e/partial_fill.rs @@ -89,8 +89,6 @@ async fn test(web3: Web3) { .contains(&buy_balance.as_u128()) ); - onchain.mint_blocks_past_reorg_threshold().await; - let settlement_event_processed = || async { onchain.mint_block().await; let order = services.get_order(&uid).await.unwrap(); diff --git a/crates/e2e/tests/e2e/partially_fillable_pool.rs b/crates/e2e/tests/e2e/partially_fillable_pool.rs index ccc78a87d6..913fd14ad1 100644 --- a/crates/e2e/tests/e2e/partially_fillable_pool.rs +++ b/crates/e2e/tests/e2e/partially_fillable_pool.rs @@ -117,7 +117,6 @@ async fn test(web3: Web3) { .contains(&buy_balance.as_u128()) ); - onchain.mint_blocks_past_reorg_threshold().await; let metadata_updated = || async { onchain.mint_block().await; let order = services.get_order(&uid).await.unwrap(); diff --git a/crates/e2e/tests/e2e/protocol_fee.rs b/crates/e2e/tests/e2e/protocol_fee.rs index 14b8da57be..00a8116eab 100644 --- a/crates/e2e/tests/e2e/protocol_fee.rs +++ b/crates/e2e/tests/e2e/protocol_fee.rs @@ -328,7 +328,6 @@ async fn execute_test( .await .unwrap(); - onchain.mint_blocks_past_reorg_threshold().await; let metadata_updated = || async { onchain.mint_block().await; let order = services.get_order(&uid).await.unwrap(); diff --git a/crates/e2e/tests/e2e/solver_competition.rs b/crates/e2e/tests/e2e/solver_competition.rs index b205c6ef67..e735e09050 100644 --- a/crates/e2e/tests/e2e/solver_competition.rs +++ b/crates/e2e/tests/e2e/solver_competition.rs @@ -90,8 +90,6 @@ async fn solver_competition(web3: Web3) { || async { token_a.balance_of(trader.address()).call().await.unwrap() == U256::zero() }; wait_for_condition(TIMEOUT, trade_happened).await.unwrap(); - onchain.mint_blocks_past_reorg_threshold().await; - let indexed_trades = || async { onchain.mint_block().await; match services.get_trades(&uid).await.unwrap().first() { diff --git a/crates/e2e/tests/e2e/univ2.rs b/crates/e2e/tests/e2e/univ2.rs index d0ff762a75..12676b21a0 100644 --- a/crates/e2e/tests/e2e/univ2.rs +++ b/crates/e2e/tests/e2e/univ2.rs @@ -88,11 +88,10 @@ async fn test(web3: Web3) { wait_for_condition(TIMEOUT, all_events_registered) .await .unwrap(); - - // mint block to re-trigger OnSettlementEventUpdater update loop - onchain.mint_block().await; - + let cip_20_data_updated = || async { + onchain.mint_block().await; + let data = match crate::database::most_recent_cip_20_data(services.db()).await { Some(data) => data, None => return false, From fca0198099bc53d8a3571b4ea0c4fb8a08e4e8ed Mon Sep 17 00:00:00 2001 From: sunce86 Date: Fri, 19 Jan 2024 13:36:12 +0100 Subject: [PATCH 16/23] small fixes --- crates/autopilot/src/on_settlement_event_updater.rs | 3 +++ crates/e2e/tests/e2e/univ2.rs | 2 +- crates/orderbook/src/database/solver_competition.rs | 2 -- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/autopilot/src/on_settlement_event_updater.rs b/crates/autopilot/src/on_settlement_event_updater.rs index 52898f76e5..89117a3f74 100644 --- a/crates/autopilot/src/on_settlement_event_updater.rs +++ b/crates/autopilot/src/on_settlement_event_updater.rs @@ -103,7 +103,10 @@ impl OnSettlementEventUpdater { } } current_block = block_stream.next().await.expect("blockchains never end"); + // Wait a bit more to not race with the event indexer. + // Otherwise we might miss event and have to wait for next block to retrigger + // loop. tokio::time::sleep(std::time::Duration::from_secs(1)).await; } } diff --git a/crates/e2e/tests/e2e/univ2.rs b/crates/e2e/tests/e2e/univ2.rs index 12676b21a0..d4aa0859e0 100644 --- a/crates/e2e/tests/e2e/univ2.rs +++ b/crates/e2e/tests/e2e/univ2.rs @@ -88,7 +88,7 @@ async fn test(web3: Web3) { wait_for_condition(TIMEOUT, all_events_registered) .await .unwrap(); - + let cip_20_data_updated = || async { onchain.mint_block().await; diff --git a/crates/orderbook/src/database/solver_competition.rs b/crates/orderbook/src/database/solver_competition.rs index ababfc419c..cfabb1065c 100644 --- a/crates/orderbook/src/database/solver_competition.rs +++ b/crates/orderbook/src/database/solver_competition.rs @@ -48,8 +48,6 @@ impl SolverCompetitionStoring for Postgres { row.tx_hash.map(|hash| H256(hash.0)), ) }), - // TODO: change this query to use the auction_transaction and settlements tables to - // find the tx hash. Identifier::Transaction(hash) => { database::solver_competition::load_by_tx_hash(&mut ex, &ByteArray(hash.0)) .await From 062549c484d446d4d94fc3a4fcd23b04fe7535bc Mon Sep 17 00:00:00 2001 From: sunce86 Date: Fri, 19 Jan 2024 14:18:22 +0100 Subject: [PATCH 17/23] remove mint blocks --- crates/e2e/tests/e2e/partial_fill.rs | 1 - crates/e2e/tests/e2e/partially_fillable_pool.rs | 1 - crates/e2e/tests/e2e/protocol_fee.rs | 1 - crates/e2e/tests/e2e/univ2.rs | 2 -- 4 files changed, 5 deletions(-) diff --git a/crates/e2e/tests/e2e/partial_fill.rs b/crates/e2e/tests/e2e/partial_fill.rs index f9b6aa8613..8b17b27093 100644 --- a/crates/e2e/tests/e2e/partial_fill.rs +++ b/crates/e2e/tests/e2e/partial_fill.rs @@ -90,7 +90,6 @@ async fn test(web3: Web3) { ); let settlement_event_processed = || async { - onchain.mint_block().await; let order = services.get_order(&uid).await.unwrap(); order.metadata.executed_surplus_fee > U256::zero() }; diff --git a/crates/e2e/tests/e2e/partially_fillable_pool.rs b/crates/e2e/tests/e2e/partially_fillable_pool.rs index 913fd14ad1..04c77d75bd 100644 --- a/crates/e2e/tests/e2e/partially_fillable_pool.rs +++ b/crates/e2e/tests/e2e/partially_fillable_pool.rs @@ -118,7 +118,6 @@ async fn test(web3: Web3) { ); let metadata_updated = || async { - onchain.mint_block().await; let order = services.get_order(&uid).await.unwrap(); !order.metadata.executed_surplus_fee.is_zero() && order.metadata.executed_buy_amount != Default::default() diff --git a/crates/e2e/tests/e2e/protocol_fee.rs b/crates/e2e/tests/e2e/protocol_fee.rs index 00a8116eab..3d2e240bc2 100644 --- a/crates/e2e/tests/e2e/protocol_fee.rs +++ b/crates/e2e/tests/e2e/protocol_fee.rs @@ -329,7 +329,6 @@ async fn execute_test( .unwrap(); let metadata_updated = || async { - onchain.mint_block().await; let order = services.get_order(&uid).await.unwrap(); is_approximately_equal(order.metadata.executed_surplus_fee, expected_surplus_fee) }; diff --git a/crates/e2e/tests/e2e/univ2.rs b/crates/e2e/tests/e2e/univ2.rs index d4aa0859e0..6fab8e4abf 100644 --- a/crates/e2e/tests/e2e/univ2.rs +++ b/crates/e2e/tests/e2e/univ2.rs @@ -90,8 +90,6 @@ async fn test(web3: Web3) { .unwrap(); let cip_20_data_updated = || async { - onchain.mint_block().await; - let data = match crate::database::most_recent_cip_20_data(services.db()).await { Some(data) => data, None => return false, From f10361da8a09736aac2dc92dbbae5deb3efd11b2 Mon Sep 17 00:00:00 2001 From: sunce86 Date: Sat, 20 Jan 2024 11:19:25 +0100 Subject: [PATCH 18/23] small fixes --- crates/database/src/settlements.rs | 17 ++++++++++++----- database/sql/V058__drop_auction_transaction.sql | 2 +- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/crates/database/src/settlements.rs b/crates/database/src/settlements.rs index b645611c81..40a626f05b 100644 --- a/crates/database/src/settlements.rs +++ b/crates/database/src/settlements.rs @@ -1,6 +1,6 @@ use { crate::{events::EventIndex, TransactionHash}, - sqlx::{postgres::PgQueryResult, PgConnection}, + sqlx::PgConnection, std::ops::Range, }; @@ -72,7 +72,7 @@ pub async fn update_settlement_auction( log_index: i64, auction_id: Option, auction_kind: AuctionKind, -) -> Result { +) -> Result<(), sqlx::Error> { const QUERY: &str = r#" UPDATE settlements SET auction_kind = $1, auction_id = $2 @@ -85,6 +85,7 @@ WHERE block_number = $3 AND log_index = $4 .bind(log_index) .execute(ex) .await + .map(|_| ()) } #[cfg(test)] @@ -176,9 +177,15 @@ mod tests { assert_eq!(settlement.block_number, event.block_number); assert_eq!(settlement.log_index, event.log_index); - update_settlement_auction(&mut db, 0, 0, Some(1), AuctionKind::Valid) - .await - .unwrap(); + update_settlement_auction( + &mut db, + event.block_number, + event.log_index, + Some(1), + AuctionKind::Valid, + ) + .await + .unwrap(); let settlement = get_settlement_without_auction(&mut db).await.unwrap(); diff --git a/database/sql/V058__drop_auction_transaction.sql b/database/sql/V058__drop_auction_transaction.sql index 27d463ffd0..a42d6fcecb 100644 --- a/database/sql/V058__drop_auction_transaction.sql +++ b/database/sql/V058__drop_auction_transaction.sql @@ -32,7 +32,7 @@ WHERE WHERE auction_kind = 'valid' ); --- Step 4: Drop the auction_transaction table, and the tx_from and tx_nonce columns from the settlements table +-- Step 3: Drop the auction_transaction table, and the tx_from and tx_nonce columns from the settlements table DROP TABLE auction_transaction; ALTER TABLE settlements DROP COLUMN tx_from, From f000781a8ade0b7b40366a4a9d156e030b808dca Mon Sep 17 00:00:00 2001 From: sunce86 Date: Mon, 22 Jan 2024 12:23:16 +0100 Subject: [PATCH 19/23] return oldest unprocessed settlement --- crates/database/src/settlements.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/database/src/settlements.rs b/crates/database/src/settlements.rs index 40a626f05b..1b41b3f3a6 100644 --- a/crates/database/src/settlements.rs +++ b/crates/database/src/settlements.rs @@ -61,6 +61,7 @@ pub async fn get_settlement_without_auction( SELECT block_number, log_index, tx_hash FROM settlements WHERE auction_kind = 'unprocessed' +ORDER BY block_number ASC LIMIT 1 "#; sqlx::query_as(QUERY).fetch_optional(ex).await From f6a01522e17703b1177113269f094a555e868368 Mon Sep 17 00:00:00 2001 From: Felix Leupold Date: Wed, 24 Jan 2024 17:04:53 +0100 Subject: [PATCH 20/23] remove auction kind --- .../database/on_settlement_event_updater.rs | 17 +- .../src/on_settlement_event_updater.rs | 313 +++++++++--------- crates/database/src/settlements.rs | 49 ++- crates/database/src/solver_competition.rs | 3 +- crates/e2e/src/setup/colocation.rs | 5 +- crates/e2e/src/setup/deploy.rs | 16 +- crates/e2e/src/setup/onchain_components.rs | 17 + crates/e2e/src/setup/services.rs | 7 +- crates/e2e/tests/e2e/app_data.rs | 18 + crates/e2e/tests/e2e/database.rs | 15 +- crates/e2e/tests/e2e/limit_orders.rs | 135 +++++++- crates/e2e/tests/e2e/partial_fill.rs | 3 + .../e2e/tests/e2e/partially_fillable_pool.rs | 2 + crates/e2e/tests/e2e/protocol_fee.rs | 2 + crates/e2e/tests/e2e/solver_competition.rs | 2 + crates/e2e/tests/e2e/univ2.rs | 4 + .../V058__add_auction_id_to_settlements.sql | 12 + .../sql/V058__drop_auction_transaction.sql | 39 --- 18 files changed, 396 insertions(+), 263 deletions(-) create mode 100644 database/sql/V058__add_auction_id_to_settlements.sql delete mode 100644 database/sql/V058__drop_auction_transaction.sql diff --git a/crates/autopilot/src/database/on_settlement_event_updater.rs b/crates/autopilot/src/database/on_settlement_event_updater.rs index d4d7ee42b5..2fa304be39 100644 --- a/crates/autopilot/src/database/on_settlement_event_updater.rs +++ b/crates/autopilot/src/database/on_settlement_event_updater.rs @@ -1,5 +1,4 @@ use { - crate::on_settlement_event_updater::AuctionKind, anyhow::{Context, Result}, database::{byte_array::ByteArray, settlement_observations::Observation}, ethcontract::U256, @@ -14,7 +13,6 @@ pub type AuctionId = i64; #[derive(Debug, Default, Clone)] pub struct AuctionData { - pub auction_id: AuctionId, pub gas_used: U256, pub effective_gas_price: U256, pub surplus: U256, @@ -26,7 +24,8 @@ pub struct AuctionData { pub struct SettlementUpdate { pub block_number: i64, pub log_index: i64, - pub auction_kind: AuctionKind, + pub auction_id: AuctionId, + /// Only set if the auction is for this environment. pub auction_data: Option, } @@ -40,20 +39,12 @@ impl super::Postgres { .with_label_values(&["update_settlement_details"]) .start_timer(); - let (auction_id, auction_kind) = match settlement_update.auction_kind { - AuctionKind::Valid { auction_id } => { - (Some(auction_id), database::settlements::AuctionKind::Valid) - } - AuctionKind::Invalid => (None, database::settlements::AuctionKind::Invalid), - }; - // update settlements database::settlements::update_settlement_auction( ex, settlement_update.block_number, settlement_update.log_index, - auction_id, - auction_kind, + settlement_update.auction_id, ) .await .context("insert_settlement_tx_info")?; @@ -77,7 +68,7 @@ impl super::Postgres { database::order_execution::save( ex, &ByteArray(order.0), - auction_data.auction_id, + settlement_update.auction_id, &u256_to_big_decimal(&executed_fee), ) .await diff --git a/crates/autopilot/src/on_settlement_event_updater.rs b/crates/autopilot/src/on_settlement_event_updater.rs index 89117a3f74..643ee1b4ef 100644 --- a/crates/autopilot/src/on_settlement_event_updater.rs +++ b/crates/autopilot/src/on_settlement_event_updater.rs @@ -36,54 +36,37 @@ use { on_settlement_event_updater::{AuctionData, SettlementUpdate}, Postgres, }, - decoded_settlement::{DecodedSettlement, DecodingError}, + decoded_settlement::DecodedSettlement, infra, }, anyhow::{Context, Result}, futures::StreamExt, primitive_types::H256, - shared::external_prices::ExternalPrices, + shared::{event_handling::MAX_REORG_BLOCK_COUNT, external_prices::ExternalPrices}, sqlx::PgConnection, web3::types::Transaction, }; -#[derive(Debug, Copy, Clone)] -pub enum AuctionKind { - /// This auction is regular and all the auction dependent data should be - /// updated. - Valid { auction_id: i64 }, - /// Some possible reasons to have invalid auction are: - /// - This auction was created by another environment (e.g. - /// production/staging) - /// - Failed to decode settlement calldata - /// - Failed to recover auction id from calldata - /// - Settlement transaction was submitted by solver other than the winner - /// - /// In this case, settlement event should be marked as invalid and no - /// auction dependent data is updated. - Invalid, -} - -impl AuctionKind { - pub fn auction_id(&self) -> Option { - match self { - AuctionKind::Valid { auction_id } => Some(*auction_id), - AuctionKind::Invalid => None, - } - } -} - pub struct OnSettlementEventUpdater { pub eth: infra::Ethereum, pub db: Postgres, } +enum AuctionIdRecoveryStatus { + /// The auction id was recovered and the auction data should be added. + AddAuctionData(i64, DecodedSettlement), + /// The auction id was recovered but the auction data should not be added. + DoNotAddAuctionData(i64), + /// The auction id was not recovered. + InvalidCalldata, +} + impl OnSettlementEventUpdater { pub async fn run_forever(self) -> ! { let mut current_block = self.eth.current_block().borrow().to_owned(); let mut block_stream = ethrpc::current_block::into_stream(self.eth.current_block().clone()); loop { - match self.update().await { + match self.update(current_block.number).await { Ok(true) => { tracing::debug!( block = current_block.number, @@ -103,31 +86,33 @@ impl OnSettlementEventUpdater { } } current_block = block_stream.next().await.expect("blockchains never end"); - - // Wait a bit more to not race with the event indexer. - // Otherwise we might miss event and have to wait for next block to retrigger - // loop. - tokio::time::sleep(std::time::Duration::from_secs(1)).await; } } /// Update database for settlement events that have not been processed yet. /// /// Returns whether an update was performed. - async fn update(&self) -> Result { + async fn update(&self, current_block: u64) -> Result { + let reorg_safe_block: i64 = current_block + .checked_sub(MAX_REORG_BLOCK_COUNT) + .context("no reorg safe block")? + .try_into() + .context("convert block")?; + let mut ex = self .db .pool .begin() .await .context("acquire DB connection")?; - let event = match database::settlements::get_settlement_without_auction(&mut ex) - .await - .context("get_settlement_without_auction")? - { - Some(event) => event, - None => return Ok(false), - }; + let event = + match database::settlements::get_settlement_without_auction(&mut ex, reorg_safe_block) + .await + .context("get_settlement_event_without_tx_info")? + { + Some(event) => event, + None => return Ok(false), + }; let hash = H256(event.tx_hash.0); tracing::debug!("updating settlement details for tx {hash:?}"); @@ -138,104 +123,28 @@ impl OnSettlementEventUpdater { .await? .with_context(|| format!("no tx {hash:?}"))?; - let auction_kind = Self::get_auction_kind(&mut ex, &transaction).await?; + let (auction_id, auction_data) = + match Self::recover_auction_id_from_calldata(&mut ex, &transaction).await? { + AuctionIdRecoveryStatus::InvalidCalldata => { + return Ok(false); + } + AuctionIdRecoveryStatus::DoNotAddAuctionData(auction_id) => (auction_id, None), + AuctionIdRecoveryStatus::AddAuctionData(auction_id, settlement) => ( + auction_id, + Some( + self.fetch_auction_data(hash, settlement, auction_id, &mut ex) + .await?, + ), + ), + }; - let mut update = SettlementUpdate { + let update = SettlementUpdate { block_number: event.block_number, log_index: event.log_index, - auction_kind, - auction_data: None, + auction_id, + auction_data, }; - // It is possible that auction_id does not exist for a transaction. - // This happens for production auctions queried from the staging environment and - // vice versa (because we have databases for both environments). - // - // If auction_id exists, we expect all other relevant information to exist as - // well. - if let AuctionKind::Valid { auction_id } = auction_kind { - let receipt = self - .eth - .transaction_receipt(hash) - .await? - .with_context(|| format!("no receipt {hash:?}"))?; - let gas_used = receipt - .gas_used - .with_context(|| format!("no gas used {hash:?}"))?; - let effective_gas_price = receipt - .effective_gas_price - .with_context(|| format!("no effective gas price {hash:?}"))?; - let auction_external_prices = Postgres::get_auction_prices(&mut ex, auction_id) - .await - .with_context(|| { - format!("no external prices for auction id {auction_id:?} and tx {hash:?}") - })?; - let external_prices = ExternalPrices::try_from_auction_prices( - self.eth.contracts().weth().address(), - auction_external_prices.clone(), - )?; - - tracing::debug!( - ?auction_id, - ?auction_external_prices, - ?external_prices, - "observations input" - ); - - // surplus and fees calculation - match DecodedSettlement::new(&transaction.input.0) { - Ok(settlement) => { - let domain_separator = self.eth.contracts().settlement_domain_separator(); - let order_uids = settlement.order_uids(domain_separator)?; - let order_fees = order_uids - .clone() - .into_iter() - .zip(Postgres::order_fees(&mut ex, &order_uids).await?) - .collect::>(); - - let surplus = settlement.total_surplus(&external_prices); - let (fee, order_executions) = { - let all_fees = settlement.all_fees(&external_prices, &order_fees); - // total unsubsidized fee used for CIP20 rewards - let fee = all_fees - .iter() - .fold(0.into(), |acc, fees| acc + fees.native); - // executed fees for each order execution - let order_executions = all_fees - .into_iter() - .zip(order_fees.iter()) - .filter_map(|(fee, (_, order_fee))| match order_fee { - // filter out orders with order_fee - // since their fee can already be fetched from the database table - // `orders` so no point in storing the same - // value twice, in another table - Some(_) => None, - None => Some((fee.order, fee.sell)), - }) - .collect(); - (fee, order_executions) - }; - - update.auction_data = Some(AuctionData { - auction_id, - surplus, - fee, - gas_used, - effective_gas_price, - order_executions, - }); - } - Err(DecodingError::InvalidSelector) => { - // we indexed a transaction initiated by solver, that was not a settlement - // for this case we want to have the entry in observations table but with zeros - update.auction_data = Some(Default::default()); - } - Err(err) => { - return Err(err.into()); - } - } - } - tracing::debug!(?hash, ?update, "updating settlement details for tx"); Postgres::update_settlement_details(&mut ex, update.clone()) @@ -245,55 +154,139 @@ impl OnSettlementEventUpdater { Ok(true) } + async fn fetch_auction_data( + &self, + hash: H256, + settlement: DecodedSettlement, + auction_id: i64, + ex: &mut PgConnection, + ) -> Result { + let receipt = self + .eth + .transaction_receipt(hash) + .await? + .with_context(|| format!("no receipt {hash:?}"))?; + let gas_used = receipt + .gas_used + .with_context(|| format!("no gas used {hash:?}"))?; + let effective_gas_price = receipt + .effective_gas_price + .with_context(|| format!("no effective gas price {hash:?}"))?; + let auction_external_prices = Postgres::get_auction_prices(ex, auction_id) + .await + .with_context(|| { + format!("no external prices for auction id {auction_id:?} and tx {hash:?}") + })?; + let external_prices = ExternalPrices::try_from_auction_prices( + self.eth.contracts().weth().address(), + auction_external_prices.clone(), + )?; + + tracing::debug!( + ?auction_id, + ?auction_external_prices, + ?external_prices, + "observations input" + ); + + // surplus and fees calculation + let domain_separator = self.eth.contracts().settlement_domain_separator(); + let order_uids = settlement.order_uids(domain_separator)?; + let order_fees = order_uids + .clone() + .into_iter() + .zip(Postgres::order_fees(ex, &order_uids).await?) + .collect::>(); + + let surplus = settlement.total_surplus(&external_prices); + let (fee, order_executions) = { + let all_fees = settlement.all_fees(&external_prices, &order_fees); + // total unsubsidized fee used for CIP20 rewards + let fee = all_fees + .iter() + .fold(0.into(), |acc, fees| acc + fees.native); + // executed fees for each order execution + let order_executions = all_fees + .into_iter() + .zip(order_fees.iter()) + .map(|(fee, (_, order_fee))| match order_fee { + // market orders have no surplus fee + Some(_) => (fee.order, 0.into()), + None => (fee.order, fee.sell), + }) + .collect(); + (fee, order_executions) + }; + + Ok(AuctionData { + surplus, + fee, + gas_used, + effective_gas_price, + order_executions, + }) + } + /// With solver driver colocation solvers are supposed to append the /// `auction_id` to the settlement calldata. This function tries to - /// recover that `auction_id`. This function only returns an error if - /// retrying the operation makes sense. If all went well and there - /// simply is no sensible `auction_id` `AuctionKind::Invalid` will be - /// returned. - async fn get_auction_kind(ex: &mut PgConnection, tx: &Transaction) -> Result { + /// recover that `auction_id`. It also indicates whether the auction + /// should be indexed with its metadata. (ie. if it comes from this + /// environment and not from a different instance of the autopilot, e.g. + /// running in barn/prod). This function only returns an error + /// if retrying the operation makes sense. + async fn recover_auction_id_from_calldata( + ex: &mut PgConnection, + tx: &Transaction, + ) -> Result { let tx_from = tx.from.context("tx is missing sender")?; - let metadata = match DecodedSettlement::new(&tx.input.0) { - Ok(settlement) => settlement.metadata, + let settlement = match DecodedSettlement::new(&tx.input.0) { + Ok(settlement) => settlement, Err(err) => { tracing::warn!( ?tx, ?err, "could not decode settlement tx, unclear which auction it belongs to" ); - return Ok(AuctionKind::Invalid); + return Ok(AuctionIdRecoveryStatus::InvalidCalldata); } }; - let auction_id = match metadata { + let auction_id = match settlement.metadata { Some(bytes) => i64::from_be_bytes(bytes.0), None => { tracing::warn!(?tx, "could not recover the auction_id from the calldata"); - return Ok(AuctionKind::Invalid); + return Ok(AuctionIdRecoveryStatus::InvalidCalldata); } }; let score = database::settlement_scores::fetch(ex, auction_id).await?; - match score { - None => { + let data_already_recorded = database::settlements::data_exists(ex, auction_id).await?; + match (score, data_already_recorded) { + (None, _) => { tracing::debug!( auction_id, "calldata claims to settle auction that has no competition" ); - Ok(AuctionKind::Invalid) + Ok(AuctionIdRecoveryStatus::DoNotAddAuctionData(auction_id)) } - Some(score) => { - if score.winner.0 != tx_from.0 { - tracing::warn!( - auction_id, - ?tx_from, - winner = ?score.winner, - "solution submitted by solver other than the winner" - ); - Ok(AuctionKind::Invalid) - } else { - Ok(AuctionKind::Valid { auction_id }) - } + (Some(score), _) if score.winner.0 != tx_from.0 => { + tracing::warn!( + auction_id, + ?tx_from, + winner = ?score.winner, + "solution submitted by solver other than the winner" + ); + Ok(AuctionIdRecoveryStatus::DoNotAddAuctionData(auction_id)) + } + (Some(_), true) => { + tracing::warn!( + auction_id, + "settlement data already recorded for this auction" + ); + Ok(AuctionIdRecoveryStatus::DoNotAddAuctionData(auction_id)) } + (Some(_), false) => Ok(AuctionIdRecoveryStatus::AddAuctionData( + auction_id, settlement, + )), } } } diff --git a/crates/database/src/settlements.rs b/crates/database/src/settlements.rs index 1b41b3f3a6..13ecc292ea 100644 --- a/crates/database/src/settlements.rs +++ b/crates/database/src/settlements.rs @@ -47,40 +47,45 @@ pub struct SettlementEvent { pub tx_hash: TransactionHash, } -#[derive(Debug, Clone, PartialEq, sqlx::Type)] -#[sqlx(type_name = "AuctionKind", rename_all = "lowercase")] -pub enum AuctionKind { - Valid, - Invalid, -} - pub async fn get_settlement_without_auction( ex: &mut PgConnection, + max_block_number: i64, ) -> Result, sqlx::Error> { const QUERY: &str = r#" SELECT block_number, log_index, tx_hash FROM settlements -WHERE auction_kind = 'unprocessed' +WHERE auction_id IS NULL +AND block_number <= $1 ORDER BY block_number ASC LIMIT 1 "#; - sqlx::query_as(QUERY).fetch_optional(ex).await + sqlx::query_as(QUERY) + .bind(max_block_number) + .fetch_optional(ex) + .await +} + +pub async fn data_exists(ex: &mut PgConnection, auction_id: i64) -> Result { + const QUERY: &str = r#"SELECT COUNT(*) FROM settlements WHERE auction_id = $1;"#; + let count: i64 = sqlx::query_scalar(QUERY) + .bind(auction_id) + .fetch_one(ex) + .await?; + Ok(count >= 1) } pub async fn update_settlement_auction( ex: &mut PgConnection, block_number: i64, log_index: i64, - auction_id: Option, - auction_kind: AuctionKind, + auction_id: i64, ) -> Result<(), sqlx::Error> { const QUERY: &str = r#" UPDATE settlements -SET auction_kind = $1, auction_id = $2 -WHERE block_number = $3 AND log_index = $4 +SET auction_id = $1 +WHERE block_number = $2 AND log_index = $3 ;"#; sqlx::query(QUERY) - .bind(auction_kind) .bind(auction_id) .bind(block_number) .bind(log_index) @@ -170,7 +175,7 @@ mod tests { .await .unwrap(); - let settlement = get_settlement_without_auction(&mut db) + let settlement = get_settlement_without_auction(&mut db, 0) .await .unwrap() .unwrap(); @@ -178,17 +183,11 @@ mod tests { assert_eq!(settlement.block_number, event.block_number); assert_eq!(settlement.log_index, event.log_index); - update_settlement_auction( - &mut db, - event.block_number, - event.log_index, - Some(1), - AuctionKind::Valid, - ) - .await - .unwrap(); + update_settlement_auction(&mut db, event.block_number, event.log_index, 1) + .await + .unwrap(); - let settlement = get_settlement_without_auction(&mut db).await.unwrap(); + let settlement = get_settlement_without_auction(&mut db, 0).await.unwrap(); assert!(settlement.is_none()); } diff --git a/crates/database/src/solver_competition.rs b/crates/database/src/solver_competition.rs index ce6f80e83f..def53d0bde 100644 --- a/crates/database/src/solver_competition.rs +++ b/crates/database/src/solver_competition.rs @@ -127,8 +127,7 @@ mod tests { &mut db, index.block_number, index.log_index, - Some(id), - crate::settlements::AuctionKind::Valid, + id, ) .await .unwrap(); diff --git a/crates/e2e/src/setup/colocation.rs b/crates/e2e/src/setup/colocation.rs index 3062f770fa..301060f0c2 100644 --- a/crates/e2e/src/setup/colocation.rs +++ b/crates/e2e/src/setup/colocation.rs @@ -1,8 +1,7 @@ use { crate::{nodes::NODE_HOST, setup::*}, - ethcontract::{H160, H256, U256}, + ethcontract::{H160, U256}, reqwest::Url, - shared::sources::uniswap_v2::UNISWAP_INIT, tokio::task::JoinHandle, }; @@ -112,7 +111,7 @@ mempool = "public" contracts.gp_settlement.address(), contracts.weth.address(), contracts.uniswap_v2_router.address(), - H256(UNISWAP_INIT), + contracts.default_pool_code(), )); let args = vec![ "driver".to_string(), diff --git a/crates/e2e/src/setup/deploy.rs b/crates/e2e/src/setup/deploy.rs index 2a0f6c4141..7abcf97a26 100644 --- a/crates/e2e/src/setup/deploy.rs +++ b/crates/e2e/src/setup/deploy.rs @@ -10,12 +10,13 @@ use { UniswapV2Router02, WETH9, }, - ethcontract::{Address, U256}, + ethcontract::{Address, H256, U256}, model::DomainSeparator, shared::ethrpc::Web3, }; pub struct Contracts { + pub chain_id: u64, pub balancer_vault: BalancerV2Vault, pub gp_settlement: GPv2Settlement, pub gp_authenticator: GPv2AllowListAuthentication, @@ -36,6 +37,9 @@ impl Contracts { let gp_settlement = GPv2Settlement::deployed(web3).await.unwrap(); Self { + chain_id: network_id + .parse() + .expect("Couldn't parse network ID to u64"), balancer_vault: BalancerV2Vault::deployed(web3).await.unwrap(), gp_authenticator: GPv2AllowListAuthentication::deployed(web3).await.unwrap(), uniswap_v2_factory: UniswapV2Factory::deployed(web3).await.unwrap(), @@ -138,6 +142,9 @@ impl Contracts { let hooks = deploy!(HooksTrampoline(gp_settlement.address())); Self { + chain_id: network_id + .parse() + .expect("Couldn't parse network ID to u64"), balancer_vault, gp_settlement, gp_authenticator, @@ -150,4 +157,11 @@ impl Contracts { hooks, } } + + pub fn default_pool_code(&self) -> H256 { + match self.chain_id { + 100 => H256(shared::sources::uniswap_v2::HONEYSWAP_INIT), + _ => H256(shared::sources::uniswap_v2::UNISWAP_INIT), + } + } } diff --git a/crates/e2e/src/setup/onchain_components.rs b/crates/e2e/src/setup/onchain_components.rs index d902d91a35..a52a776aee 100644 --- a/crates/e2e/src/setup/onchain_components.rs +++ b/crates/e2e/src/setup/onchain_components.rs @@ -475,6 +475,23 @@ impl OnchainComponents { .unwrap(); } + /// We will only index events when they are 64 blocks old so we don't have + /// to throw out indexed data on reorgs. + /// This function executes enough transactions to ensure that all events + /// before calling this function are old enough to be indexed. + pub async fn mint_blocks_past_reorg_threshold(&self) { + tracing::info!("mining blocks to get past the reorg safety threshold for indexing events"); + + let current_block = || async { self.web3.eth().block_number().await.unwrap() }; + let target = current_block().await + 65; + // Simply issuing n transactions may not result in n blocks being mined. + // Therefore we continually call `evm_mine` until we can confirm the block + // number increased the expected number of times. + while current_block().await <= target { + let _ = self.web3.transport().execute("evm_mine", vec![]).await; + } + } + pub async fn mint_block(&self) { tracing::info!("mining block"); self.web3 diff --git a/crates/e2e/src/setup/services.rs b/crates/e2e/src/setup/services.rs index a8892a4f1c..1c90ae4fe3 100644 --- a/crates/e2e/src/setup/services.rs +++ b/crates/e2e/src/setup/services.rs @@ -69,7 +69,7 @@ impl<'a> Services<'a> { format!( "--custom-univ2-baseline-sources={:?}|{:?}", self.contracts.uniswap_v2_router.address(), - H256(shared::sources::uniswap_v2::UNISWAP_INIT), + self.contracts.default_pool_code(), ), format!( "--settlement-contract-address={:?}", @@ -113,15 +113,10 @@ impl<'a> Services<'a> { pub async fn start_api(&self, extra_args: Vec) { let args = [ "orderbook".to_string(), - "--enable-presign-orders=true".to_string(), - "--enable-eip1271-orders=true".to_string(), - "--enable-custom-interactions=true".to_string(), - "--allow-placing-partially-fillable-limit-orders=true".to_string(), format!( "--hooks-contract-address={:?}", self.contracts.hooks.address() ), - "--enable-eth-smart-contract-payments=true".to_string(), ] .into_iter() .chain(self.api_autopilot_solver_arguments()) diff --git a/crates/e2e/tests/e2e/app_data.rs b/crates/e2e/tests/e2e/app_data.rs index 26bfd0679a..43c6f09151 100644 --- a/crates/e2e/tests/e2e/app_data.rs +++ b/crates/e2e/tests/e2e/app_data.rs @@ -4,6 +4,7 @@ use { model::{ app_data::AppDataHash, order::{OrderCreation, OrderCreationAppData, OrderKind}, + quote::{OrderQuoteRequest, OrderQuoteSide, SellAmount}, signature::EcdsaSigningScheme, }, reqwest::StatusCode, @@ -101,6 +102,23 @@ async fn app_data(web3: Web3) { let order3 = create_order(OrderCreationAppData::Hash { hash: app_data_hash, }); + services + .submit_quote(&OrderQuoteRequest { + sell_token: order3.sell_token, + buy_token: order3.buy_token, + side: OrderQuoteSide::Sell { + sell_amount: SellAmount::AfterFee { + value: order3.sell_amount.try_into().unwrap(), + }, + }, + app_data: OrderCreationAppData::Hash { + hash: app_data_hash, + }, + ..Default::default() + }) + .await + .unwrap(); + let uid = services.create_order(&order3).await.unwrap(); let order3_ = services.get_order(&uid).await.unwrap(); assert_eq!(order3_.data.app_data, app_data_hash); diff --git a/crates/e2e/tests/e2e/database.rs b/crates/e2e/tests/e2e/database.rs index 58a89c00ec..d18364e146 100644 --- a/crates/e2e/tests/e2e/database.rs +++ b/crates/e2e/tests/e2e/database.rs @@ -21,6 +21,8 @@ pub async fn events_of_order(db: &Db, uid: &OrderUid) -> Vec Option { let mut db = db.acquire().await.unwrap(); - const LAST_AUCTION_ID: &str = "SELECT auction_id FROM settlements WHERE auction_id IS NOT \ - NULL ORDER BY auction_id DESC LIMIT 1"; + const LAST_AUCTION_ID: &str = + "SELECT auction_id FROM auction_transaction ORDER BY auction_id DESC LIMIT 1"; let auction_id: i64 = sqlx::query_scalar(LAST_AUCTION_ID) .fetch_optional(db.deref_mut()) .await .unwrap()?; const TX_QUERY: &str = r" -SELECT * FROM settlements WHERE auction_id = $1"; - +SELECT s.* +FROM auction_transaction at +JOIN settlements s ON s.tx_from = at.tx_from AND s.tx_nonce = at.tx_nonce +WHERE at.auction_id = $1 + "; let tx: AuctionTransaction = sqlx::query_as(TX_QUERY) .bind(auction_id) .fetch_optional(db.deref_mut()) diff --git a/crates/e2e/tests/e2e/limit_orders.rs b/crates/e2e/tests/e2e/limit_orders.rs index 8075e84576..b212215354 100644 --- a/crates/e2e/tests/e2e/limit_orders.rs +++ b/crates/e2e/tests/e2e/limit_orders.rs @@ -36,19 +36,37 @@ async fn local_node_mixed_limit_and_market_orders() { } /// The block number from which we will fetch state for the forked tests. -pub const FORK_BLOCK: u64 = 18477910; -/// USDC whale address as per [FORK_BLOCK]. -pub const USDC_WHALE: H160 = H160(hex_literal::hex!( +const FORK_BLOCK_MAINNET: u64 = 18477910; +/// USDC whale address as per [FORK_BLOCK_MAINNET]. +const USDC_WHALE_MAINNET: H160 = H160(hex_literal::hex!( "28c6c06298d514db089934071355e5743bf21d60" )); #[tokio::test] #[ignore] -async fn forked_node_single_limit_order_mainnet() { +async fn forked_node_mainnet_single_limit_order() { run_forked_test_with_block_number( - forked_single_limit_order_test, - std::env::var("FORK_URL").expect("FORK_URL must be set to run forked tests"), - FORK_BLOCK, + forked_mainnet_single_limit_order_test, + std::env::var("FORK_URL_MAINNET") + .expect("FORK_URL_MAINNET must be set to run forked tests"), + FORK_BLOCK_MAINNET, + ) + .await; +} + +const FORK_BLOCK_GNOSIS: u64 = 32070725; +/// USDC whale address as per [FORK_BLOCK_GNOSIS]. +const USDC_WHALE_GNOSIS: H160 = H160(hex_literal::hex!( + "ba12222222228d8ba445958a75a0704d566bf2c8" +)); + +#[tokio::test] +#[ignore] +async fn forked_node_gnosis_single_limit_order() { + run_forked_test_with_block_number( + forked_gnosis_single_limit_order_test, + std::env::var("FORK_URL_GNOSIS").expect("FORK_URL_GNOSIS must be set to run forked tests"), + FORK_BLOCK_GNOSIS, ) .await; } @@ -455,7 +473,7 @@ async fn too_many_limit_orders_test(web3: Web3) { assert!(body.contains("TooManyLimitOrders")); } -async fn forked_single_limit_order_test(web3: Web3) { +async fn forked_mainnet_single_limit_order_test(web3: Web3) { let mut onchain = OnchainComponents::deployed(web3.clone()).await; let forked_node_api = web3.api::>(); @@ -478,7 +496,10 @@ async fn forked_single_limit_order_test(web3: Web3) { ); // Give trader some USDC - let usdc_whale = forked_node_api.impersonate(&USDC_WHALE).await.unwrap(); + let usdc_whale = forked_node_api + .impersonate(&USDC_WHALE_MAINNET) + .await + .unwrap(); tx!( usdc_whale, token_usdc.transfer(trader.address(), to_wei_with_exp(1000, 6)) @@ -547,3 +568,99 @@ async fn forked_single_limit_order_test(web3: Web3) { assert!(sell_token_balance_before > sell_token_balance_after); assert!(buy_token_balance_after >= buy_token_balance_before + to_wei_with_exp(500, 6)); } + +async fn forked_gnosis_single_limit_order_test(web3: Web3) { + let mut onchain = OnchainComponents::deployed(web3.clone()).await; + let forked_node_api = web3.api::>(); + + let [solver] = onchain.make_solvers_forked(to_wei(1)).await; + + let [trader] = onchain.make_accounts(to_wei(1)).await; + + let token_usdc = ERC20::at( + &web3, + "0xddafbb505ad214d7b80b1f830fccc89b60fb7a83" + .parse() + .unwrap(), + ); + + let token_wxdai = ERC20::at( + &web3, + "0xe91d153e0b41518a2ce8dd3d7944fa863463a97d" + .parse() + .unwrap(), + ); + + // Give trader some USDC + let usdc_whale = forked_node_api + .impersonate(&USDC_WHALE_GNOSIS) + .await + .unwrap(); + tx!( + usdc_whale, + token_usdc.transfer(trader.address(), to_wei_with_exp(1000, 6)) + ); + + // Approve GPv2 for trading + tx!( + trader.account(), + token_usdc.approve(onchain.contracts().allowance, to_wei_with_exp(1000, 6)) + ); + + // Place Orders + let services = Services::new(onchain.contracts()).await; + services.start_protocol(solver).await; + + let order = OrderCreation { + sell_token: token_usdc.address(), + sell_amount: to_wei_with_exp(1000, 6), + buy_token: token_wxdai.address(), + buy_amount: to_wei(500), + valid_to: model::time::now_in_epoch_seconds() + 300, + kind: OrderKind::Sell, + ..Default::default() + } + .sign( + EcdsaSigningScheme::Eip712, + &onchain.contracts().domain_separator, + SecretKeyRef::from(&SecretKey::from_slice(trader.private_key()).unwrap()), + ); + let order_id = services.create_order(&order).await.unwrap(); + let limit_order = services.get_order(&order_id).await.unwrap(); + assert_eq!(limit_order.metadata.class, OrderClass::Limit); + + // Drive solution + tracing::info!("Waiting for trade."); + let sell_token_balance_before = token_usdc + .balance_of(trader.address()) + .call() + .await + .unwrap(); + let buy_token_balance_before = token_wxdai + .balance_of(trader.address()) + .call() + .await + .unwrap(); + + wait_for_condition(TIMEOUT, || async { services.solvable_orders().await == 1 }) + .await + .unwrap(); + + wait_for_condition(TIMEOUT, || async { services.solvable_orders().await == 0 }) + .await + .unwrap(); + + let sell_token_balance_after = token_usdc + .balance_of(trader.address()) + .call() + .await + .unwrap(); + let buy_token_balance_after = token_wxdai + .balance_of(trader.address()) + .call() + .await + .unwrap(); + + assert!(sell_token_balance_before > sell_token_balance_after); + assert!(buy_token_balance_after >= buy_token_balance_before + to_wei(500)); +} diff --git a/crates/e2e/tests/e2e/partial_fill.rs b/crates/e2e/tests/e2e/partial_fill.rs index 8b17b27093..58c396fee2 100644 --- a/crates/e2e/tests/e2e/partial_fill.rs +++ b/crates/e2e/tests/e2e/partial_fill.rs @@ -89,7 +89,10 @@ async fn test(web3: Web3) { .contains(&buy_balance.as_u128()) ); + onchain.mint_blocks_past_reorg_threshold().await; + let settlement_event_processed = || async { + onchain.mint_block().await; let order = services.get_order(&uid).await.unwrap(); order.metadata.executed_surplus_fee > U256::zero() }; diff --git a/crates/e2e/tests/e2e/partially_fillable_pool.rs b/crates/e2e/tests/e2e/partially_fillable_pool.rs index 04c77d75bd..ccc78a87d6 100644 --- a/crates/e2e/tests/e2e/partially_fillable_pool.rs +++ b/crates/e2e/tests/e2e/partially_fillable_pool.rs @@ -117,7 +117,9 @@ async fn test(web3: Web3) { .contains(&buy_balance.as_u128()) ); + onchain.mint_blocks_past_reorg_threshold().await; let metadata_updated = || async { + onchain.mint_block().await; let order = services.get_order(&uid).await.unwrap(); !order.metadata.executed_surplus_fee.is_zero() && order.metadata.executed_buy_amount != Default::default() diff --git a/crates/e2e/tests/e2e/protocol_fee.rs b/crates/e2e/tests/e2e/protocol_fee.rs index 3d2e240bc2..14b8da57be 100644 --- a/crates/e2e/tests/e2e/protocol_fee.rs +++ b/crates/e2e/tests/e2e/protocol_fee.rs @@ -328,7 +328,9 @@ async fn execute_test( .await .unwrap(); + onchain.mint_blocks_past_reorg_threshold().await; let metadata_updated = || async { + onchain.mint_block().await; let order = services.get_order(&uid).await.unwrap(); is_approximately_equal(order.metadata.executed_surplus_fee, expected_surplus_fee) }; diff --git a/crates/e2e/tests/e2e/solver_competition.rs b/crates/e2e/tests/e2e/solver_competition.rs index e735e09050..b205c6ef67 100644 --- a/crates/e2e/tests/e2e/solver_competition.rs +++ b/crates/e2e/tests/e2e/solver_competition.rs @@ -90,6 +90,8 @@ async fn solver_competition(web3: Web3) { || async { token_a.balance_of(trader.address()).call().await.unwrap() == U256::zero() }; wait_for_condition(TIMEOUT, trade_happened).await.unwrap(); + onchain.mint_blocks_past_reorg_threshold().await; + let indexed_trades = || async { onchain.mint_block().await; match services.get_trades(&uid).await.unwrap().first() { diff --git a/crates/e2e/tests/e2e/univ2.rs b/crates/e2e/tests/e2e/univ2.rs index 6fab8e4abf..58335da635 100644 --- a/crates/e2e/tests/e2e/univ2.rs +++ b/crates/e2e/tests/e2e/univ2.rs @@ -89,6 +89,8 @@ async fn test(web3: Web3) { .await .unwrap(); + onchain.mint_blocks_past_reorg_threshold().await; + let cip_20_data_updated = || async { let data = match crate::database::most_recent_cip_20_data(services.db()).await { Some(data) => data, @@ -102,6 +104,8 @@ async fn test(web3: Web3) { && data.participants.iter().any(|p| p.participant.0 == solver.address().0) // and won the auction && data.score.winner.0 == solver.address().0 + // and submitted the solution + && data.tx.tx_from.0 == solver.address().0 // and calldata is present && !data.call_data.call_data.is_empty() }; diff --git a/database/sql/V058__add_auction_id_to_settlements.sql b/database/sql/V058__add_auction_id_to_settlements.sql new file mode 100644 index 0000000000..c1a3a4a23c --- /dev/null +++ b/database/sql/V058__add_auction_id_to_settlements.sql @@ -0,0 +1,12 @@ +-- Step 1: Add the new column to the settlements table +ALTER TABLE settlements + ADD COLUMN auction_id bigint; + +-- Step 2: Populate auction_id columns +-- For all settlements that already have an auction_transaction record, set auction_id to the auction_transaction's auction_id +UPDATE settlements +SET auction_id = auction_transaction.auction_id +FROM auction_transaction +WHERE settlements.tx_from = auction_transaction.tx_from AND settlements.tx_nonce = auction_transaction.tx_nonce; + +-- Step 3: (Once migration has been successful) Drop the auction_transaction table, and the tx_from and tx_nonce columns from the settlements table diff --git a/database/sql/V058__drop_auction_transaction.sql b/database/sql/V058__drop_auction_transaction.sql deleted file mode 100644 index a42d6fcecb..0000000000 --- a/database/sql/V058__drop_auction_transaction.sql +++ /dev/null @@ -1,39 +0,0 @@ --- Step 1: Add the new columns to the settlements table -CREATE TYPE AuctionKind AS ENUM ('valid', 'invalid', 'unprocessed'); - -ALTER TABLE settlements - ADD COLUMN auction_kind AuctionKind NOT NULL DEFAULT 'unprocessed', - ADD COLUMN auction_id bigint; - --- Step 2a: Populate auction_kind and auction_id columns --- For all settlements that already have an auction_transaction record, set auction_kind to 'valid' and auction_id to the auction_transaction's auction_id -UPDATE settlements -SET auction_kind = 'valid', - auction_id = auction_transaction.auction_id -FROM auction_transaction -WHERE settlements.tx_from = auction_transaction.tx_from AND settlements.tx_nonce = auction_transaction.tx_nonce; - --- Step 2b: Populate auction_kind and auction_id columns --- For all settlements that have auction_id = NULL, set auction_kind to 'invalid' -UPDATE settlements -SET auction_kind = 'invalid' -WHERE auction_id IS NULL; - --- Step 2c: Populate auction_kind and auction_id columns --- For all settlements, going from the most recent to the oldest, set auction_kind to 'unprocessed' until the first settlement with auction_kind = 'valid' is found -UPDATE settlements -SET - auction_kind = 'unprocessed' -WHERE - auction_kind = 'invalid' - AND block_number > ( - SELECT MAX(block_number) - FROM settlements - WHERE auction_kind = 'valid' - ); - --- Step 3: Drop the auction_transaction table, and the tx_from and tx_nonce columns from the settlements table -DROP TABLE auction_transaction; -ALTER TABLE settlements - DROP COLUMN tx_from, - DROP COLUMN tx_nonce; From bae70f53f051c8690b6150bbd7d37f9e3b4da4ca Mon Sep 17 00:00:00 2001 From: Felix Leupold Date: Wed, 24 Jan 2024 18:15:17 +0100 Subject: [PATCH 21/23] fix e2e test --- crates/e2e/tests/e2e/database.rs | 15 +++++---------- crates/e2e/tests/e2e/univ2.rs | 2 -- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/crates/e2e/tests/e2e/database.rs b/crates/e2e/tests/e2e/database.rs index d18364e146..58a89c00ec 100644 --- a/crates/e2e/tests/e2e/database.rs +++ b/crates/e2e/tests/e2e/database.rs @@ -21,8 +21,6 @@ pub async fn events_of_order(db: &Db, uid: &OrderUid) -> Vec Option { let mut db = db.acquire().await.unwrap(); - const LAST_AUCTION_ID: &str = - "SELECT auction_id FROM auction_transaction ORDER BY auction_id DESC LIMIT 1"; + const LAST_AUCTION_ID: &str = "SELECT auction_id FROM settlements WHERE auction_id IS NOT \ + NULL ORDER BY auction_id DESC LIMIT 1"; let auction_id: i64 = sqlx::query_scalar(LAST_AUCTION_ID) .fetch_optional(db.deref_mut()) .await .unwrap()?; const TX_QUERY: &str = r" -SELECT s.* -FROM auction_transaction at -JOIN settlements s ON s.tx_from = at.tx_from AND s.tx_nonce = at.tx_nonce -WHERE at.auction_id = $1 - "; +SELECT * FROM settlements WHERE auction_id = $1"; + let tx: AuctionTransaction = sqlx::query_as(TX_QUERY) .bind(auction_id) .fetch_optional(db.deref_mut()) diff --git a/crates/e2e/tests/e2e/univ2.rs b/crates/e2e/tests/e2e/univ2.rs index 58335da635..bbb56b3a8b 100644 --- a/crates/e2e/tests/e2e/univ2.rs +++ b/crates/e2e/tests/e2e/univ2.rs @@ -104,8 +104,6 @@ async fn test(web3: Web3) { && data.participants.iter().any(|p| p.participant.0 == solver.address().0) // and won the auction && data.score.winner.0 == solver.address().0 - // and submitted the solution - && data.tx.tx_from.0 == solver.address().0 // and calldata is present && !data.call_data.call_data.is_empty() }; From 5e4401e064ae5d6c0e446043349da90c71ac27f6 Mon Sep 17 00:00:00 2001 From: Felix Leupold Date: Wed, 24 Jan 2024 18:18:03 +0100 Subject: [PATCH 22/23] rename data_exists --- crates/autopilot/src/on_settlement_event_updater.rs | 3 ++- crates/database/src/settlements.rs | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/crates/autopilot/src/on_settlement_event_updater.rs b/crates/autopilot/src/on_settlement_event_updater.rs index 643ee1b4ef..ccdccc978c 100644 --- a/crates/autopilot/src/on_settlement_event_updater.rs +++ b/crates/autopilot/src/on_settlement_event_updater.rs @@ -259,7 +259,8 @@ impl OnSettlementEventUpdater { }; let score = database::settlement_scores::fetch(ex, auction_id).await?; - let data_already_recorded = database::settlements::data_exists(ex, auction_id).await?; + let data_already_recorded = + database::settlements::already_processed(ex, auction_id).await?; match (score, data_already_recorded) { (None, _) => { tracing::debug!( diff --git a/crates/database/src/settlements.rs b/crates/database/src/settlements.rs index 13ecc292ea..409ce64fa5 100644 --- a/crates/database/src/settlements.rs +++ b/crates/database/src/settlements.rs @@ -65,7 +65,10 @@ LIMIT 1 .await } -pub async fn data_exists(ex: &mut PgConnection, auction_id: i64) -> Result { +pub async fn already_processed( + ex: &mut PgConnection, + auction_id: i64, +) -> Result { const QUERY: &str = r#"SELECT COUNT(*) FROM settlements WHERE auction_id = $1;"#; let count: i64 = sqlx::query_scalar(QUERY) .bind(auction_id) From 40138bfa4946685d5164951449b0f7932e06a3b1 Mon Sep 17 00:00:00 2001 From: Felix Leupold Date: Thu, 25 Jan 2024 13:30:51 +0100 Subject: [PATCH 23/23] set default auciton id for old settlements from other environment --- database/sql/V058__add_auction_id_to_settlements.sql | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/database/sql/V058__add_auction_id_to_settlements.sql b/database/sql/V058__add_auction_id_to_settlements.sql index c1a3a4a23c..2278aa1d28 100644 --- a/database/sql/V058__add_auction_id_to_settlements.sql +++ b/database/sql/V058__add_auction_id_to_settlements.sql @@ -3,6 +3,13 @@ ALTER TABLE settlements ADD COLUMN auction_id bigint; -- Step 2: Populate auction_id columns + +-- For all settlements that are not potentially still waiting to get indexed, set the auction ID to a default value of 0 +-- Using 1000 blocks here to leave some margin for possible settlement updates that are still inflight. +UPDATE settlements +SET auction_id = 0 +WHERE block_number < (SELECT max(block_number) FROM settlements) - 1000; + -- For all settlements that already have an auction_transaction record, set auction_id to the auction_transaction's auction_id UPDATE settlements SET auction_id = auction_transaction.auction_id