From e6a90c4a735e7a5d229aa05b010cf18a7432618f Mon Sep 17 00:00:00 2001 From: Szegoo Date: Mon, 5 Feb 2024 11:08:28 +0100 Subject: [PATCH 1/4] grouping --- Cargo.lock | 11 ++++-- routes/Cargo.toml | 1 + routes/src/consumption.rs | 78 +++++++++++++++++++++++++++++++++++++-- routes/tests/mock.rs | 45 +++++++++++++++++++++- types/src/lib.rs | 2 +- 5 files changed, 127 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 200f68b..7edd08f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -922,14 +922,16 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.31" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +checksum = "9f13690e35a5e4ace198e7beea2895d29f3a9cc55015fcebe6336bd2010af9eb" dependencies = [ "android-tzdata", "iana-time-zone", + "js-sys", "num-traits", - "windows-targets 0.48.5", + "wasm-bindgen", + "windows-targets 0.52.0", ] [[package]] @@ -2087,7 +2089,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.10", + "socket2 0.5.5", "tokio", "tower-service", "tracing", @@ -3424,6 +3426,7 @@ dependencies = [ name = "routes" version = "0.1.0" dependencies = [ + "chrono", "log", "maplit", "polkadot-core-primitives", diff --git a/routes/Cargo.toml b/routes/Cargo.toml index e65439d..ef528cd 100644 --- a/routes/Cargo.toml +++ b/routes/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" [dependencies] log = "0.4" +chrono = "0.4.33" rocket = { version = "0.5.0", features=["json"] } rocket_cors = "0.6.0" serde = "1.0.193" diff --git a/routes/src/consumption.rs b/routes/src/consumption.rs index 5072249..fe6008b 100644 --- a/routes/src/consumption.rs +++ b/routes/src/consumption.rs @@ -14,14 +14,51 @@ // along with RegionX. If not, see . use crate::Error; -use rocket::get; +use chrono::NaiveDateTime; +use rocket::{ + form, + form::{FromFormField, ValueField}, + get, +}; use shared::{consumption::get_consumption, registry::registered_para}; -use types::{ParaId, Timestamp, WeightConsumption}; +use std::collections::HashMap; +use types::{DispatchClassConsumption, ParaId, Timestamp, WeightConsumption}; + +#[derive(Clone, Debug, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(crate = "rocket::serde")] +pub enum Grouping { + BlockNumber, + Day, + Month, + Year, +} + +#[rocket::async_trait] +impl<'r> FromFormField<'r> for Grouping { + fn from_value(field: ValueField<'r>) -> form::Result<'r, Self> { + match field.value { + "day" => Ok(Grouping::Day), + "month" => Ok(Grouping::Month), + "year" => Ok(Grouping::Year), + _ => Err(form::Error::validation("invalid Grouping").into()), + } + } +} + +#[derive(Default, Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)] +#[serde(crate = "rocket::serde")] +pub struct AggregatedData { + /// The aggregated ref_time consumption over all the dispatch classes. + pub ref_time: DispatchClassConsumption, + /// The aggregated proof size over all dispatch classes. + pub proof_size: DispatchClassConsumption, + pub count: usize, +} /// Query the consumption data of a parachain. /// /// This will return an error in case there is no data associated with the specific parachain. -#[get("/consumption//?&&&")] +#[get("/consumption//?&&&&")] pub fn consumption( relay: &str, para_id: ParaId, @@ -29,6 +66,7 @@ pub fn consumption( end: Option, page: Option, page_size: Option, + grouping: Option, ) -> Result { let para = registered_para(relay.into(), para_id).ok_or(Error::NotRegistered)?; @@ -44,5 +82,37 @@ pub fn consumption( .take(page_size as usize) .collect(); - serde_json::to_string(&weight_consumptions).map_err(|_| Error::InvalidData) + let grouping = grouping.unwrap_or(Grouping::BlockNumber); + + let grouped: HashMap = + weight_consumptions.iter().fold(HashMap::new(), |mut acc, datum| { + let key = get_aggregation_key(datum.clone(), grouping); + let entry = acc.entry(key).or_insert_with(Default::default); + + entry.ref_time.normal += datum.ref_time.normal; + entry.ref_time.operational += datum.ref_time.operational; + entry.ref_time.mandatory += datum.ref_time.mandatory; + + entry.proof_size.normal += datum.proof_size.normal; + entry.proof_size.operational += datum.proof_size.operational; + entry.proof_size.mandatory += datum.proof_size.mandatory; + + entry.count += 1; + + acc + }); + + serde_json::to_string(&grouped).map_err(|_| Error::InvalidData) +} + +fn get_aggregation_key(datum: WeightConsumption, grouping: Grouping) -> String { + let datetime = + NaiveDateTime::from_timestamp_opt((datum.timestamp / 1000) as i64, 0).unwrap_or_default(); + + match grouping { + Grouping::BlockNumber => datum.block_number.to_string(), + Grouping::Day => datetime.format("%Y-%m-%d").to_string(), + Grouping::Month => datetime.format("%Y-%m").to_string(), + Grouping::Year => datetime.format("%Y").to_string(), + } } diff --git a/routes/tests/mock.rs b/routes/tests/mock.rs index fb129ce..1e5382a 100644 --- a/routes/tests/mock.rs +++ b/routes/tests/mock.rs @@ -15,13 +15,14 @@ #[cfg(test)] use maplit::hashmap; +use routes::consumption::{AggregatedData, Grouping}; use scopeguard::guard; use shared::{ chaindata::get_para, consumption::write_consumption, registry::update_registry, reset_mock_environment, }; use std::collections::HashMap; -use types::{Parachain, RelayChain::*, WeightConsumption}; +use types::{ParaId, Parachain, RelayChain, RelayChain::*, WeightConsumption}; #[derive(Default)] pub struct MockEnvironment { @@ -96,3 +97,45 @@ pub fn mock_consumption() -> HashMap> { ], } } + +pub fn aggregated_mock_consumption( + para: (RelayChain, ParaId), + grouping: Grouping, +) -> HashMap { + match para { + (Polkadot, 2000) => { + hashmap! { + "1".to_string() => AggregatedData { + ref_time: (0.5, 0.3, 0.2).into(), + proof_size: (0.5, 0.3, 0.2).into(), + count: 1, + }, + "2".to_string() => AggregatedData { + ref_time: (0.1, 0.4, 0.2).into(), + proof_size: (0.2, 0.3, 0.3).into(), + count: 1, + }, + "3".to_string() => AggregatedData { + ref_time: (0.0, 0.2, 0.4).into(), + proof_size: (0.1, 0.0, 0.3).into(), + count: 1, + }, + "4".to_string() => AggregatedData { + ref_time: (0.1, 0.0, 0.4).into(), + proof_size: (0.2, 0.1, 0.3).into(), + count: 1, + } + } + }, + (Polkadot, 2004) => { + hashmap! { + "1".to_string() => AggregatedData { + ref_time: (0.8, 0.0, 0.1).into(), + proof_size: (0.6, 0.2, 0.1).into(), + count: 1, + } + } + }, + _ => panic!("No consumption data"), + } +} diff --git a/types/src/lib.rs b/types/src/lib.rs index 334e75a..4e8dea3 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -94,7 +94,7 @@ pub struct WeightConsumption { pub proof_size: DispatchClassConsumption, } -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] +#[derive(Default, Debug, Serialize, PartialEq, Deserialize, Clone)] pub struct DispatchClassConsumption { /// The percentage of the weight used by user submitted extrinsics compared to the /// maximum potential. From 469d5a127a2008dd32e507d4cf56b7782c801500 Mon Sep 17 00:00:00 2001 From: Szegoo Date: Mon, 5 Feb 2024 11:22:43 +0100 Subject: [PATCH 2/4] fix tests --- routes/mock-parachains.json | 27 +++++++++++- routes/src/consumption.rs | 34 ++++++++------- routes/tests/consumption.rs | 82 ++++++++++++++++++++++++------------- routes/tests/mock.rs | 42 ------------------- 4 files changed, 99 insertions(+), 86 deletions(-) diff --git a/routes/mock-parachains.json b/routes/mock-parachains.json index 0637a08..c9d718d 100644 --- a/routes/mock-parachains.json +++ b/routes/mock-parachains.json @@ -1 +1,26 @@ -[] \ No newline at end of file +[ + { + "name": "Moonbeam", + "rpcs": [ + "wss://moonbeam-rpc.dwellir.com", + "wss://1rpc.io/glmr", + "wss://wss.api.moonbeam.network", + "wss://moonbeam.unitedbloc.com" + ], + "para_id": 2004, + "relay_chain": "Polkadot", + "last_payment_timestamp": 0 + }, + { + "name": "Acala", + "rpcs": [ + "wss://acala-rpc.dwellir.com", + "wss://acala-rpc-0.aca-api.network", + "wss://acala-rpc-1.aca-api.network", + "wss://acala-rpc-3.aca-api.network/ws" + ], + "para_id": 2000, + "relay_chain": "Polkadot", + "last_payment_timestamp": 0 + } +] \ No newline at end of file diff --git a/routes/src/consumption.rs b/routes/src/consumption.rs index fe6008b..6f549ee 100644 --- a/routes/src/consumption.rs +++ b/routes/src/consumption.rs @@ -84,25 +84,31 @@ pub fn consumption( let grouping = grouping.unwrap_or(Grouping::BlockNumber); - let grouped: HashMap = - weight_consumptions.iter().fold(HashMap::new(), |mut acc, datum| { - let key = get_aggregation_key(datum.clone(), grouping); - let entry = acc.entry(key).or_insert_with(Default::default); + let grouped: HashMap = group_consumption(weight_consumptions, grouping); - entry.ref_time.normal += datum.ref_time.normal; - entry.ref_time.operational += datum.ref_time.operational; - entry.ref_time.mandatory += datum.ref_time.mandatory; + serde_json::to_string(&grouped).map_err(|_| Error::InvalidData) +} - entry.proof_size.normal += datum.proof_size.normal; - entry.proof_size.operational += datum.proof_size.operational; - entry.proof_size.mandatory += datum.proof_size.mandatory; +pub fn group_consumption( + weight_consumptions: Vec, + grouping: Grouping, +) -> HashMap { + weight_consumptions.iter().fold(HashMap::new(), |mut acc, datum| { + let key = get_aggregation_key(datum.clone(), grouping); + let entry = acc.entry(key).or_insert_with(Default::default); - entry.count += 1; + entry.ref_time.normal += datum.ref_time.normal; + entry.ref_time.operational += datum.ref_time.operational; + entry.ref_time.mandatory += datum.ref_time.mandatory; - acc - }); + entry.proof_size.normal += datum.proof_size.normal; + entry.proof_size.operational += datum.proof_size.operational; + entry.proof_size.mandatory += datum.proof_size.mandatory; - serde_json::to_string(&grouped).map_err(|_| Error::InvalidData) + entry.count += 1; + + acc + }) } fn get_aggregation_key(datum: WeightConsumption, grouping: Grouping) -> String { diff --git a/routes/tests/consumption.rs b/routes/tests/consumption.rs index 9fc61fb..6048d3c 100644 --- a/routes/tests/consumption.rs +++ b/routes/tests/consumption.rs @@ -18,8 +18,12 @@ use rocket::{ local::blocking::{Client, LocalResponse}, routes, }; -use routes::{consumption::consumption, Error}; +use routes::{ + consumption::{consumption, group_consumption, AggregatedData, Grouping}, + Error, +}; use shared::{chaindata::get_para, registry::update_registry, reset_mock_environment}; +use std::collections::HashMap; use types::{RelayChain::*, WeightConsumption}; mod mock; @@ -36,7 +40,11 @@ fn getting_all_consumption_data_works() { assert_eq!(response.status(), Status::Ok); let consumption_data = parse_ok_response(response); - assert_eq!(consumption_data, mock_consumption().get(¶).unwrap().clone()); + let expected_consumption = group_consumption( + mock_consumption().get(¶).unwrap().clone(), + Grouping::BlockNumber, + ); + assert_eq!(consumption_data, expected_consumption); }); } @@ -88,27 +96,31 @@ fn pagination_works() { assert_eq!(response.status(), Status::Ok); let consumption_data = parse_ok_response(response); + let expected_data = + group_consumption(vec![mock_data.first().unwrap().clone()], Grouping::BlockNumber); // Should only contain the first consumption data. - assert_eq!(consumption_data, vec![mock_data.first().unwrap().clone()]); + assert_eq!(consumption_data, expected_data); // CASE 2: Specifying the page without page size will still show all the data. let response = client.get("/consumption/polkadot/2000?page=0").dispatch(); assert_eq!(response.status(), Status::Ok); let consumption_data = parse_ok_response(response); + let expected_data = group_consumption(mock_data.clone(), Grouping::BlockNumber); // Should only contain the first consumption data. - assert_eq!(consumption_data, mock_data); + assert_eq!(consumption_data, expected_data); // CASE 3: Specifying the page and page size works. let response = client.get("/consumption/polkadot/2000?page=1&page_size=2").dispatch(); assert_eq!(response.status(), Status::Ok); let consumption_data = parse_ok_response(response); - // Should skip the first page and take the second one. - assert_eq!( - consumption_data, - mock_data.into_iter().skip(2).take(2).collect::>() + let expected_data = group_consumption( + mock_data.into_iter().skip(2).take(2).collect::>(), + Grouping::BlockNumber, ); + // Should skip the first page and take the second one. + assert_eq!(consumption_data, expected_data); // CASE 4: An out-of-bound page and page size will return an empty vector. let response = client.get("/consumption/polkadot/2000?page=69&page_size=42").dispatch(); @@ -134,11 +146,14 @@ fn timestamp_based_filtering_works() { assert_eq!(response.status(), Status::Ok); let response_data = parse_ok_response(response); - let expected_data = mock_data - .clone() - .into_iter() - .filter(|c| c.timestamp >= start_timestamp) - .collect::>(); + let expected_data = group_consumption( + mock_data + .clone() + .into_iter() + .filter(|c| c.timestamp >= start_timestamp) + .collect::>(), + Grouping::BlockNumber, + ); // Should only contain the consumption where the timestamp is greater than or equal to 6. assert_eq!(response_data, expected_data); @@ -149,11 +164,14 @@ fn timestamp_based_filtering_works() { assert_eq!(response.status(), Status::Ok); let response_data = parse_ok_response(response); - let expected_data = mock_data - .clone() - .into_iter() - .filter(|c| c.timestamp <= end_timestamp) - .collect::>(); + let expected_data = group_consumption( + mock_data + .clone() + .into_iter() + .filter(|c| c.timestamp <= end_timestamp) + .collect::>(), + Grouping::BlockNumber, + ); // Should only contain the consumption where the timestamp is less than or equal to 12. assert_eq!(response_data, expected_data); @@ -165,10 +183,13 @@ fn timestamp_based_filtering_works() { assert_eq!(response.status(), Status::Ok); let response_data = parse_ok_response(response); - let expected_data = mock_data - .into_iter() - .filter(|c| c.timestamp >= start_timestamp && c.timestamp <= end_timestamp) - .collect::>(); + let expected_data = group_consumption( + mock_data + .into_iter() + .filter(|c| c.timestamp >= start_timestamp && c.timestamp <= end_timestamp) + .collect::>(), + Grouping::BlockNumber, + ); assert_eq!(response_data, expected_data); // Should only contain one consumption data since the `start` and `end` are set to the same @@ -199,19 +220,22 @@ fn pagination_and_timestamp_filtering_works() { assert_eq!(response.status(), Status::Ok); let response_data = parse_ok_response(response); - let expected_data = mock_data - .into_iter() - .filter(|c| c.timestamp >= start_timestamp) - .skip(page_size * page_number) - .take(page_size) - .collect::>(); + let expected_data = group_consumption( + mock_data + .into_iter() + .filter(|c| c.timestamp >= start_timestamp) + .skip(page_size * page_number) + .take(page_size) + .collect::>(), + Grouping::BlockNumber, + ); // Check if the data is filtered by timestamp and then paginated assert_eq!(response_data, expected_data); }); } -fn parse_ok_response<'a>(response: LocalResponse<'a>) -> Vec { +fn parse_ok_response<'a>(response: LocalResponse<'a>) -> HashMap { let body = response.into_string().unwrap(); serde_json::from_str(&body).expect("can't parse value") } diff --git a/routes/tests/mock.rs b/routes/tests/mock.rs index 1e5382a..1f466ea 100644 --- a/routes/tests/mock.rs +++ b/routes/tests/mock.rs @@ -97,45 +97,3 @@ pub fn mock_consumption() -> HashMap> { ], } } - -pub fn aggregated_mock_consumption( - para: (RelayChain, ParaId), - grouping: Grouping, -) -> HashMap { - match para { - (Polkadot, 2000) => { - hashmap! { - "1".to_string() => AggregatedData { - ref_time: (0.5, 0.3, 0.2).into(), - proof_size: (0.5, 0.3, 0.2).into(), - count: 1, - }, - "2".to_string() => AggregatedData { - ref_time: (0.1, 0.4, 0.2).into(), - proof_size: (0.2, 0.3, 0.3).into(), - count: 1, - }, - "3".to_string() => AggregatedData { - ref_time: (0.0, 0.2, 0.4).into(), - proof_size: (0.1, 0.0, 0.3).into(), - count: 1, - }, - "4".to_string() => AggregatedData { - ref_time: (0.1, 0.0, 0.4).into(), - proof_size: (0.2, 0.1, 0.3).into(), - count: 1, - } - } - }, - (Polkadot, 2004) => { - hashmap! { - "1".to_string() => AggregatedData { - ref_time: (0.8, 0.0, 0.1).into(), - proof_size: (0.6, 0.2, 0.1).into(), - count: 1, - } - } - }, - _ => panic!("No consumption data"), - } -} From e1cfbb1977954c50e4684c0add98bbe6f904a048 Mon Sep 17 00:00:00 2001 From: Szegoo Date: Mon, 5 Feb 2024 11:26:27 +0100 Subject: [PATCH 3/4] add test --- routes/mock-parachains.json | 27 +-------------------- routes/tests/consumption.rs | 47 +++++++++++++++++++++++++++++++++++++ routes/tests/mock.rs | 3 +-- 3 files changed, 49 insertions(+), 28 deletions(-) diff --git a/routes/mock-parachains.json b/routes/mock-parachains.json index c9d718d..0637a08 100644 --- a/routes/mock-parachains.json +++ b/routes/mock-parachains.json @@ -1,26 +1 @@ -[ - { - "name": "Moonbeam", - "rpcs": [ - "wss://moonbeam-rpc.dwellir.com", - "wss://1rpc.io/glmr", - "wss://wss.api.moonbeam.network", - "wss://moonbeam.unitedbloc.com" - ], - "para_id": 2004, - "relay_chain": "Polkadot", - "last_payment_timestamp": 0 - }, - { - "name": "Acala", - "rpcs": [ - "wss://acala-rpc.dwellir.com", - "wss://acala-rpc-0.aca-api.network", - "wss://acala-rpc-1.aca-api.network", - "wss://acala-rpc-3.aca-api.network/ws" - ], - "para_id": 2000, - "relay_chain": "Polkadot", - "last_payment_timestamp": 0 - } -] \ No newline at end of file +[] \ No newline at end of file diff --git a/routes/tests/consumption.rs b/routes/tests/consumption.rs index 6048d3c..40e7461 100644 --- a/routes/tests/consumption.rs +++ b/routes/tests/consumption.rs @@ -235,6 +235,53 @@ fn pagination_and_timestamp_filtering_works() { }); } +#[test] +fn grouping_works() { + MockEnvironment::new().execute_with(|| { + let rocket = rocket::build().mount("/", routes![consumption]); + let client = Client::tracked(rocket).expect("valid rocket instance"); + + // Default is grouping by block number: + let para = get_para(Polkadot, 2000).unwrap(); + let response = client.get("/consumption/polkadot/2000").dispatch(); + assert_eq!(response.status(), Status::Ok); + + let consumption_data = parse_ok_response(response); + let expected_consumption = group_consumption( + mock_consumption().get(¶).unwrap().clone(), + Grouping::BlockNumber, + ); + assert_eq!(consumption_data, expected_consumption); + + // Grouping by day: + let response = client.get("/consumption/polkadot/2000?grouping=day").dispatch(); + assert_eq!(response.status(), Status::Ok); + + let consumption_data = parse_ok_response(response); + let expected_consumption = + group_consumption(mock_consumption().get(¶).unwrap().clone(), Grouping::Day); + assert_eq!(consumption_data, expected_consumption); + + // Grouping by month: + let response = client.get("/consumption/polkadot/2000?grouping=month").dispatch(); + assert_eq!(response.status(), Status::Ok); + + let consumption_data = parse_ok_response(response); + let expected_consumption = + group_consumption(mock_consumption().get(¶).unwrap().clone(), Grouping::Month); + assert_eq!(consumption_data, expected_consumption); + + // Grouping by year: + let response = client.get("/consumption/polkadot/2000?grouping=year").dispatch(); + assert_eq!(response.status(), Status::Ok); + + let consumption_data = parse_ok_response(response); + let expected_consumption = + group_consumption(mock_consumption().get(¶).unwrap().clone(), Grouping::Year); + assert_eq!(consumption_data, expected_consumption); + }); +} + fn parse_ok_response<'a>(response: LocalResponse<'a>) -> HashMap { let body = response.into_string().unwrap(); serde_json::from_str(&body).expect("can't parse value") diff --git a/routes/tests/mock.rs b/routes/tests/mock.rs index 1f466ea..fb129ce 100644 --- a/routes/tests/mock.rs +++ b/routes/tests/mock.rs @@ -15,14 +15,13 @@ #[cfg(test)] use maplit::hashmap; -use routes::consumption::{AggregatedData, Grouping}; use scopeguard::guard; use shared::{ chaindata::get_para, consumption::write_consumption, registry::update_registry, reset_mock_environment, }; use std::collections::HashMap; -use types::{ParaId, Parachain, RelayChain, RelayChain::*, WeightConsumption}; +use types::{Parachain, RelayChain::*, WeightConsumption}; #[derive(Default)] pub struct MockEnvironment { From e8e5d53ff63c65fffa7dcfebc8badacfad8dbb51 Mon Sep 17 00:00:00 2001 From: Szegoo Date: Mon, 5 Feb 2024 11:34:51 +0100 Subject: [PATCH 4/4] fix clippy --- routes/src/consumption.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/src/consumption.rs b/routes/src/consumption.rs index 6f549ee..579868a 100644 --- a/routes/src/consumption.rs +++ b/routes/src/consumption.rs @@ -95,7 +95,7 @@ pub fn group_consumption( ) -> HashMap { weight_consumptions.iter().fold(HashMap::new(), |mut acc, datum| { let key = get_aggregation_key(datum.clone(), grouping); - let entry = acc.entry(key).or_insert_with(Default::default); + let entry = acc.entry(key).or_default(); entry.ref_time.normal += datum.ref_time.normal; entry.ref_time.operational += datum.ref_time.operational;