From 168622d11ebabed96bdabb028cc6c346425602d3 Mon Sep 17 00:00:00 2001 From: Yurii Koba Date: Fri, 4 Oct 2024 13:49:21 +0300 Subject: [PATCH] Revert "migrate to fastnear (#355)" (#357) This reverts commit ec9bed7133f9cb772b57edbd215084d0d543f0b1. --- Cargo.lock | 510 +++++++++-------------- Cargo.toml | 2 +- cache-storage/src/lib.rs | 87 ++++ config.toml | 5 +- configuration/Cargo.toml | 3 + configuration/example.config.toml | 15 +- configuration/src/configs/lake.rs | 77 +++- near-state-indexer/Cargo.toml | 1 + near-state-indexer/src/main.rs | 24 +- near-state-indexer/src/utils.rs | 110 +++++ rpc-server/src/config.rs | 43 +- rpc-server/src/main.rs | 56 ++- rpc-server/src/metrics.rs | 5 - rpc-server/src/modules/blocks/methods.rs | 53 ++- rpc-server/src/modules/blocks/mod.rs | 27 +- rpc-server/src/modules/blocks/utils.rs | 12 +- rpc-server/src/utils.rs | 176 +++++--- state-indexer/src/main.rs | 5 +- tx-indexer/src/main.rs | 2 +- 19 files changed, 753 insertions(+), 460 deletions(-) create mode 100644 near-state-indexer/src/utils.rs diff --git a/Cargo.lock b/Cargo.lock index 3022d33b..bc518b61 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -104,7 +104,7 @@ dependencies = [ "encoding_rs", "flate2", "futures-core", - "h2 0.3.26", + "h2", "http 0.2.12", "httparse", "httpdate", @@ -130,7 +130,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ "quote", - "syn 2.0.79", + "syn 2.0.77", ] [[package]] @@ -269,7 +269,7 @@ dependencies = [ "actix-router", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.77", ] [[package]] @@ -280,7 +280,7 @@ checksum = "b6ac1e58cded18cb28ddc17143c4dea5345b3ad575e14f32f66e4054a56eb271" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.77", ] [[package]] @@ -487,9 +487,9 @@ dependencies = [ [[package]] name = "async-stream" -version = "0.3.6" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" dependencies = [ "async-stream-impl", "futures-core", @@ -498,24 +498,24 @@ dependencies = [ [[package]] name = "async-stream-impl" -version = "0.3.6" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.77", ] [[package]] name = "async-trait" -version = "0.1.83" +version = "0.1.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" +checksum = "a27b8a3a6e1a44fa4c8baf1f653e4172e81486d4941f2237e20dc2d0cf4ddff1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.77", ] [[package]] @@ -527,12 +527,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "atomic-waker" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" - [[package]] name = "attohttpc" version = "0.19.1" @@ -551,9 +545,9 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.4.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "awc" @@ -574,7 +568,7 @@ dependencies = [ "derive_more", "futures-core", "futures-util", - "h2 0.3.26", + "h2", "http 0.2.12", "itoa", "log", @@ -591,9 +585,9 @@ dependencies = [ [[package]] name = "aws-config" -version = "1.5.7" +version = "1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8191fb3091fa0561d1379ef80333c3c7191c6f0435d986e85821bcf7acbd1126" +checksum = "848d7b9b605720989929279fa644ce8f244d0ce3146fcca5b70e4eb7b3c020fc" dependencies = [ "aws-credential-types", "aws-runtime", @@ -684,9 +678,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.52.0" +version = "1.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f571deb0a80c20d21d9f3e8418c1712af9ff4bf399d057e5549a934eca4844e2" +checksum = "c09fd4b5c7ed75f52b913b4f3ff0501dae7f8cb9125f6d45db4553980cbc0528" dependencies = [ "ahash 0.8.11", "aws-credential-types", @@ -719,9 +713,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.44.0" +version = "1.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b90cfe6504115e13c41d3ea90286ede5aa14da294f3fe077027a6e83850843c" +checksum = "70a9d27ed1c12b1140c47daf1bc541606c43fdafd918c4797d520db0043ceef2" dependencies = [ "aws-credential-types", "aws-runtime", @@ -741,9 +735,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.45.0" +version = "1.44.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167c0fad1f212952084137308359e8e4c4724d1c643038ce163f06de9662c1d0" +checksum = "44514a6ca967686cde1e2a1b81df6ef1883d0e3e570da8d8bc5c491dcb6fc29b" dependencies = [ "aws-credential-types", "aws-runtime", @@ -763,9 +757,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.44.0" +version = "1.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cb5f98188ec1435b68097daa2a37d74b9d17c9caa799466338a8d1544e71b9d" +checksum = "cd7a4d279762a35b9df97209f6808b95d4fe78547fe2316b4d200a0283960c5a" dependencies = [ "aws-credential-types", "aws-runtime", @@ -908,17 +902,17 @@ dependencies = [ "aws-smithy-types", "bytes", "fastrand 2.1.1", - "h2 0.3.26", + "h2", "http 0.2.12", "http-body 0.4.6", "http-body 1.0.1", "httparse", "hyper 0.14.30", - "hyper-rustls 0.24.2", + "hyper-rustls", "once_cell", "pin-project-lite", "pin-utils", - "rustls 0.21.12", + "rustls", "tokio", "tracing", ] @@ -942,9 +936,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.2.7" +version = "1.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147100a7bea70fa20ef224a6bad700358305f5dc0f84649c53769761395b355b" +checksum = "03701449087215b5369c7ea17fef0dd5d24cb93439ec5af0c7615f58c3f22605" dependencies = [ "base64-simd", "bytes", @@ -1133,7 +1127,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.79", + "syn 2.0.77", ] [[package]] @@ -1281,7 +1275,7 @@ dependencies = [ "proc-macro-crate 3.2.0", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.77", "syn_derive", ] @@ -1444,9 +1438,9 @@ checksum = "a2698f953def977c68f935bb0dfa959375ad4638570e969e2f1e9f433cbf1af6" [[package]] name = "cc" -version = "1.1.24" +version = "1.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812acba72f0a070b003d3697490d2b55b837230ae7c6c6497f05cc2ddbb8d938" +checksum = "07b1695e2c7e8fc85310cde85aeaab7e3097f593c91d209d3f9df76c928100f0" dependencies = [ "jobserver", "libc", @@ -1517,9 +1511,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.19" +version = "4.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7be5744db7978a28d9df86a214130d106a89ce49644cbc4e3f0c22c3fba30615" +checksum = "3e5a21b8495e732f1b3c364c9949b201ca7bae518c502c80256c96ad79eaf6ac" dependencies = [ "clap_builder", "clap_derive", @@ -1527,9 +1521,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.19" +version = "4.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5fbc17d3ef8278f55b282b2a2e75ae6f6c7d4bb70ed3d0382375104bfafdb4b" +checksum = "8cf2dd12af7a047ad9d6da2b6b249759a22a7abc0f474c1dae1777afa4b21a73" dependencies = [ "anstream", "anstyle", @@ -1539,14 +1533,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.18" +version = "4.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" +checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0" dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.77", ] [[package]] @@ -1621,6 +1615,9 @@ name = "configuration" version = "0.2.16" dependencies = [ "anyhow", + "aws-credential-types", + "aws-sdk-s3", + "aws-types", "dotenv", "google-cloud-storage", "lazy_static", @@ -1964,9 +1961,9 @@ dependencies = [ [[package]] name = "curl" -version = "0.4.47" +version = "0.4.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9fb4d13a1be2b58f14d60adba57c9834b78c62fd86c3e76a148f732686e9265" +checksum = "1e2161dd6eba090ff1594084e95fd67aeccf04382ffea77999ea94ed42ec67b6" dependencies = [ "curl-sys", "libc", @@ -1979,9 +1976,9 @@ dependencies = [ [[package]] name = "curl-sys" -version = "0.4.77+curl-8.10.1" +version = "0.4.75+curl-8.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f469e8a5991f277a208224f6c7ad72ecb5f986e36d09ae1f2c1bb9259478a480" +checksum = "2a4fd752d337342e4314717c0d9b6586b059a120c80029ebe4d49b11fec7875e" dependencies = [ "cc", "libc", @@ -2017,7 +2014,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.77", ] [[package]] @@ -2065,7 +2062,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.79", + "syn 2.0.77", ] [[package]] @@ -2087,7 +2084,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core 0.20.10", "quote", - "syn 2.0.79", + "syn 2.0.77", ] [[package]] @@ -2171,7 +2168,7 @@ checksum = "62d671cc41a825ebabc75757b62d3d168c577f9149b2d49ece1dad1f72119d25" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.77", ] [[package]] @@ -2182,7 +2179,7 @@ checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.77", ] [[package]] @@ -2226,7 +2223,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version 0.4.1", - "syn 2.0.79", + "syn 2.0.77", ] [[package]] @@ -2457,7 +2454,7 @@ checksum = "f282cfdfe92516eb26c2af8589c274c7c17681f5ecc03c18255fe741c6aa64eb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.77", ] [[package]] @@ -2478,7 +2475,7 @@ dependencies = [ "darling 0.20.10", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.77", ] [[package]] @@ -2615,9 +2612,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flate2" -version = "1.0.34" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0" +checksum = "324a1be68054ef05ad64b861cc9eaf1d623d2d8cb25b4bf2cb9cdd902b4bf253" dependencies = [ "crc32fast", "miniz_oxide", @@ -2773,7 +2770,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.77", ] [[package]] @@ -2878,7 +2875,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" dependencies = [ "fallible-iterator", - "indexmap 2.6.0", + "indexmap 2.5.0", "stable_deref_trait", ] @@ -2906,7 +2903,7 @@ dependencies = [ "google-cloud-token", "home", "jsonwebtoken 9.3.0", - "reqwest 0.12.8", + "reqwest 0.12.7", "serde", "serde_json", "thiserror", @@ -2922,7 +2919,7 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04f945a208886a13d07636f38fb978da371d0abc3e34bad338124b9f8c135a8f" dependencies = [ - "reqwest 0.12.8", + "reqwest 0.12.7", "thiserror", "tokio", ] @@ -2947,7 +2944,7 @@ dependencies = [ "percent-encoding", "pkcs8 0.10.2", "regex", - "reqwest 0.12.8", + "reqwest 0.12.7", "reqwest-middleware", "ring 0.17.8", "serde", @@ -2992,26 +2989,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.6.0", - "slab", - "tokio", - "tokio-util 0.7.12", - "tracing", -] - -[[package]] -name = "h2" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" -dependencies = [ - "atomic-waker", - "bytes", - "fnv", - "futures-core", - "futures-sink", - "http 1.1.0", - "indexmap 2.6.0", + "indexmap 2.5.0", "slab", "tokio", "tokio-util 0.7.12", @@ -3055,12 +3033,6 @@ dependencies = [ "allocator-api2", ] -[[package]] -name = "hashbrown" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" - [[package]] name = "hashlink" version = "0.8.4" @@ -3194,9 +3166,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.9.5" +version = "1.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" +checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" [[package]] name = "httpdate" @@ -3220,7 +3192,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2 0.3.26", + "h2", "http 0.2.12", "http-body 0.4.6", "httparse", @@ -3243,7 +3215,6 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.6", "http 1.1.0", "http-body 1.0.1", "httparse", @@ -3264,27 +3235,10 @@ dependencies = [ "http 0.2.12", "hyper 0.14.30", "log", - "rustls 0.21.12", + "rustls", "rustls-native-certs", "tokio", - "tokio-rustls 0.24.1", -] - -[[package]] -name = "hyper-rustls" -version = "0.27.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" -dependencies = [ - "futures-util", - "http 1.1.0", - "hyper 1.4.1", - "hyper-util", - "rustls 0.23.13", - "rustls-pki-types", - "tokio", - "tokio-rustls 0.26.0", - "tower-service", + "tokio-rustls", ] [[package]] @@ -3330,9 +3284,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.9" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41296eb09f183ac68eec06e03cdbea2e759633d4067b2f6552fc2e009bcad08b" +checksum = "da62f120a8a37763efb0cf8fdf264b884c7b8b9ac8660b900c8661030c00e6ba" dependencies = [ "bytes", "futures-channel", @@ -3343,6 +3297,7 @@ dependencies = [ "pin-project-lite", "socket2", "tokio", + "tower", "tower-service", "tracing", ] @@ -3419,12 +3374,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.6.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" +checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" dependencies = [ "equivalent", - "hashbrown 0.15.0", + "hashbrown 0.14.5", "serde", ] @@ -3617,9 +3572,9 @@ checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" [[package]] name = "libc" -version = "0.2.159" +version = "0.2.158" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" +checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" [[package]] name = "libloading" @@ -3791,9 +3746,9 @@ dependencies = [ [[package]] name = "lz4-sys" -version = "1.11.1+lz4-1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" +checksum = "fcb44a01837a858d47e5a630d2ccf304c8efcc4b83b8f9f75b7a9ee4fcc6e57d" dependencies = [ "cc", "libc", @@ -3851,7 +3806,7 @@ checksum = "5cf92c10c7e361d6b99666ec1c6f9805b0bea2c3bd8c78dc6fe98ac5bd78db11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.77", ] [[package]] @@ -4007,9 +3962,9 @@ checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" [[package]] name = "mutually_exclusive_features" -version = "0.1.0" +version = "0.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94e1e6445d314f972ff7395df2de295fe51b71821694f0b0e1e79c4f12c8577" +checksum = "6d02c0b00610773bb7fc61d85e13d86c7858cbdf00e1a120bfc41bc055dbaa0e" [[package]] name = "native-tls" @@ -4065,7 +4020,7 @@ source = "git+https://github.com/kobayurii/nearcore.git?branch=2.2.1-fork1#3a975 dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.77", ] [[package]] @@ -4543,7 +4498,7 @@ dependencies = [ "near-crypto 2.2.1", "near-jsonrpc-primitives 2.2.1", "near-primitives 2.2.1", - "reqwest 0.12.8", + "reqwest 0.12.7", "serde", "serde_json", "thiserror", @@ -4599,7 +4554,7 @@ dependencies = [ [[package]] name = "near-lake-framework" version = "0.0.0" -source = "git+https://github.com/kobayurii/near-lake-framework-rs.git?branch=fork/0.7.11#bb52d6d37945aa3cf6e022a43a4d8ea9dd676145" +source = "git+https://github.com/kobayurii/near-lake-framework-rs.git?branch=0.7.20#285e88382b25823141a0f80840d30f918c999ca2" dependencies = [ "anyhow", "async-stream", @@ -4612,7 +4567,6 @@ dependencies = [ "derive_builder", "futures", "near-indexer-primitives", - "reqwest 0.12.8", "serde", "serde_json", "thiserror", @@ -4666,7 +4620,7 @@ dependencies = [ "opentelemetry 0.22.0", "parking_lot 0.12.3", "pin-project", - "protobuf 3.6.0", + "protobuf 3.5.1", "protobuf-codegen", "rand 0.8.5", "rayon", @@ -4796,7 +4750,7 @@ version = "2.2.1" source = "git+https://github.com/kobayurii/nearcore.git?branch=2.2.1-fork1#3a97591c46c1b62082de8a48a7da48bb0778a1bb" dependencies = [ "quote", - "syn 2.0.79", + "syn 2.0.77", ] [[package]] @@ -4881,7 +4835,7 @@ dependencies = [ "near-time", "num-rational", "once_cell", - "ordered-float 4.3.0", + "ordered-float 4.2.2", "primitive-types", "rand 0.8.5", "rand_chacha 0.3.1", @@ -4978,7 +4932,7 @@ checksum = "80fca203c51edd9595ec14db1d13359fb9ede32314990bf296b6c5c4502f6ab7" dependencies = [ "quote", "serde", - "syn 2.0.79", + "syn 2.0.77", ] [[package]] @@ -4988,7 +4942,7 @@ source = "git+https://github.com/kobayurii/nearcore.git?branch=2.2.1-fork1#3a975 dependencies = [ "quote", "serde", - "syn 2.0.79", + "syn 2.0.77", ] [[package]] @@ -5000,7 +4954,7 @@ dependencies = [ "fs2", "near-rpc-error-core 0.20.1", "serde", - "syn 2.0.79", + "syn 2.0.77", ] [[package]] @@ -5010,7 +4964,7 @@ source = "git+https://github.com/kobayurii/nearcore.git?branch=2.2.1-fork1#3a975 dependencies = [ "near-rpc-error-core 2.2.1", "serde", - "syn 2.0.79", + "syn 2.0.77", ] [[package]] @@ -5020,6 +4974,7 @@ dependencies = [ "actix", "actix-web", "anyhow", + "cache-storage", "clap", "configuration", "database", @@ -5598,7 +5553,7 @@ checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" dependencies = [ "crc32fast", "hashbrown 0.14.5", - "indexmap 2.6.0", + "indexmap 2.5.0", "memchr", ] @@ -5613,12 +5568,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.20.1" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82881c4be219ab5faaf2ad5e5e5ecdff8c66bd7402ca3160975c93b24961afd1" -dependencies = [ - "portable-atomic", -] +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "opaque-debug" @@ -5649,7 +5601,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.77", ] [[package]] @@ -5884,7 +5836,7 @@ dependencies = [ "glob", "once_cell", "opentelemetry 0.22.0", - "ordered-float 4.3.0", + "ordered-float 4.2.2", "percent-encoding", "rand 0.8.5", "thiserror", @@ -5903,9 +5855,9 @@ dependencies = [ [[package]] name = "ordered-float" -version = "4.3.0" +version = "4.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d501f1a72f71d3c063a6bbc8f7271fa73aa09fe5d6283b6571e2ed176a2537" +checksum = "4a91171844676f8c7990ce64959210cd2eaef32c2612c50f9fae9f8aaa6065a6" dependencies = [ "borsh 1.5.1", "num-traits", @@ -6114,7 +6066,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall 0.5.7", + "redox_syscall 0.5.4", "smallvec", "windows-targets 0.52.6", ] @@ -6193,7 +6145,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap 2.6.0", + "indexmap 2.5.0", ] [[package]] @@ -6213,7 +6165,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.77", ] [[package]] @@ -6261,9 +6213,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.31" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" [[package]] name = "polling" @@ -6281,12 +6233,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "portable-atomic" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2" - [[package]] name = "powerfmt" version = "0.2.0" @@ -6315,7 +6261,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479cf940fbbb3426c32c5d5176f62ad57549a0bb84773423ba8be9d089f5faba" dependencies = [ "proc-macro2", - "syn 2.0.79", + "syn 2.0.77", ] [[package]] @@ -6457,7 +6403,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.77", ] [[package]] @@ -6478,9 +6424,9 @@ checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" [[package]] name = "protobuf" -version = "3.6.0" +version = "3.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3018844a02746180074f621e847703737d27d89d7f0721a7a4da317f88b16385" +checksum = "0bcc343da15609eaecd65f8aa76df8dc4209d325131d8219358c0aaaebab0bf6" dependencies = [ "once_cell", "protobuf-support", @@ -6489,13 +6435,13 @@ dependencies = [ [[package]] name = "protobuf-codegen" -version = "3.6.0" +version = "3.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "411c15a212b4de05eb8bc989fd066a74c86bd3c04e27d6e86bd7703b806d7734" +checksum = "c4d0cde5642ea4df842b13eb9f59ea6fafa26dcb43e3e1ee49120e9757556189" dependencies = [ "anyhow", "once_cell", - "protobuf 3.6.0", + "protobuf 3.5.1", "protobuf-parse", "regex", "tempfile", @@ -6504,14 +6450,14 @@ dependencies = [ [[package]] name = "protobuf-parse" -version = "3.6.0" +version = "3.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06f45f16b522d92336e839b5e40680095a045e36a1e7f742ba682ddc85236772" +checksum = "1b0e9b447d099ae2c4993c0cbb03c7a9d6c937b17f2d56cfc0b1550e6fcfdb76" dependencies = [ "anyhow", - "indexmap 2.6.0", + "indexmap 2.5.0", "log", - "protobuf 3.6.0", + "protobuf 3.5.1", "protobuf-support", "tempfile", "thiserror", @@ -6520,9 +6466,9 @@ dependencies = [ [[package]] name = "protobuf-support" -version = "3.6.0" +version = "3.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faf96d872914fcda2b66d66ea3fff2be7c66865d31c7bb2790cff32c0e714880" +checksum = "f0766e3675a627c327e4b3964582594b0e8741305d628a98a5de75a1d15f99b9" dependencies = [ "thiserror", ] @@ -6790,9 +6736,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.7" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" +checksum = "0884ad60e090bf1345b93da0a5de8923c93884cd03f40dfcfddd3b4bee661853" dependencies = [ "bitflags 2.6.0", ] @@ -6845,14 +6791,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.11.0" +version = "1.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8" +checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.8", - "regex-syntax 0.8.5", + "regex-automata 0.4.7", + "regex-syntax 0.8.4", ] [[package]] @@ -6866,13 +6812,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.8" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" +checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.5", + "regex-syntax 0.8.4", ] [[package]] @@ -6889,9 +6835,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.5" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" [[package]] name = "region" @@ -6925,7 +6871,7 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2 0.3.26", + "h2", "http 0.2.12", "http-body 0.4.6", "hyper 0.14.30", @@ -6943,7 +6889,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "sync_wrapper 0.1.2", - "system-configuration 0.5.1", + "system-configuration", "tokio", "tokio-native-tls", "tokio-util 0.7.12", @@ -6958,21 +6904,19 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.8" +version = "0.12.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f713147fbe92361e52392c73b8c9e48c04c6625bce969ef54dc901e58e042a7b" +checksum = "f8f4955649ef5c38cc7f9e8aa41761d48fb9677197daea9984dc54f56aad5e63" dependencies = [ "base64 0.22.1", "bytes", "encoding_rs", "futures-core", "futures-util", - "h2 0.4.6", "http 1.1.0", "http-body 1.0.1", "http-body-util", "hyper 1.4.1", - "hyper-rustls 0.27.3", "hyper-tls 0.6.0", "hyper-util", "ipnet", @@ -6984,12 +6928,11 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls-pemfile 2.2.0", + "rustls-pemfile 2.1.3", "serde", "serde_json", "serde_urlencoded", "sync_wrapper 1.0.1", - "system-configuration 0.6.1", "tokio", "tokio-native-tls", "tokio-util 0.7.12", @@ -7011,7 +6954,7 @@ dependencies = [ "anyhow", "async-trait", "http 1.1.0", - "reqwest 0.12.8", + "reqwest 0.12.7", "serde", "thiserror", "tower-service", @@ -7234,23 +7177,10 @@ checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", "ring 0.17.8", - "rustls-webpki 0.101.7", + "rustls-webpki", "sct", ] -[[package]] -name = "rustls" -version = "0.23.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2dabaac7466917e566adb06783a81ca48944c6898a1b08b9374106dd671f4c8" -dependencies = [ - "once_cell", - "rustls-pki-types", - "rustls-webpki 0.102.8", - "subtle", - "zeroize", -] - [[package]] name = "rustls-native-certs" version = "0.6.3" @@ -7274,18 +7204,19 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "2.2.0" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" dependencies = [ + "base64 0.22.1", "rustls-pki-types", ] [[package]] name = "rustls-pki-types" -version = "1.9.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e696e35370c65c9c541198af4543ccd580cf17fc25d8e05c5a242b202488c55" +checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" [[package]] name = "rustls-webpki" @@ -7297,17 +7228,6 @@ dependencies = [ "untrusted 0.9.0", ] -[[package]] -name = "rustls-webpki" -version = "0.102.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" -dependencies = [ - "ring 0.17.8", - "rustls-pki-types", - "untrusted 0.9.0", -] - [[package]] name = "rustversion" version = "1.0.17" @@ -7416,9 +7336,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.12.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6" +checksum = "75da29fe9b9b08fe9d6b22b5b4bcbc75d8db3aa31e639aa56bb62e9d46bfceaf" dependencies = [ "core-foundation-sys", "libc", @@ -7493,7 +7413,7 @@ checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.77", ] [[package]] @@ -7525,14 +7445,14 @@ checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.77", ] [[package]] name = "serde_spanned" -version = "0.6.8" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" +checksum = "eb5b1b31579f3811bf615c144393417496f152e12ac8b7663bf664f4a815306d" dependencies = [ "serde", ] @@ -7551,15 +7471,15 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.10.0" +version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9720086b3357bcb44fce40117d769a4d068c70ecfa190850a980a71755f66fcc" +checksum = "69cecfa94848272156ea67b2b1a53f20fc7bc638c4a46d2f8abde08f05f4b857" dependencies = [ "base64 0.22.1", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.6.0", + "indexmap 2.5.0", "serde", "serde_derive", "serde_json", @@ -7569,14 +7489,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.10.0" +version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f1abbfe725f27678f4663bcacb75a83e829fd464c25d78dd038a3a29e307cec" +checksum = "a8fee4991ef4f274617a51ad4af30519438dacb2f56ac773b08a1922ff743350" dependencies = [ "darling 0.20.10", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.77", ] [[package]] @@ -7585,7 +7505,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.6.0", + "indexmap 2.5.0", "itoa", "ryu", "serde", @@ -7676,9 +7596,9 @@ dependencies = [ [[package]] name = "simdutf8" -version = "0.1.5" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" [[package]] name = "simple_asn1" @@ -7863,7 +7783,7 @@ dependencies = [ "futures-util", "hashlink", "hex", - "indexmap 2.6.0", + "indexmap 2.5.0", "log", "memchr", "native-tls", @@ -8141,9 +8061,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.79" +version = "2.0.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" +checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" dependencies = [ "proc-macro2", "quote", @@ -8159,7 +8079,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.77", ] [[package]] @@ -8200,18 +8120,7 @@ checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" dependencies = [ "bitflags 1.3.2", "core-foundation", - "system-configuration-sys 0.5.0", -] - -[[package]] -name = "system-configuration" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" -dependencies = [ - "bitflags 2.6.0", - "core-foundation", - "system-configuration-sys 0.6.0", + "system-configuration-sys", ] [[package]] @@ -8224,16 +8133,6 @@ dependencies = [ "libc", ] -[[package]] -name = "system-configuration-sys" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "tap" version = "1.0.1" @@ -8254,9 +8153,9 @@ checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" [[package]] name = "tempfile" -version = "3.13.0" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b" +checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" dependencies = [ "cfg-if 1.0.0", "fastrand 2.1.1", @@ -8267,22 +8166,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.64" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" +checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.64" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" +checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.77", ] [[package]] @@ -8410,7 +8309,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.77", ] [[package]] @@ -8451,18 +8350,7 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls 0.21.12", - "tokio", -] - -[[package]] -name = "tokio-rustls" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" -dependencies = [ - "rustls 0.23.13", - "rustls-pki-types", + "rustls", "tokio", ] @@ -8536,11 +8424,11 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.22" +version = "0.22.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" +checksum = "3b072cee73c449a636ffd6f32bd8de3a9f7119139aff882f44943ce2986dc5cf" dependencies = [ - "indexmap 2.6.0", + "indexmap 2.5.0", "serde", "serde_spanned", "toml_datetime", @@ -8559,7 +8447,7 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "h2 0.3.26", + "h2", "http 0.2.12", "http-body 0.4.6", "hyper 0.14.30", @@ -8589,7 +8477,7 @@ dependencies = [ "axum", "base64 0.21.7", "bytes", - "h2 0.3.26", + "h2", "http 0.2.12", "http-body 0.4.6", "hyper 0.14.30", @@ -8663,9 +8551,9 @@ dependencies = [ [[package]] name = "tracing-actix-web" -version = "0.7.13" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15bc0cd5f72e837e310f4d978a90abf202a7f7d8ef3272246bae381d0086d3bf" +checksum = "284586dc201db407be8c9d721abad1b3a6dacbbce5cccecd4fd15a37db95ab0d" dependencies = [ "actix-web", "mutually_exclusive_features", @@ -8694,7 +8582,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.77", ] [[package]] @@ -8929,9 +8817,9 @@ dependencies = [ [[package]] name = "unicode-properties" -version = "0.1.3" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0" +checksum = "52ea75f83c0137a9b98608359a5f1af8144876eb67bcb1ce837368e906a9f524" [[package]] name = "unicode-segmentation" @@ -8941,9 +8829,9 @@ checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" [[package]] name = "unicode-width" -version = "0.1.14" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" +checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" [[package]] name = "unicode_categories" @@ -9028,7 +8916,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.77", ] [[package]] @@ -9116,7 +9004,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.77", "wasm-bindgen-shared", ] @@ -9150,7 +9038,7 @@ checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.77", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -9181,9 +9069,9 @@ dependencies = [ [[package]] name = "wasm-streams" -version = "0.4.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e072d4e72f700fb3443d8fe94a39315df013eef1104903cdb0a2abd322bbecd" +checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129" dependencies = [ "futures-util", "js-sys", @@ -9398,7 +9286,7 @@ version = "0.115.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e06c0641a4add879ba71ccb3a1e4278fd546f76f1eafb21d8f7b07733b547cd5" dependencies = [ - "indexmap 2.6.0", + "indexmap 2.5.0", "semver 1.0.23", ] @@ -9423,7 +9311,7 @@ dependencies = [ "bumpalo", "cfg-if 1.0.0", "fxprof-processed-profile", - "indexmap 2.6.0", + "indexmap 2.5.0", "libc", "log", "object 0.32.2", @@ -9502,7 +9390,7 @@ dependencies = [ "anyhow", "cranelift-entity", "gimli 0.28.1", - "indexmap 2.6.0", + "indexmap 2.5.0", "log", "object 0.32.2", "serde", @@ -9568,7 +9456,7 @@ dependencies = [ "anyhow", "cc", "cfg-if 1.0.0", - "indexmap 2.6.0", + "indexmap 2.5.0", "libc", "log", "mach", @@ -9608,7 +9496,7 @@ checksum = "09b5575a75e711ca6c36bb9ad647c93541cdc8e34218031acba5da3f35919dd3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.77", ] [[package]] @@ -9676,15 +9564,15 @@ version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "372d5b87f58ec45c384ba03563b03544dc5fadc3983e434b286913f5b4a9bb6d" dependencies = [ - "redox_syscall 0.5.7", + "redox_syscall 0.5.4", "wasite", ] [[package]] name = "wildmatch" -version = "2.4.0" +version = "2.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ce1ab1f8c62655ebe1350f589c61e505cf94d385bc6a12899442d9081e71fd" +checksum = "3928939971918220fed093266b809d1ee4ec6c1a2d72692ff6876898f3b16c19" [[package]] name = "winapi" @@ -9897,9 +9785,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.6.20" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" +checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f" dependencies = [ "memchr", ] @@ -9968,7 +9856,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.77", ] [[package]] @@ -9988,7 +9876,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.77", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 1e98c8bc..847cc6ee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,5 +68,5 @@ near-vm-runner = { git = 'https://github.com/kobayurii/nearcore.git', branch = " "near_vm", ] } -near-lake-framework = { git = 'https://github.com/kobayurii/near-lake-framework-rs.git', branch = 'fork/0.7.11' } +near-lake-framework = { git = 'https://github.com/kobayurii/near-lake-framework-rs.git', branch = '0.7.20' } near-jsonrpc-client = { git = 'https://github.com/kobayurii/near-jsonrpc-client-rs.git', branch = '0.13.3' } diff --git a/cache-storage/src/lib.rs b/cache-storage/src/lib.rs index 8f0055a4..bb9ffcb7 100644 --- a/cache-storage/src/lib.rs +++ b/cache-storage/src/lib.rs @@ -1,4 +1,5 @@ use futures::FutureExt; +use near_indexer_primitives::near_primitives; mod utils; @@ -114,6 +115,92 @@ impl RedisCacheStorage { } } +#[derive(Clone)] +pub struct BlocksByFinalityCache { + cache_storage: RedisCacheStorage, +} + +/// Sets the keys in Redis shared between the ReadRPC components about the most recent +/// blocks based on finalities (final or optimistic). +/// `final_height` of `optimistic_height` depending on `block_type` passed. +/// Additionally, sets the JSON serialized `StreamerMessage` into keys `final` or `optimistic` +/// accordingly. +impl BlocksByFinalityCache { + // Use redis database 0(default for redis) for handling the blocks by finality cache. + pub async fn new(redis_url: String) -> anyhow::Result { + Ok(Self { + cache_storage: RedisCacheStorage::new(redis_url, 0).await?, + }) + } + + pub async fn update_block_by_finality( + &self, + finality: near_indexer_primitives::near_primitives::types::Finality, + streamer_message: &near_indexer_primitives::StreamerMessage, + ) -> anyhow::Result<()> { + let block_height = streamer_message.block.header.height; + let block_type = serde_json::to_string(&finality)?; + + let last_height = self + .cache_storage + .get(format!("{}_height", block_type)) + .await + .unwrap_or(0); + + // If the block height is greater than the last height, update the block streamer message + // if we have a few indexers running, we need to make sure that we are not updating the same block + // or block which is already processed or block less than the last processed block + if block_height > last_height { + let json_streamer_message = serde_json::to_string(streamer_message)?; + // Update the last block height + // Create a clone of the redis client and redis cmd to avoid borrowing issues + let update_height_feature = self + .cache_storage + .set(format!("{}_height", block_type), block_height); + + // Update the block streamer message + // Create a clone of the redis client and redis cmd to avoid borrowing issues + let update_stream_msg_feature = + self.cache_storage.set(block_type, json_streamer_message); + + // Wait for both futures to complete + futures::future::join_all([ + update_height_feature.boxed(), + update_stream_msg_feature.boxed(), + ]) + .await + .into_iter() + .collect::>()?; + }; + + Ok(()) + } + + pub async fn get_block_by_finality( + &self, + finality: near_indexer_primitives::near_primitives::types::Finality, + ) -> anyhow::Result { + let block_type = serde_json::to_string(&finality)?; + let resp: String = self.cache_storage.get(block_type).await?; + Ok(serde_json::from_str(&resp)?) + } + + pub async fn update_protocol_version( + &self, + protocol_version: near_primitives::types::ProtocolVersion, + ) -> anyhow::Result<()> { + self.cache_storage + .set("protocol_version", protocol_version) + .await + } + + pub async fn get_protocol_version( + &self, + ) -> anyhow::Result { + self.cache_storage.get("protocol_version").await + } +} + #[derive(Clone)] pub struct TxIndexerCache { cache_storage: RedisCacheStorage, diff --git a/config.toml b/config.toml index 8b5e9b41..ba66f03a 100644 --- a/config.toml +++ b/config.toml @@ -34,7 +34,10 @@ tracked_accounts = "${TRACKED_ACCOUNTS}" tracked_changes = "${TRACKED_CHANGES}" [lake_config] -num_threads = "${LAKE_NUM_THREADS}" +aws_access_key_id = "${AWS_ACCESS_KEY_ID}" +aws_secret_access_key = "${AWS_SECRET_ACCESS_KEY}" +aws_default_region = "${AWS_DEFAULT_REGION}" +aws_bucket_name = "${AWS_BUCKET_NAME}" [tx_details_storage] bucket_name = "${TX_BUCKET_NAME}" diff --git a/configuration/Cargo.toml b/configuration/Cargo.toml index c094aafc..c9902522 100644 --- a/configuration/Cargo.toml +++ b/configuration/Cargo.toml @@ -9,6 +9,9 @@ license.workspace = true [dependencies] anyhow = "1.0.70" +aws-credential-types = "1.1.4" +aws-sdk-s3 = { version = "1.14.0", features = ["behavior-version-latest"] } +aws-types = "1.1.4" dotenv = "0.15.0" google-cloud-storage = "0.20.0" lazy_static = "1.4.0" diff --git a/configuration/example.config.toml b/configuration/example.config.toml index 63b8c98d..40a26188 100644 --- a/configuration/example.config.toml +++ b/configuration/example.config.toml @@ -115,9 +115,18 @@ near_rpc_url = "https://beta.rpc.mainnet.near.org" ### Lake framework configuration [lake_config] -# Number of threads to use for fetching data from fastnear -# Default: 2 * available threads -#num_threads = 8 + +## Lake framework AWS access key id +aws_access_key_id = "${AWS_ACCESS_KEY_ID}" + +## Lake framework AWS secret access key +aws_secret_access_key = "${AWS_SECRET_ACCESS_KEY}" + +## Lake framework AWS default region +aws_default_region = "eu-central-1" + +## Lake framework bucket name +aws_bucket_name = "near-lake-data-mainnet" [tx_details_storage] ## Transaction details are stored in the S3-compatibe object storage (Google Cloud Storage by default) diff --git a/configuration/src/configs/lake.rs b/configuration/src/configs/lake.rs index ea6c00aa..d4782bfb 100644 --- a/configuration/src/configs/lake.rs +++ b/configuration/src/configs/lake.rs @@ -1,43 +1,88 @@ -use crate::configs::deserialize_optional_data_or_env; +use aws_sdk_s3::config::StalledStreamProtectionConfig; use near_lake_framework::near_indexer_primitives::near_primitives; use serde_derive::Deserialize; +use crate::configs::{deserialize_optional_data_or_env, required_value_or_panic}; + #[derive(Debug, Clone)] pub struct LakeConfig { - pub num_threads: Option, + pub aws_access_key_id: String, + pub aws_secret_access_key: String, + pub aws_default_region: String, + pub aws_bucket_name: String, } impl LakeConfig { + pub async fn s3_config(&self) -> aws_sdk_s3::Config { + let credentials = aws_credential_types::Credentials::new( + &self.aws_access_key_id, + &self.aws_secret_access_key, + None, + None, + "", + ); + aws_sdk_s3::Config::builder() + .stalled_stream_protection(StalledStreamProtectionConfig::disabled()) + .credentials_provider(credentials) + .region(aws_types::region::Region::new( + self.aws_default_region.clone(), + )) + .build() + } + pub async fn lake_config( &self, start_block_height: near_primitives::types::BlockHeight, - chain_id: crate::ChainId, - ) -> anyhow::Result { - let mut config_builder = near_lake_framework::FastNearConfigBuilder::default(); - match chain_id { - crate::ChainId::Mainnet => config_builder = config_builder.mainnet(), - // Testnet is the default chain for other chain_id - _ => config_builder = config_builder.testnet(), - }; - if let Some(num_threads) = self.num_threads { - config_builder = config_builder.num_threads(num_threads); - }; + ) -> anyhow::Result { + let config_builder = near_lake_framework::LakeConfigBuilder::default(); Ok(config_builder + .s3_config(self.s3_config().await) + .s3_region_name(&self.aws_default_region) + .s3_bucket_name(&self.aws_bucket_name) .start_block_height(start_block_height) - .build()?) + .build() + .expect("Failed to build LakeConfig")) + } + + pub async fn lake_s3_client(&self) -> near_lake_framework::s3_fetchers::LakeS3Client { + let s3_config = self.s3_config().await; + near_lake_framework::s3_fetchers::LakeS3Client::new(aws_sdk_s3::Client::from_conf( + s3_config, + )) } } #[derive(Deserialize, Debug, Clone, Default)] pub struct CommonLakeConfig { #[serde(deserialize_with = "deserialize_optional_data_or_env", default)] - pub num_threads: Option, + pub aws_access_key_id: Option, + #[serde(deserialize_with = "deserialize_optional_data_or_env", default)] + pub aws_secret_access_key: Option, + #[serde(deserialize_with = "deserialize_optional_data_or_env", default)] + pub aws_default_region: Option, + #[serde(deserialize_with = "deserialize_optional_data_or_env", default)] + pub aws_bucket_name: Option, } impl From for LakeConfig { fn from(common_config: CommonLakeConfig) -> Self { Self { - num_threads: common_config.num_threads, + aws_access_key_id: required_value_or_panic( + "aws_access_key_id", + common_config.aws_access_key_id, + ), + aws_secret_access_key: required_value_or_panic( + "aws_secret_access_key", + common_config.aws_secret_access_key, + ), + aws_default_region: required_value_or_panic( + "aws_default_region", + common_config.aws_default_region, + ), + aws_bucket_name: required_value_or_panic( + "aws_bucket_name", + common_config.aws_bucket_name, + ), } } } diff --git a/near-state-indexer/Cargo.toml b/near-state-indexer/Cargo.toml index 4f919bd6..f20c0868 100644 --- a/near-state-indexer/Cargo.toml +++ b/near-state-indexer/Cargo.toml @@ -29,6 +29,7 @@ tokio = { version = "1.36.0", features = [ tokio-stream = "0.1" tracing = "0.1.34" +cache-storage.workspace = true configuration.workspace = true database.workspace = true logic-state-indexer.workspace = true diff --git a/near-state-indexer/src/main.rs b/near-state-indexer/src/main.rs index ee9b9f65..51aa0f32 100644 --- a/near-state-indexer/src/main.rs +++ b/near-state-indexer/src/main.rs @@ -8,6 +8,7 @@ use logic_state_indexer::{handle_streamer_message, NearClient, INDEXER}; mod configs; mod metrics; mod near_client; +mod utils; #[actix_web::main] async fn main() -> anyhow::Result<()> { @@ -58,6 +59,12 @@ async fn run(home_dir: std::path::PathBuf) -> anyhow::Result<()> { let state_indexer_config = configuration::read_configuration::().await?; + tracing::info!(target: INDEXER, "Connecting to redis..."); + let finality_blocks_storage = cache_storage::BlocksByFinalityCache::new( + state_indexer_config.general.redis_url.to_string(), + ) + .await?; + tracing::info!(target: INDEXER, "Setup near_indexer..."); let indexer_config = near_indexer::IndexerConfig { home_dir, @@ -73,7 +80,7 @@ async fn run(home_dir: std::path::PathBuf) -> anyhow::Result<()> { // Regular indexer process starts here tracing::info!(target: INDEXER, "Instantiating the stream..."); let stream = indexer.streamer(); - let (view_client, _) = indexer.client_actors(); + let (view_client, client) = indexer.client_actors(); let near_client = near_client::NearViewClient::new(view_client.clone()); let protocol_config_view = near_client.protocol_config().await?; @@ -91,6 +98,21 @@ async fn run(home_dir: std::path::PathBuf) -> anyhow::Result<()> { near_client.clone(), )); + // Initiate the job of updating the optimistic blocks to Redis + tokio::spawn(utils::update_block_in_redis_by_finality( + view_client.clone(), + client.clone(), + finality_blocks_storage.clone(), + near_indexer_primitives::near_primitives::types::Finality::None, + )); + // And the same job for the final blocks + tokio::spawn(utils::update_block_in_redis_by_finality( + view_client.clone(), + client.clone(), + finality_blocks_storage.clone(), + near_indexer_primitives::near_primitives::types::Finality::Final, + )); + // ! Note that the `handle_streamer_message` doesn't interact with the Redis tracing::info!(target: INDEXER, "Starting near_state_indexer..."); let mut handlers = tokio_stream::wrappers::ReceiverStream::new(stream) diff --git a/near-state-indexer/src/utils.rs b/near-state-indexer/src/utils.rs new file mode 100644 index 00000000..9d517c99 --- /dev/null +++ b/near-state-indexer/src/utils.rs @@ -0,0 +1,110 @@ +use near_indexer::near_primitives; +use near_o11y::WithSpanContextExt; + +const INTERVAL: std::time::Duration = std::time::Duration::from_secs(1); + +/// The universal function that fetches the block by the given finality. +/// It is used in the `update_block_in_redis_by_finality` function. +/// ! The function does not support the DoomSlug finality. +pub(crate) async fn fetch_block_by_finality( + client: &actix::Addr, + finality: &near_primitives::types::Finality, +) -> anyhow::Result { + let block_reference = near_primitives::types::BlockReference::Finality(finality.clone()); + Ok(client + .send(near_client::GetBlock(block_reference).with_span_context()) + .await??) +} + +pub(crate) async fn fetch_status( + client: &actix::Addr, +) -> anyhow::Result { + tracing::debug!(target: crate::INDEXER, "Fetching status"); + Ok(client + .send( + near_client::Status { + is_health_check: false, + detailed: false, + } + .with_span_context(), + ) + .await??) +} + +/// This function starts a busy-loop that does the similar job to the near-indexer one. +/// However, this one deals with the blocks by provided finality, and instead of streaming them to +/// the client, it stores the block directly to the Redis instance shared between +/// ReadRPC components. +pub async fn update_block_in_redis_by_finality( + view_client: actix::Addr, + client: actix::Addr, + finality_blocks_storage: cache_storage::BlocksByFinalityCache, + finality: near_primitives::types::Finality, +) { + let block_type = serde_json::to_string(&finality).unwrap(); + tracing::info!(target: crate::INDEXER, "Starting [{}] block update job...", block_type); + + let mut last_stored_block_height: Option = None; + loop { + tokio::time::sleep(INTERVAL).await; + + if let Ok(status) = fetch_status(&client).await { + // Update protocol version in Redis + // This is need for read-rpc to know the current protocol version + if let Err(err) = finality_blocks_storage + .update_protocol_version(status.protocol_version) + .await + { + tracing::error!( + target: crate::INDEXER, + "Failed to update protocol version in Redis: {:?}", err + ); + }; + + // If the node is not fully synced the optimistic blocks are outdated + // and are useless for our case. To avoid any misleading in our Redis + // we don't update blocks until the node is fully synced. + if status.sync_info.syncing { + continue; + } + } + + if let Ok(block) = fetch_block_by_finality(&view_client, &finality).await { + let height = block.header.height; + if let Some(block_height) = last_stored_block_height { + if height <= block_height { + continue; + } else { + last_stored_block_height = Some(height); + } + } else { + last_stored_block_height = Some(height); + }; + let response = near_indexer::build_streamer_message(&view_client, block).await; + match response { + Ok(streamer_message) => { + tracing::debug!(target: crate::INDEXER, "[{}] block {:?}", block_type, last_stored_block_height); + if let Err(err) = finality_blocks_storage + .update_block_by_finality( + near_primitives::types::Finality::None, + &streamer_message, + ) + .await + { + tracing::error!( + target: crate::INDEXER, + "Failed to publish [{}] block streamer message: {:?}", block_type, err + ); + }; + } + Err(err) => { + tracing::error!( + target: crate::INDEXER, + "Missing data, skipping block #{}...", height + ); + tracing::error!(target: crate::INDEXER, "{:#?}", err); + } + } + }; + } +} diff --git a/rpc-server/src/config.rs b/rpc-server/src/config.rs index 5999ee51..f3271afc 100644 --- a/rpc-server/src/config.rs +++ b/rpc-server/src/config.rs @@ -20,7 +20,8 @@ pub struct GenesisInfo { impl GenesisInfo { pub async fn get( near_rpc_client: &crate::utils::JsonRpcClient, - fastnear_client: &near_lake_framework::fastnear_client::FastNearClient, + s3_client: &near_lake_framework::s3_fetchers::LakeS3Client, + s3_bucket_name: &str, ) -> Self { tracing::info!("Get genesis config..."); let genesis_config = near_rpc_client @@ -31,21 +32,25 @@ impl GenesisInfo { .await .expect("Error to get genesis config"); - let genesis_block = - near_lake_framework::providers::fastnear::fetchers::fetch_first_block(fastnear_client) - .await; + let genesis_block = near_lake_framework::s3_fetchers::fetch_block( + s3_client, + s3_bucket_name, + genesis_config.genesis_height, + ) + .await + .expect("Error to get genesis block"); Self { genesis_config, - genesis_block_cache: CacheBlock::from(&genesis_block.block), + genesis_block_cache: CacheBlock::from(&genesis_block), } } } #[derive(Clone)] pub struct ServerContext { - /// Fastnear client - pub fastnear_client: near_lake_framework::fastnear_client::FastNearClient, + /// Lake s3 client + pub s3_client: near_lake_framework::s3_fetchers::LakeS3Client, /// Database manager pub db_manager: std::sync::Arc>, /// TransactionDetails storage @@ -56,6 +61,8 @@ pub struct ServerContext { pub genesis_info: GenesisInfo, /// Near rpc client pub near_rpc_client: crate::utils::JsonRpcClient, + /// AWS s3 lake bucket name + pub s3_bucket_name: String, /// Blocks cache pub blocks_cache: std::sync::Arc>, /// Final block info include final_block_cache and current_validators_info @@ -102,17 +109,7 @@ impl ServerContext { let blocks_info_by_finality = std::sync::Arc::new(BlocksInfoByFinality::new(&near_rpc_client, &blocks_cache).await); - let fastnear_client = rpc_server_config - .lake_config - .lake_config( - blocks_info_by_finality - .optimistic_cache_block() - .await - .block_height, - rpc_server_config.general.chain_id.clone(), - ) - .await? - .client(); + let s3_client = rpc_server_config.lake_config.lake_s3_client().await; let tx_details_storage = tx_details_storage::TxDetailsStorage::new( rpc_server_config.tx_details_storage.storage_client().await, @@ -127,7 +124,12 @@ impl ServerContext { }) .ok(); - let genesis_info = GenesisInfo::get(&near_rpc_client, &fastnear_client).await; + let genesis_info = GenesisInfo::get( + &near_rpc_client, + &s3_client, + &rpc_server_config.lake_config.aws_bucket_name, + ) + .await; let default_epoch_config = EpochConfig::from(&genesis_info.genesis_config); let all_epoch_config = AllEpochConfig::new( @@ -152,12 +154,13 @@ impl ServerContext { std::sync::Arc::new(CompiledCodeCache::new(contract_code_cache_size_in_bytes)); Ok(Self { - fastnear_client, + s3_client, db_manager: std::sync::Arc::new(Box::new(db_manager)), tx_details_storage: std::sync::Arc::new(tx_details_storage), tx_cache_storage, genesis_info, near_rpc_client, + s3_bucket_name: rpc_server_config.lake_config.aws_bucket_name.clone(), blocks_cache, blocks_info_by_finality, compiled_contract_code_cache, diff --git a/rpc-server/src/main.rs b/rpc-server/src/main.rs index 545484af..848fb32e 100644 --- a/rpc-server/src/main.rs +++ b/rpc-server/src/main.rs @@ -338,27 +338,63 @@ async fn main() -> anyhow::Result<()> { config::ServerContext::init(rpc_server_config.clone(), near_rpc_client.clone()).await?, ); - // Update final block from fastnear let blocks_cache_clone = std::sync::Arc::clone(&server_context.blocks_cache); let blocks_info_by_finality_clone = std::sync::Arc::clone(&server_context.blocks_info_by_finality); - let fastnear_client = server_context.fastnear_client.clone(); + let near_rpc_client_clone = near_rpc_client.clone(); + + let finality_blocks_storage = + cache_storage::BlocksByFinalityCache::new(rpc_server_config.general.redis_url.to_string()) + .await + .map_err(|err| { + crate::metrics::OPTIMISTIC_UPDATING.set_not_working(); + tracing::warn!("Failed to connect to Redis: {:?}", err); + }) + .ok(); + + // We need to update final block from Redis and Lake + // Because we can't be sure that Redis has the latest block + // And Lake can be used as a backup source + + // Update final block from Redis if Redis is available + if let Some(finality_blocks_storage) = finality_blocks_storage.clone() { + tokio::spawn(async move { + utils::update_final_block_regularly_from_redis( + blocks_cache_clone, + blocks_info_by_finality_clone, + finality_blocks_storage, + near_rpc_client_clone, + ) + .await + }); + } + + // Update final block from Lake + let blocks_cache_clone = std::sync::Arc::clone(&server_context.blocks_cache); + let blocks_info_by_finality_clone = + std::sync::Arc::clone(&server_context.blocks_info_by_finality); tokio::spawn(async move { - utils::update_final_block_regularly( + utils::update_final_block_regularly_from_lake( blocks_cache_clone, blocks_info_by_finality_clone, - fastnear_client, + rpc_server_config, near_rpc_client, ) .await }); - // Update optimistic block from fastnear - let blocks_info_by_finality = std::sync::Arc::clone(&server_context.blocks_info_by_finality); - let fastnear_client = server_context.fastnear_client.clone(); - tokio::spawn(async move { - utils::update_optimistic_block_regularly(blocks_info_by_finality, fastnear_client).await - }); + // Update optimistic block from Redis if Redis is available + if let Some(finality_blocks_storage) = finality_blocks_storage { + let blocks_info_by_finality = + std::sync::Arc::clone(&server_context.blocks_info_by_finality); + tokio::spawn(async move { + utils::update_optimistic_block_regularly( + blocks_info_by_finality, + finality_blocks_storage, + ) + .await + }); + } actix_web::HttpServer::new(move || { let cors = actix_cors::Cors::permissive(); diff --git a/rpc-server/src/metrics.rs b/rpc-server/src/metrics.rs index 7c006cd8..206f3509 100644 --- a/rpc-server/src/metrics.rs +++ b/rpc-server/src/metrics.rs @@ -113,11 +113,6 @@ lazy_static! { "Optimistic updating status. 0: working, 1: not working", ).unwrap(); - pub(crate) static ref CURRENT_PROTOCOL_VERSION: IntGauge = try_create_int_gauge( - "current_protocol_version", - "Current protocol version", - ).unwrap(); - pub(crate) static ref LEGACY_DATABASE_TX_DETAILS: IntCounterVec = register_int_counter_vec( "legacy_database_tx_details", "Total number of calls to the legacy database for transaction details", diff --git a/rpc-server/src/modules/blocks/methods.rs b/rpc-server/src/modules/blocks/methods.rs index 88a5243f..7b7e79c5 100644 --- a/rpc-server/src/modules/blocks/methods.rs +++ b/rpc-server/src/modules/blocks/methods.rs @@ -4,7 +4,7 @@ use near_primitives::views::StateChangeValueView; use crate::config::ServerContext; use crate::modules::blocks::utils::{ - fetch_block_from_cache_or_get, fetch_chunk_from_fastnear, is_matching_change, + fetch_block_from_cache_or_get, fetch_chunk_from_s3, is_matching_change, }; /// `block` rpc method implementation @@ -339,13 +339,14 @@ pub async fn fetch_block( Ok(data.genesis_info.genesis_block_cache.block_height) } }?; - let block_view = near_lake_framework::providers::fastnear::fetchers::fetch_block_or_retry( - &data.fastnear_client, + let block_view = near_lake_framework::s3_fetchers::fetch_block( + &data.s3_client, + &data.s3_bucket_name, block_height, ) .await .map_err(|err| { - tracing::error!("Failed to fetch block from fastnear: {}", err); + tracing::error!("Failed to fetch block from S3: {}", err); near_jsonrpc::primitives::types::blocks::RpcBlockError::UnknownBlock { error_message: format!("BLOCK HEIGHT: {:?}", block_height), } @@ -401,8 +402,13 @@ pub async fn fetch_chunk( ) .map(|block_height_shard_id| (block_height_shard_id.0, block_height_shard_id.1))?, }; - let chunk_view = - fetch_chunk_from_fastnear(&data.fastnear_client, block_height, shard_id).await?; + let chunk_view = fetch_chunk_from_s3( + &data.s3_client, + &data.s3_bucket_name, + block_height, + shard_id, + ) + .await?; // increase block category metrics crate::metrics::increase_request_category_metrics( data, @@ -576,16 +582,27 @@ async fn fetch_shards_by_cache_block( data: &Data, cache_block: crate::modules::blocks::CacheBlock, ) -> anyhow::Result> { - match near_lake_framework::providers::fastnear::fetchers::fetch_streamer_message( - &data.fastnear_client, - cache_block.block_height, - ) - .await - { - Some(streamer_message) => Ok(streamer_message.shards), - None => Err(anyhow::anyhow!( - "Failed to fetch shards for block {}", - cache_block.block_height, - )), - } + let fetch_shards_futures = (0..cache_block.chunks_included) + .collect::>() + .into_iter() + .map(|shard_id| { + near_lake_framework::s3_fetchers::fetch_shard( + &data.s3_client, + &data.s3_bucket_name, + cache_block.block_height, + shard_id, + ) + }); + + futures::future::join_all(fetch_shards_futures) + .await + .into_iter() + .collect::>() + .map_err(|err| { + anyhow::anyhow!( + "Failed to fetch shards for block {} with error: {}", + cache_block.block_height, + err + ) + }) } diff --git a/rpc-server/src/modules/blocks/mod.rs b/rpc-server/src/modules/blocks/mod.rs index c0c90acd..7fb9e714 100644 --- a/rpc-server/src/modules/blocks/mod.rs +++ b/rpc-server/src/modules/blocks/mod.rs @@ -10,6 +10,7 @@ pub struct CacheBlock { pub block_timestamp: u64, pub gas_price: near_primitives::types::Balance, pub latest_protocol_version: near_primitives::types::ProtocolVersion, + pub chunks_included: u64, pub state_root: near_primitives::hash::CryptoHash, pub epoch_id: near_primitives::hash::CryptoHash, } @@ -60,6 +61,7 @@ impl From<&near_primitives::views::BlockView> for CacheBlock { block_timestamp: block.header.timestamp, gas_price: block.header.gas_price, latest_protocol_version: block.header.latest_protocol_version, + chunks_included: block.header.chunks_included, state_root: block.header.prev_state_root, epoch_id: block.header.epoch_id, } @@ -299,13 +301,11 @@ impl BlocksInfoByFinality { let final_block_future = crate::utils::get_final_block(near_rpc_client, false); let optimistic_block_future = crate::utils::get_final_block(near_rpc_client, true); let validators_future = crate::utils::get_current_validators(near_rpc_client); - let protocol_version_future = crate::utils::get_current_protocol_version(near_rpc_client); - let (final_block, optimistic_block, validators, protocol_version) = futures::try_join!( + let (final_block, optimistic_block, validators) = futures::try_join!( final_block_future, optimistic_block_future, validators_future, - protocol_version_future ) .map_err(|err| { tracing::error!("Error to fetch final block info: {:?}", err); @@ -327,7 +327,7 @@ impl BlocksInfoByFinality { optimistic_changes: futures_locks::RwLock::new(OptimisticChanges::new()), current_validators: futures_locks::RwLock::new(CurrentValidatorInfo { validators }), current_protocol_version: futures_locks::RwLock::new(CurrentProtocolVersion { - protocol_version, + protocol_version: near_primitives::version::PROTOCOL_VERSION, }), } } @@ -368,13 +368,18 @@ impl BlocksInfoByFinality { &self, near_rpc_client: &crate::utils::JsonRpcClient, ) -> anyhow::Result<()> { - let current_validators_future = crate::utils::get_current_validators(near_rpc_client); - let current_protocol_version_future = - crate::utils::get_current_protocol_version(near_rpc_client); - let (current_validators, current_protocol_version) = - futures::try_join!(current_validators_future, current_protocol_version_future,)?; - self.current_validators.write().await.validators = current_validators; - self.current_protocol_version.write().await.protocol_version = current_protocol_version; + self.current_validators.write().await.validators = + crate::utils::get_current_validators(near_rpc_client).await?; + Ok(()) + } + + // Update current protocol version in the cache. + // This method executes when the protocol version changes. + pub async fn update_current_protocol_version( + &self, + protocol_version: near_primitives::types::ProtocolVersion, + ) -> anyhow::Result<()> { + self.current_protocol_version.write().await.protocol_version = protocol_version; Ok(()) } diff --git a/rpc-server/src/modules/blocks/utils.rs b/rpc-server/src/modules/blocks/utils.rs index 5679ecdf..8ad3b422 100644 --- a/rpc-server/src/modules/blocks/utils.rs +++ b/rpc-server/src/modules/blocks/utils.rs @@ -6,10 +6,11 @@ use crate::modules::blocks::CacheBlock; #[cfg_attr( feature = "tracing-instrumentation", - tracing::instrument(skip(fastnear_client)) + tracing::instrument(skip(s3_client)) )] -pub async fn fetch_chunk_from_fastnear( - fastnear_client: &near_lake_framework::fastnear_client::FastNearClient, +pub async fn fetch_chunk_from_s3( + s3_client: &near_lake_framework::s3_fetchers::LakeS3Client, + s3_bucket_name: &str, block_height: near_primitives::types::BlockHeight, shard_id: near_primitives::types::ShardId, ) -> Result @@ -19,8 +20,9 @@ pub async fn fetch_chunk_from_fastnear( block_height, shard_id ); - match near_lake_framework::providers::fastnear::fetchers::fetch_shard_or_retry( - fastnear_client, + match near_lake_framework::s3_fetchers::fetch_shard( + s3_client, + s3_bucket_name, block_height, shard_id, ) diff --git a/rpc-server/src/utils.rs b/rpc-server/src/utils.rs index 4b849efe..c6df311b 100644 --- a/rpc-server/src/utils.rs +++ b/rpc-server/src/utils.rs @@ -1,6 +1,7 @@ use crate::modules::blocks::{BlockInfo, BlocksInfoByFinality, CacheBlock}; #[cfg(feature = "shadow-data-consistency")] use assert_json_diff::{assert_json_matches_no_panic, CompareMode, Config, NumericMode}; +use futures::StreamExt; #[cfg(feature = "shadow-data-consistency")] const DEFAULT_RETRY_COUNT: u8 = 3; @@ -145,29 +146,20 @@ pub async fn get_current_validators( Ok(near_rpc_client.call(params, None).await?) } -pub async fn get_current_protocol_version( - near_rpc_client: &JsonRpcClient, -) -> anyhow::Result { - let params = near_jsonrpc_client::methods::status::RpcStatusRequest; - let protocol_version = near_rpc_client.call(params, None).await?.protocol_version; - crate::metrics::CURRENT_PROTOCOL_VERSION.set(protocol_version as i64); - Ok(protocol_version) -} - async fn handle_streamer_message( streamer_message: near_indexer_primitives::StreamerMessage, blocks_cache: std::sync::Arc>, blocks_info_by_finality: std::sync::Arc, near_rpc_client: &JsonRpcClient, ) -> anyhow::Result<()> { - if streamer_message.block.header.height as i64 + let block = BlockInfo::new_from_streamer_message(streamer_message).await; + let block_cache = block.block_cache; + + if block_cache.block_height as i64 > crate::metrics::LATEST_BLOCK_HEIGHT_BY_FINALITIY .with_label_values(&["final"]) .get() { - let block = BlockInfo::new_from_streamer_message(streamer_message).await; - let block_cache = block.block_cache; - if blocks_info_by_finality.final_cache_block().await.epoch_id != block_cache.epoch_id { tracing::info!("New epoch started: {:?}", block_cache.epoch_id); blocks_info_by_finality @@ -186,62 +178,142 @@ async fn handle_streamer_message( Ok(()) } +pub async fn update_final_block_regularly_from_lake( + blocks_cache: std::sync::Arc>, + blocks_info_by_finality: std::sync::Arc, + rpc_server_config: configuration::RpcServerConfig, + near_rpc_client: JsonRpcClient, +) -> anyhow::Result<()> { + tracing::info!("Task to get final block from lake and store in the cache started"); + let lake_config = rpc_server_config + .lake_config + .lake_config( + blocks_info_by_finality + .optimistic_cache_block() + .await + .block_height, + ) + .await?; + let (sender, stream) = near_lake_framework::streamer(lake_config); + let mut handlers = tokio_stream::wrappers::ReceiverStream::new(stream) + .map(|streamer_message| { + handle_streamer_message( + streamer_message, + std::sync::Arc::clone(&blocks_cache), + std::sync::Arc::clone(&blocks_info_by_finality), + &near_rpc_client, + ) + }) + .buffer_unordered(1usize); + + while let Some(_handle_message) = handlers.next().await { + if let Err(err) = _handle_message { + tracing::warn!("{:?}", err); + } + } + drop(handlers); // close the channel so the sender will stop + + // propagate errors from the sender + match sender.await { + Ok(Ok(())) => Ok(()), + Ok(Err(e)) => Err(e), + Err(e) => Err(anyhow::Error::from(e)), // JoinError + } +} + // Task to get and store final block in the cache -pub async fn update_final_block_regularly( +// Subscribe to the redis channel and update the final block in the cache +pub async fn update_final_block_regularly_from_redis( blocks_cache: std::sync::Arc>, blocks_info_by_finality: std::sync::Arc, - fastnear_client: near_lake_framework::fastnear_client::FastNearClient, + finality_blocks_storage: cache_storage::BlocksByFinalityCache, near_rpc_client: JsonRpcClient, ) { tracing::info!("Task to get and store final block in the cache started"); + let mut current_protocol_version = blocks_info_by_finality.current_protocol_version().await; loop { - // Update final block from fastnear regularly - let streamer_message = - near_lake_framework::providers::fastnear::fetchers::fetch_last_block(&fastnear_client) - .await; - if let Err(err) = handle_streamer_message( - streamer_message, - std::sync::Arc::clone(&blocks_cache), - std::sync::Arc::clone(&blocks_info_by_finality), - &near_rpc_client, - ) - .await + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + + // Update protocol version from redis regularly + match finality_blocks_storage.get_protocol_version().await { + Ok(protocol_version) => { + if protocol_version != current_protocol_version { + if let Err(err) = blocks_info_by_finality + .update_current_protocol_version(protocol_version) + .await + { + tracing::error!("Failed to update protocol version from Redis: {:?}", err); + } else { + // If the protocol version is updated from the Redis, update the local value + // otherwise, we will keep trying to update from the Redis + current_protocol_version = protocol_version; + }; + }; + } + Err(err) => { + tracing::error!("Failed to get protocol version from Redis: {:?}", err); + } + } + + // Update final block from Redis regularly + match finality_blocks_storage + .get_block_by_finality(near_primitives::types::Finality::Final) + .await { - tracing::error!("Error in fn handle_streamer_message(): {:?}", err); - }; - // Sleep for 500ms before the next iteration - tokio::time::sleep(std::time::Duration::from_millis(500)).await; + Ok(streamer_message) => { + if let Err(err) = handle_streamer_message( + streamer_message, + std::sync::Arc::clone(&blocks_cache), + std::sync::Arc::clone(&blocks_info_by_finality), + &near_rpc_client, + ) + .await + { + tracing::error!("Error to handle_streamer_message: {:?}", err); + } + } + Err(err) => { + tracing::warn!("Error to get final block from redis: {:?}", err); + } + } } } // Task to get and store optimistic block in the cache +// Subscribe to the redis channel and update the optimistic block in the cache pub async fn update_optimistic_block_regularly( blocks_info_by_finality: std::sync::Arc, - fastnear_client: near_lake_framework::fastnear_client::FastNearClient, + finality_blocks_storage: cache_storage::BlocksByFinalityCache, ) { tracing::info!("Task to get and store optimistic block in the cache started"); loop { - let streamer_message = - near_lake_framework::providers::fastnear::fetchers::fetch_optimistic_block( - &fastnear_client, - ) - .await; - if streamer_message.block.header.height as i64 - > crate::metrics::LATEST_BLOCK_HEIGHT_BY_FINALITIY - .with_label_values(&["optimistic"]) - .get() + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + match finality_blocks_storage + .get_block_by_finality(near_primitives::types::Finality::None) + .await { - let optimistic_block = BlockInfo::new_from_streamer_message(streamer_message).await; - let optimistic_block_cache = optimistic_block.block_cache; - blocks_info_by_finality - .update_optimistic_block(optimistic_block) - .await; - crate::metrics::LATEST_BLOCK_HEIGHT_BY_FINALITIY - .with_label_values(&["optimistic"]) - .set( - i64::try_from(optimistic_block_cache.block_height) - .expect("Invalid optimistic block height"), - ); + Ok(streamer_message) => { + let optimistic_block = BlockInfo::new_from_streamer_message(streamer_message).await; + let optimistic_block_cache = optimistic_block.block_cache; + if optimistic_block_cache.block_height as i64 + > crate::metrics::LATEST_BLOCK_HEIGHT_BY_FINALITIY + .with_label_values(&["optimistic"]) + .get() + { + blocks_info_by_finality + .update_optimistic_block(optimistic_block) + .await; + crate::metrics::LATEST_BLOCK_HEIGHT_BY_FINALITIY + .with_label_values(&["optimistic"]) + .set( + i64::try_from(optimistic_block_cache.block_height) + .expect("Invalid optimistic block height"), + ); + } + } + Err(err) => { + tracing::warn!("Error to get optimistic block from redis: {:?}", err); + } } // When an optimistic block is not updated, or it is lower than the final block @@ -273,8 +345,6 @@ pub async fn update_optimistic_block_regularly( crate::metrics::OPTIMISTIC_UPDATING.set_working(); tracing::info!("Optimistic block updating is resumed."); }; - // Sleep for 500ms before the next iteration - tokio::time::sleep(std::time::Duration::from_millis(500)).await; } } diff --git a/state-indexer/src/main.rs b/state-indexer/src/main.rs index 534e9b2f..df2a3f8d 100644 --- a/state-indexer/src/main.rs +++ b/state-indexer/src/main.rs @@ -36,10 +36,7 @@ async fn main() -> anyhow::Result<()> { ) .await?; - let lake_config = indexer_config - .lake_config - .lake_config(start_block_height, indexer_config.general.chain_id.clone()) - .await?; + let lake_config = indexer_config.lake_config.lake_config(start_block_height).await?; let (sender, stream) = near_lake_framework::streamer(lake_config); // Initiate metrics http server diff --git a/tx-indexer/src/main.rs b/tx-indexer/src/main.rs index c8f8d45b..97d5ffda 100644 --- a/tx-indexer/src/main.rs +++ b/tx-indexer/src/main.rs @@ -63,7 +63,7 @@ async fn main() -> anyhow::Result<()> { tracing::info!(target: INDEXER, "Generating LakeConfig..."); let lake_config = indexer_config .lake_config - .lake_config(start_block_height, indexer_config.general.chain_id.clone()) + .lake_config(start_block_height) .await?; tracing::info!(target: INDEXER, "Creating cache storage...");