diff --git a/Cargo.lock b/Cargo.lock index 1e68eb558b..fa507d4475 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -52,6 +52,19 @@ dependencies = [ "subtle", ] +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "getrandom", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.2" @@ -198,15 +211,6 @@ dependencies = [ "syn 2.0.38", ] -[[package]] -name = "atomic-polyfill" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cf2bce30dfe09ef0bfaef228b9d414faaf7e563035494d7fe092dba54b300f4" -dependencies = [ - "critical-section", -] - [[package]] name = "atty" version = "0.2.14" @@ -1253,6 +1257,29 @@ dependencies = [ "typenum", ] +[[package]] +name = "cssparser" +version = "0.31.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b3df4f93e5fbbe73ec01ec8d3f68bba73107993a5b1e7519273c32db9b0d5be" +dependencies = [ + "cssparser-macros", + "dtoa-short", + "itoa", + "phf 0.11.2", + "smallvec", +] + +[[package]] +name = "cssparser-macros" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" +dependencies = [ + "quote", + "syn 2.0.38", +] + [[package]] name = "csv" version = "1.3.0" @@ -1667,6 +1694,21 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "669a445ee724c5c69b1b06fe0b63e70a1c84bc9bb7d9696cd4f4e3ec45050408" +[[package]] +name = "dtoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcbb2bf8e87535c23f7a8a321e364ce21462d0ff10cb6407820e8e96dfff6653" + +[[package]] +name = "dtoa-short" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbaceec3c6e4211c79e7b1800fb9680527106beb2f9c51904a3210c03a448c74" +dependencies = [ + "dtoa", +] + [[package]] name = "ecdsa" version = "0.16.8" @@ -1704,6 +1746,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "ego-tree" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a68a4904193147e0a8dec3314640e6db742afd5f6e634f428a6af230d9b3591" + [[package]] name = "either" version = "1.9.0" @@ -1961,6 +2009,16 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" +[[package]] +name = "futf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843" +dependencies = [ + "mac", + "new_debug_unreachable", +] + [[package]] name = "futures" version = "0.1.31" @@ -2074,6 +2132,15 @@ dependencies = [ "slab", ] +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -2085,6 +2152,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "getopts" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" +dependencies = [ + "unicode-width", +] + [[package]] name = "getrandom" version = "0.2.10" @@ -2344,6 +2420,20 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "html5ever" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bea68cab48b8459f17cf1c944c67ddc572d272d9f2b274140f223ecb1da4a3b7" +dependencies = [ + "log", + "mac", + "markup5ever", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "http" version = "0.2.9" @@ -2975,6 +3065,26 @@ dependencies = [ "winapi", ] +[[package]] +name = "mac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" + +[[package]] +name = "markup5ever" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2629bb1404f3d34c2e921f21fd34ba00b206124c81f65c50b43b6aaefeb016" +dependencies = [ + "log", + "phf 0.10.1", + "phf_codegen", + "string_cache", + "string_cache_codegen", + "tendril", +] + [[package]] name = "matchers" version = "0.1.0" @@ -3241,12 +3351,14 @@ dependencies = [ "hyper", "jsonrpc", "log", + "markup5ever", "minotari_app_grpc", "minotari_app_utilities", "minotari_node_grpc_client", "minotari_wallet_grpc_client", "monero", "reqwest", + "scraper", "serde", "serde_json", "tari_common", @@ -3594,6 +3706,12 @@ dependencies = [ "tempfile", ] +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + [[package]] name = "newtype-ops" version = "0.1.4" @@ -3788,12 +3906,12 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" dependencies = [ - "atomic-polyfill", "critical-section", + "portable-atomic", ] [[package]] @@ -4177,6 +4295,86 @@ dependencies = [ "zeroize", ] +[[package]] +name = "phf" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabbf1ead8a5bcbc20f5f8b939ee3f5b0f6f281b6ad3468b84656b658b455259" +dependencies = [ + "phf_shared 0.10.0", +] + +[[package]] +name = "phf" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" +dependencies = [ + "phf_macros", + "phf_shared 0.11.2", +] + +[[package]] +name = "phf_codegen" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb1c3a8bc4dd4e5cfce29b44ffc14bedd2ee294559a294e2a4d4c9e9a6a13cd" +dependencies = [ + "phf_generator 0.10.0", + "phf_shared 0.10.0", +] + +[[package]] +name = "phf_generator" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6" +dependencies = [ + "phf_shared 0.10.0", + "rand", +] + +[[package]] +name = "phf_generator" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" +dependencies = [ + "phf_shared 0.11.2", + "rand", +] + +[[package]] +name = "phf_macros" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b" +dependencies = [ + "phf_generator 0.11.2", + "phf_shared 0.11.2", + "proc-macro2", + "quote", + "syn 2.0.38", +] + +[[package]] +name = "phf_shared" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" +dependencies = [ + "siphasher", +] + +[[package]] +name = "phf_shared" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project" version = "0.4.30" @@ -4313,6 +4511,12 @@ dependencies = [ "universal-hash", ] +[[package]] +name = "portable-atomic" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" + [[package]] name = "powerfmt" version = "0.2.0" @@ -4325,6 +4529,12 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + [[package]] name = "prettyplease" version = "0.1.25" @@ -5018,6 +5228,22 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scraper" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b80b33679ff7a0ea53d37f3b39de77ea0c75b12c5805ac43ec0c33b3051af1b" +dependencies = [ + "ahash", + "cssparser", + "ego-tree", + "getopts", + "html5ever", + "once_cell", + "selectors", + "tendril", +] + [[package]] name = "sct" version = "0.7.1" @@ -5077,6 +5303,25 @@ dependencies = [ "libc", ] +[[package]] +name = "selectors" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4eb30575f3638fc8f6815f448d50cb1a2e255b0897985c8c59f4d37b72a07b06" +dependencies = [ + "bitflags 2.4.1", + "cssparser", + "derive_more", + "fxhash", + "log", + "new_debug_unreachable", + "phf 0.10.1", + "phf_codegen", + "precomputed-hash", + "servo_arc", + "smallvec", +] + [[package]] name = "semver" version = "1.0.20" @@ -5169,6 +5414,15 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "servo_arc" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d036d71a959e00c77a63538b90a6c2390969f9772b096ea837205c6bd0491a44" +dependencies = [ + "stable_deref_trait", +] + [[package]] name = "sha1" version = "0.6.0" @@ -5276,6 +5530,12 @@ dependencies = [ "rand_core", ] +[[package]] +name = "siphasher" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" + [[package]] name = "slab" version = "0.4.9" @@ -5388,6 +5648,12 @@ dependencies = [ "der", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "stack-buf" version = "0.1.6" @@ -5406,6 +5672,32 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e08d8363704e6c71fc928674353e6b7c23dcea9d82d7012c8faf2a3a025f8d0" +[[package]] +name = "string_cache" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f91138e76242f575eb1d3b38b4f1362f10d3a43f47d182a5b359af488a02293b" +dependencies = [ + "new_debug_unreachable", + "once_cell", + "parking_lot 0.12.1", + "phf_shared 0.10.0", + "precomputed-hash", + "serde", +] + +[[package]] +name = "string_cache_codegen" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988" +dependencies = [ + "phf_generator 0.10.0", + "phf_shared 0.10.0", + "proc-macro2", + "quote", +] + [[package]] name = "strip-ansi-escapes" version = "0.2.0" @@ -6250,6 +6542,17 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "tendril" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0" +dependencies = [ + "futf", + "mac", + "utf-8", +] + [[package]] name = "termcolor" version = "1.3.0" @@ -6995,6 +7298,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + [[package]] name = "utf8parse" version = "0.2.1" @@ -7510,6 +7819,26 @@ dependencies = [ "time", ] +[[package]] +name = "zerocopy" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.38", +] + [[package]] name = "zeroize" version = "1.6.0" diff --git a/applications/minotari_merge_mining_proxy/Cargo.toml b/applications/minotari_merge_mining_proxy/Cargo.toml index 8440579378..b058c4eeae 100644 --- a/applications/minotari_merge_mining_proxy/Cargo.toml +++ b/applications/minotari_merge_mining_proxy/Cargo.toml @@ -44,6 +44,10 @@ tokio = { version = "1.36", features = ["macros"] } tonic = "0.8.3" tracing = "0.1" url = "2.1.1" +scraper = "0.19.0" [build-dependencies] tari_features = { path = "../../common/tari_features", version = "1.0.0-pre.11a"} + +[dev-dependencies] +markup5ever = "0.11.0" \ No newline at end of file diff --git a/applications/minotari_merge_mining_proxy/log4rs_sample.yml b/applications/minotari_merge_mining_proxy/log4rs_sample.yml index 9b56009301..7cbebfcec3 100644 --- a/applications/minotari_merge_mining_proxy/log4rs_sample.yml +++ b/applications/minotari_merge_mining_proxy/log4rs_sample.yml @@ -55,4 +55,15 @@ loggers: - stdout - proxy additive: false - \ No newline at end of file + html5ever: + level: error + appenders: + - stdout + - proxy + additive: false + selectors: + level: error + appenders: + - stdout + - proxy + additive: false diff --git a/applications/minotari_merge_mining_proxy/src/block_template_data.rs b/applications/minotari_merge_mining_proxy/src/block_template_data.rs index e0c2ecc52d..fa90b116e2 100644 --- a/applications/minotari_merge_mining_proxy/src/block_template_data.rs +++ b/applications/minotari_merge_mining_proxy/src/block_template_data.rs @@ -20,7 +20,7 @@ // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE // USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -//! Provides methods for for building template data and storing them with timestamps. +//! Provides methods for building template data and storing them with timestamps. use std::{collections::HashMap, convert::TryFrom, sync::Arc}; diff --git a/applications/minotari_merge_mining_proxy/src/config.rs b/applications/minotari_merge_mining_proxy/src/config.rs index d9418a5554..0f8db4b9a4 100644 --- a/applications/minotari_merge_mining_proxy/src/config.rs +++ b/applications/minotari_merge_mining_proxy/src/config.rs @@ -32,12 +32,23 @@ use tari_common_types::tari_address::TariAddress; use tari_comms::multiaddr::Multiaddr; use tari_core::transactions::transaction_components::RangeProofType; +// The default Monero fail URL for mainnet +const MONERO_FAIL_MAINNET_URL: &str = "https://monero.fail/?chain=monero&network=mainnet&all=true"; + #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] #[allow(clippy::struct_excessive_bools)] pub struct MergeMiningProxyConfig { override_from: Option, - /// URL to monerod + /// Use dynamic monerod URL obtained form the official Monero website (https://monero.fail/) + pub use_dynamic_fail_data: bool, + /// The monero fail URL to get the monerod URLs from - must be pointing to the official Monero website. + /// Valid alternatives are: + /// - mainnet: 'https://monero.fail/?chain=monero&network=mainnet&all=true' + /// - stagenet: `https://monero.fail/?chain=monero&network=stagenet&all=true` + /// - testnet: `https://monero.fail/?chain=monero&network=testnet&all=true` + pub monero_fail_url: String, + /// URL to monerod (you can add your own server here or use public nodes from https://monero.fail/) pub monerod_url: StringList, /// Username for curl pub monerod_username: String, @@ -89,6 +100,8 @@ impl Default for MergeMiningProxyConfig { fn default() -> Self { Self { override_from: None, + use_dynamic_fail_data: true, + monero_fail_url: MONERO_FAIL_MAINNET_URL.into(), monerod_url: StringList::default(), monerod_username: String::new(), monerod_password: String::new(), diff --git a/applications/minotari_merge_mining_proxy/src/error.rs b/applications/minotari_merge_mining_proxy/src/error.rs index fe58c43f3a..afd2d4b56a 100644 --- a/applications/minotari_merge_mining_proxy/src/error.rs +++ b/applications/minotari_merge_mining_proxy/src/error.rs @@ -77,6 +77,8 @@ pub enum MmProxyError { }, #[error("HTTP error: {0}")] HttpError(#[from] hyper::http::Error), + #[error("HTML parse error: {0}")] + HtmlParseError(String), #[error("Could not parse URL: {0}")] UrlParseError(#[from] url::ParseError), #[error("Bincode error: {0}")] diff --git a/applications/minotari_merge_mining_proxy/src/lib.rs b/applications/minotari_merge_mining_proxy/src/lib.rs index 9674852c82..95e975c245 100644 --- a/applications/minotari_merge_mining_proxy/src/lib.rs +++ b/applications/minotari_merge_mining_proxy/src/lib.rs @@ -33,6 +33,7 @@ mod error; mod proxy; mod run_merge_miner; use run_merge_miner::start_merge_miner; +mod monero_fail; pub async fn merge_miner(cli: Cli) -> Result<(), anyhow::Error> { start_merge_miner(cli).await diff --git a/applications/minotari_merge_mining_proxy/src/main.rs b/applications/minotari_merge_mining_proxy/src/main.rs index da4bf48e13..30c1943a74 100644 --- a/applications/minotari_merge_mining_proxy/src/main.rs +++ b/applications/minotari_merge_mining_proxy/src/main.rs @@ -28,6 +28,7 @@ mod cli; mod common; mod config; mod error; +mod monero_fail; mod proxy; mod run_merge_miner; diff --git a/applications/minotari_merge_mining_proxy/src/monero_fail.rs b/applications/minotari_merge_mining_proxy/src/monero_fail.rs new file mode 100644 index 0000000000..1065ce1c80 --- /dev/null +++ b/applications/minotari_merge_mining_proxy/src/monero_fail.rs @@ -0,0 +1,372 @@ +// Copyright 2020, The Tari Project +// +// Redistribution and use in source and binary forms, with or without modification, are permitted provided that the +// following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following +// disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the +// following disclaimer in the documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote +// products derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +// USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +use std::time::Duration; + +use log::*; +use scraper::{Html, Selector}; +use tokio::time::timeout; +use url::Url; + +use crate::error::MmProxyError; + +const LOG_TARGET: &str = "minotari_mm_proxy::monero_detect"; + +/// Monero public server information +#[derive(Debug)] +pub struct MonerodEntry { + /// The type of address + pub address_type: String, + /// The URL of the server + pub url: String, + /// The monero blockchain height reported by the server + pub height: u64, + /// Whether the server is currently up + pub up: bool, + /// Whether the server is web compatible + pub web_compatible: bool, + /// The network the server is on (mainnet, stagenet, testnet) + pub network: String, + /// Time since the server was checked + pub last_checked: String, + /// The history of the server being up + pub up_history: Vec, + /// Response time + pub response_time: Option, +} + +/// Get the latest monerod public nodes (by scraping the HTML frm the monero.fail website) that are +/// currently up and has a full history of being up all the time. +#[allow(clippy::too_many_lines)] +pub async fn get_monerod_info( + number_of_entries: usize, + connection_test_timeout: Duration, + monero_fail_url: &str, +) -> Result, MmProxyError> { + let document = get_monerod_html(monero_fail_url).await?; + + // The HTML table definition and an example entry looks like this: + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + + // Define selectors for table elements + let row_selector = + Selector::parse("tr.js-sort-table").map_err(|e| MmProxyError::HtmlParseError(format!("{}", e)))?; + let type_selector = + Selector::parse("td:nth-child(1)").map_err(|e| MmProxyError::HtmlParseError(format!("{}", e)))?; + let url_selector = + Selector::parse("td:nth-child(2) .nodeURL").map_err(|e| MmProxyError::HtmlParseError(format!("{}", e)))?; + let height_selector = + Selector::parse("td:nth-child(3)").map_err(|e| MmProxyError::HtmlParseError(format!("{}", e)))?; + let up_selector = Selector::parse("td:nth-child(4) .dot.glowing-green, td:nth-child(4) .dot.glowing-red") + .map_err(|e| MmProxyError::HtmlParseError(format!("{}", e)))?; + let web_compatible_selector = Selector::parse("td:nth-child(5) img.filter-green, td:nth-child(5) img.filter-red") + .map_err(|e| MmProxyError::HtmlParseError(format!("{}", e)))?; + let network_selector = + Selector::parse("td:nth-child(6)").map_err(|e| MmProxyError::HtmlParseError(format!("{}", e)))?; + let last_checked_selector = + Selector::parse("td:nth-child(7)").map_err(|e| MmProxyError::HtmlParseError(format!("{}", e)))?; + let history_selector = Selector::parse("td:nth-child(8) .dot.glowing-green, td:nth-child(8) .dot.glowing-red") + .map_err(|e| MmProxyError::HtmlParseError(format!("{}", e)))?; + + let mut entries = Vec::new(); + + // Iterate over table rows and extract data + for row in document.select(&row_selector) { + let address_type = match row.select(&type_selector).next() { + Some(val) => val.text().collect::().trim().to_string(), + None => return Err(MmProxyError::HtmlParseError("address type".to_string())), + }; + + let url = match row.select(&url_selector).next() { + Some(val) => val.text().collect::().trim().to_string(), + None => return Err(MmProxyError::HtmlParseError("url".to_string())), + }; + + let height = match row.select(&height_selector).next() { + Some(val) => val.text().collect::().trim().parse::().unwrap_or_default(), + None => return Err(MmProxyError::HtmlParseError("height".to_string())), + }; + + let mut up = false; + let iter = row.select(&up_selector); + for item in iter { + let class = item.value().attr("class").unwrap_or(""); + if class.contains("dot glowing-green") { + up = true; + break; + } + } + + let mut web_compatible = false; + let iter = row.select(&web_compatible_selector); + for item in iter { + let class = item.value().attr("class").unwrap_or(""); + if class.contains("filter-green") { + web_compatible = true; + break; + } + } + + let network = match row.select(&network_selector).next() { + Some(val) => val.text().collect::().trim().to_string(), + None => return Err(MmProxyError::HtmlParseError("network".to_string())), + }; + + let last_checked = match row.select(&last_checked_selector).next() { + Some(val) => val.text().collect::().trim().to_string(), + None => return Err(MmProxyError::HtmlParseError("last checked".to_string())), + }; + + let mut up_history = Vec::new(); + let iter = row.select(&history_selector); + for item in iter { + let class = item.value().attr("class").unwrap_or(""); + up_history.push(class.contains("dot glowing-green")); + } + + let entry = MonerodEntry { + address_type: address_type.to_lowercase(), + url, + height, + up, + web_compatible, + network: network.to_lowercase(), + last_checked, + up_history, + response_time: None, + }; + entries.push(entry); + } + + // Only retain nodes that are currently up and has a full history of being up all the time + let max_history_length = entries.iter().map(|entry| entry.up_history.len()).max().unwrap_or(0); + entries.retain(|entry| { + entry.up && entry.up_history.iter().filter(|&&v| v).collect::>().len() == max_history_length + }); + // Only retain non-tor and non-i2p nodes + entries.retain(|entry| entry.address_type != *"tor" && entry.address_type != *"i2p"); + // Give preference to nodes with the best height + entries.sort_by(|a, b| b.height.cmp(&a.height)); + // Determine connection times - use slightly more nodes than requested + entries.truncate(number_of_entries + 10); + for entry in &mut entries { + let uri = format!("{}/getheight", entry.url).parse::()?; + let start = std::time::Instant::now(); + if (timeout(connection_test_timeout, reqwest::get(uri.clone())).await).is_ok() { + entry.response_time = Some(start.elapsed()); + debug!(target: LOG_TARGET, "Response time '{:.2?}' for Monerod server at: {}", entry.response_time, uri.as_str()); + } else { + debug!(target: LOG_TARGET, "Response time 'n/a' for Monerod server at: {}, timed out", uri.as_str()); + } + } + // Sort by response time + entries.sort_by(|a, b| { + a.response_time + .unwrap_or_else(|| Duration::from_secs(100)) + .cmp(&b.response_time.unwrap_or_else(|| Duration::from_secs(100))) + }); + // Truncate to the requested number of entries + entries.truncate(number_of_entries); + + if entries.is_empty() { + return Err(MmProxyError::HtmlParseError( + "No public monero servers available".to_string(), + )); + } + Ok(entries) +} + +async fn get_monerod_html(url: &str) -> Result { + let body = match reqwest::get(url).await { + Ok(resp) => match resp.text().await { + Ok(html) => html, + Err(e) => { + error!("Failed to fetch monerod info: {}", e); + return Err(MmProxyError::MonerodRequestFailed(e)); + }, + }, + Err(e) => { + error!("Failed to fetch monerod info: {}", e); + return Err(MmProxyError::MonerodRequestFailed(e)); + }, + }; + + Ok(Html::parse_document(&body)) +} + +#[cfg(test)] +mod test { + use std::{ops::Deref, time::Duration}; + + use markup5ever::{local_name, namespace_url, ns, QualName}; + use scraper::Html; + + use crate::{ + config::MergeMiningProxyConfig, + monero_fail::{get_monerod_html, get_monerod_info}, + }; + + #[tokio::test] + async fn test_get_monerod_info() { + // Monero mainnet + let config = MergeMiningProxyConfig::default(); + let entries = get_monerod_info(5, Duration::from_secs(2), &config.monero_fail_url) + .await + .unwrap(); + for (i, entry) in entries.iter().enumerate() { + assert!(entry.up && entry.up_history.iter().all(|&v| v)); + if i > 0 { + assert!( + entry.response_time.unwrap_or_else(|| Duration::from_secs(100)) >= + entries[i - 1].response_time.unwrap_or_else(|| Duration::from_secs(100)) + ); + } + println!("{}: {:?}", i, entry); + } + + // Monero stagenet + const MONERO_FAIL_STAGNET_URL: &str = "https://monero.fail/?chain=monero&network=stagenet&all=true"; + let entries = get_monerod_info(5, Duration::from_secs(2), MONERO_FAIL_STAGNET_URL) + .await + .unwrap(); + for (i, entry) in entries.iter().enumerate() { + assert!(entry.up && entry.up_history.iter().all(|&v| v)); + if i > 0 { + assert!( + entry.response_time.unwrap_or_else(|| Duration::from_secs(100)) >= + entries[i - 1].response_time.unwrap_or_else(|| Duration::from_secs(100)) + ); + } + println!("{}: {:?}", i, entry); + } + + // Monero testnet + const MONERO_FAIL_TESTNET_URL: &str = "https://monero.fail/?chain=monero&network=testnet&all=true"; + let entries = get_monerod_info(5, Duration::from_secs(2), MONERO_FAIL_TESTNET_URL) + .await + .unwrap(); + for (i, entry) in entries.iter().enumerate() { + assert!(entry.up && entry.up_history.iter().all(|&v| v)); + if i > 0 { + assert!( + entry.response_time.unwrap_or_else(|| Duration::from_secs(100)) >= + entries[i - 1].response_time.unwrap_or_else(|| Duration::from_secs(100)) + ); + } + println!("{}: {:?}", i, entry); + } + } + + #[tokio::test] + async fn test_table_structure() { + let config = MergeMiningProxyConfig::default(); + let html_content = get_monerod_html(&config.monero_fail_url).await.unwrap(); + + let table_structure = extract_table_structure(&html_content); + + let expected_structure = vec![ + "Type", + "URL", + "Height", + "Up", + "Web", + "Compatible", + "Network", + "Last Checked", + "History", + ]; + + // Compare the actual and expected table structures + assert_eq!(table_structure, expected_structure); + } + + // Function to extract table structure from the document + fn extract_table_structure(html_document: &Html) -> Vec<&str> { + let mut table_structure = Vec::new(); + if let Some(table) = html_document.tree.root().descendants().find(|n| { + n.value().is_element() && + n.value().as_element().unwrap().name == QualName::new(None, ns!(html), local_name!("table")) + }) { + if let Some(thead) = table.descendants().find(|n| { + n.value().is_element() && + n.value().as_element().unwrap().name == QualName::new(None, ns!(html), local_name!("thead")) + }) { + if let Some(tr) = thead.descendants().find(|n| { + n.value().is_element() && + n.value().as_element().unwrap().name == QualName::new(None, ns!(html), local_name!("tr")) + }) { + for th in tr.descendants().filter(|n| { + n.value().is_element() && + n.value().as_element().unwrap().name == QualName::new(None, ns!(html), local_name!("th")) + }) { + for child in th.children() { + if let Some(text) = child.value().as_text() { + table_structure.push(text.deref().trim()); + } + } + } + } + } + } + table_structure + } +} diff --git a/applications/minotari_merge_mining_proxy/src/proxy.rs b/applications/minotari_merge_mining_proxy/src/proxy.rs index 6e55ca1c36..ccbcb7c6de 100644 --- a/applications/minotari_merge_mining_proxy/src/proxy.rs +++ b/applications/minotari_merge_mining_proxy/src/proxy.rs @@ -31,7 +31,7 @@ use std::{ RwLock, }, task::{Context, Poll}, - time::Instant, + time::{Duration, Instant}, }; use borsh::BorshSerialize; @@ -49,6 +49,7 @@ use tari_core::{ proof_of_work::{monero_rx, monero_rx::FixedByteArray, randomx_difficulty, randomx_factory::RandomXFactory}, }; use tari_utilities::hex::Hex; +use tokio::time::timeout; use tracing::{debug, error, info, instrument, trace, warn}; use crate::{ @@ -653,7 +654,8 @@ impl InnerService { for next_url in iter { let uri = format!("{}{}", next_url, uri.path()).parse::()?; - match reqwest::get(uri.clone()).await { + debug!(target: LOG_TARGET, "Trying to connect to Monerod server at: {}", uri.as_str()); + match timeout(Duration::from_secs(10), reqwest::get(uri.clone())).await { Ok(_) => { let mut lock = self.current_monerod_server.write().expect("Write lock should not fail"); *lock = Some(next_url.to_string()); @@ -662,11 +664,11 @@ impl InnerService { .write() .expect("Write lock should not fail"); *lock = Some(next_url.to_string()); - info!(target: LOG_TARGET, "Monerod server available: {:?}", uri.clone()); + info!(target: LOG_TARGET, "Monerod server available: {}", uri.as_str()); return Ok(uri); }, Err(_) => { - warn!(target: LOG_TARGET, "Monerod server unavailable: {:?}", uri); + warn!(target: LOG_TARGET, "Monerod server unavailable: {}", uri.as_str()); }, } } diff --git a/applications/minotari_merge_mining_proxy/src/run_merge_miner.rs b/applications/minotari_merge_mining_proxy/src/run_merge_miner.rs index ec076c94b6..34ceb2a856 100644 --- a/applications/minotari_merge_mining_proxy/src/run_merge_miner.rs +++ b/applications/minotari_merge_mining_proxy/src/run_merge_miner.rs @@ -34,7 +34,7 @@ use minotari_app_utilities::parse_miner_input::{ }; use minotari_node_grpc_client::{grpc, grpc::base_node_client::BaseNodeClient}; use minotari_wallet_grpc_client::ClientAuthenticationInterceptor; -use tari_common::{load_configuration, DefaultConfigLoader}; +use tari_common::{configuration::StringList, load_configuration, DefaultConfigLoader}; use tari_comms::utils::multiaddr::multiaddr_to_socketaddr; use tari_core::proof_of_work::randomx_factory::RandomXFactory; use tokio::time::Duration; @@ -44,6 +44,7 @@ use crate::{ block_template_data::BlockTemplateRepository, config::MergeMiningProxyConfig, error::MmProxyError, + monero_fail::get_monerod_info, proxy::MergeMiningProxyService, Cli, }; @@ -55,6 +56,12 @@ pub async fn start_merge_miner(cli: Cli) -> Result<(), anyhow::Error> { let cfg = load_configuration(&config_path, true, cli.non_interactive_mode, &cli)?; let mut config = MergeMiningProxyConfig::load_from(&cfg)?; config.set_base_path(cli.common.get_base_path()); + if config.use_dynamic_fail_data { + let entries = get_monerod_info(15, Duration::from_secs(5), &config.monero_fail_url).await?; + if !entries.is_empty() { + config.monerod_url = StringList::from(entries.into_iter().map(|entry| entry.url).collect::>()); + } + } info!(target: LOG_TARGET, "Configuration: {:?}", config); let client = reqwest::Client::builder() diff --git a/common/config/presets/f_merge_mining_proxy.toml b/common/config/presets/f_merge_mining_proxy.toml index 0083fd4921..fdaccccf53 100644 --- a/common/config/presets/f_merge_mining_proxy.toml +++ b/common/config/presets/f_merge_mining_proxy.toml @@ -7,24 +7,37 @@ [merge_mining_proxy] -# URL to monerod (default = "") + +# Use dynamic monerod URL obtained form the official Monero website (https://monero.fail/) (default: true) +#use_dynamic_fail_data = true + +# The monero fail URL to get the monerod URLs from - must be pointing to the official Monero website. +# Valid alternatives are: +# - mainnet: 'https://monero.fail/?chain=monero&network=mainnet&all=true' (default) +# - stagenet: `https://monero.fail/?chain=monero&network=stagenet&all=true` +# - testnet: `https://monero.fail/?chain=monero&network=testnet&all=true` +#monero_fail_url = "https://monero.fail/?chain=monero&network=mainnet&all=true" + +# URL to monerod (you can add your own server here or use public nodes from https://monero.fail/), only if +# 'use_dynamic_fail_data = false' (default = "") + #monerod_url = [# stagenet # "http://stagenet.xmr-tw.org:38081", -# "http://stagenet.community.xmr.to:38081", -# "http://monero-stagenet.exan.tech:38081", +# "http://node.monerodevs.org:38089", +# "http://node3.monerodevs.org:38089", # "http://xmr-lux.boldsuck.org:38081", # "http://singapore.node.xmr.pm:38081", #] monerod_url = [ # mainnet - # more reliable - "http://xmr.support:18081", "http://node1.xmr-tw.org:18081", + "https://monero.homeqloud.com:443", + "http://monero1.com:18089", + "http://node.c3pool.org:18081", + "http://xmr-full.p2pool.uk:18089", + "https://monero.stackwallet.com:18081", + "http://xmr.support:18081", "http://xmr.nthrow.nyc:18081", - # not so reliable - "http://node.xmrig.com:18081", - "http://monero.exan.tech:18081", - "http://18.132.124.81:18081", ] # Username for curl. (default = "") diff --git a/integration_tests/src/merge_mining_proxy.rs b/integration_tests/src/merge_mining_proxy.rs index 87e7efa4e8..26663bab8a 100644 --- a/integration_tests/src/merge_mining_proxy.rs +++ b/integration_tests/src/merge_mining_proxy.rs @@ -117,8 +117,8 @@ impl MergeMiningProxyProcess { "merge_mining_proxy.monerod_url".to_string(), [ "http://stagenet.xmr-tw.org:38081", - "http://stagenet.community.xmr.to:38081", - "http://monero-stagenet.exan.tech:38081", + "http://node.monerodevs.org:38089", + "http://node3.monerodevs.org:38089", "http://xmr-lux.boldsuck.org:38081", "http://singapore.node.xmr.pm:38081", ] @@ -140,6 +140,10 @@ impl MergeMiningProxyProcess { wallet_payment_address.to_hex(), ), ("merge_mining_proxy.stealth_payment".to_string(), stealth.to_string()), + ( + "merge_mining_proxy.use_dynamic_fail_data".to_string(), + "false".to_string(), + ), ], }, non_interactive_mode: false,
TypeURLHeightUpWeb
Compatible		NetworkLast CheckedHistory
+ // + // + // + // http://node.liumin.io:18089 + // 3119644 + // + // + // + // mainnet5 hours ago + // + // + // + // + // + // + //