From 8d8605c59037edb564fe686d6e101f2e632f7d89 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 17 Feb 2025 11:19:11 -0600 Subject: [PATCH 1/9] GH-1155 Track connections by connection id instead of connection pointer --- plugins/net_plugin/net_plugin.cpp | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index bd649a3d35..61cd452291 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -288,6 +288,7 @@ namespace eosio { struct connection_detail { std::string host; connection_ptr c; + uint32_t connection_id() const; }; using connection_details_index = multi_index_container< @@ -298,8 +299,8 @@ namespace eosio { key<&connection_detail::host> >, ordered_unique< - tag, - key<&connection_detail::c> + tag, + const_mem_fun > > >; @@ -1090,6 +1091,8 @@ namespace eosio { } }; // class connection + uint32_t connections_manager::connection_detail::connection_id() const { return c->connection_id; } + const string connection::unknown = ""; // called from connection strand @@ -3881,6 +3884,7 @@ namespace eosio { const auto& tid = trx->id(); peer_dlog( this, "received packed_transaction ${id}", ("id", tid) ); + ++unique_trxs_rcvd_count; size_t trx_size = calc_trx_size( trx ); trx_in_progress_size += trx_size; @@ -4870,7 +4874,7 @@ namespace eosio { vector result; { std::shared_lock g( connections_mtx ); - auto& index = connections.get(); + auto& index = connections.get(); result.reserve( index.size() ); conns.reserve( index.size() ); for( const connection_detail& cd : index ) { @@ -4930,17 +4934,17 @@ namespace eosio { } } std::scoped_lock g( connections_mtx ); - auto& index = connections.get(); + auto& index = connections.get(); for( auto& c : removing ) { - index.erase(c); + index.erase(c->connection_id); } }; auto max_time = fc::time_point::now().safe_add(max_cleanup_time); std::vector reconnecting, removing; auto from = from_connection.lock(); std::unique_lock g( connections_mtx ); - auto& index = connections.get(); - auto it = (from ? index.find(from) : index.begin()); + auto& index = connections.get(); + auto it = (from ? index.find(from->connection_id) : index.begin()); if (it == index.end()) it = index.begin(); while (it != index.end()) { if (fc::time_point::now() >= max_time) { @@ -4990,7 +4994,7 @@ namespace eosio { assert(update_p2p_connection_metrics); auto from = from_connection.lock(); std::shared_lock g(connections_mtx); - const auto& index = connections.get(); + const auto& index = connections.get(); size_t num_clients = 0, num_peers = 0, num_bp_peers = 0; net_plugin::p2p_per_connection_metrics per_connection(index.size()); for (auto it = index.begin(); it != index.end(); ++it) { From e8c6a743c87911a95092edae260b309cc0507672 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 17 Feb 2025 11:33:24 -0600 Subject: [PATCH 2/9] GH-1155 Add unique_trxs_rcvd_count --- plugins/net_plugin/net_plugin.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 61cd452291..bcb17ecf99 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -851,6 +851,7 @@ namespace eosio { std::atomic peer_fork_db_head_block_num{0}; std::atomic last_received_block_num{0}; std::atomic unique_blocks_rcvd_count{0}; + std::atomic unique_trxs_rcvd_count{0}; std::atomic bytes_received{0}; std::atomic last_bytes_received{0ns}; std::atomic bytes_sent{0}; From ed8c5aef3df9fb9b0b7da12bb60c925137b0a0da Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Tue, 18 Feb 2025 08:06:17 -0600 Subject: [PATCH 3/9] GH-1155 Track unique votes received --- plugins/net_plugin/net_plugin.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index bcb17ecf99..47fb0241dc 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -361,6 +361,8 @@ namespace eosio { string disconnect(const string& host); void close_all(); + connection_ptr find_connection(uint32_t connection_id) const; + std::optional status(const string& host) const; vector connection_statuses() const; @@ -903,6 +905,7 @@ namespace eosio { std::atomic is_bp_connection = false; block_status_monitor block_status_monitor_; std::atomic last_vote_received; + std::atomic unique_votes_rcvd_count{0}; alignas(hardware_destructive_interference_sz) fc::mutex sync_response_expected_timer_mtx; @@ -4122,6 +4125,8 @@ namespace eosio { switch( status ) { case vote_result_t::success: bcast_vote_message(connection_id, msg); + if (auto c = my_impl->connections.find_connection(connection_id)) + ++c->unique_votes_rcvd_count; break; case vote_result_t::unknown_public_key: case vote_result_t::invalid_signature: @@ -4704,6 +4709,14 @@ namespace eosio { return connections.size(); } + connection_ptr connections_manager::find_connection(uint32_t connection_id) const { + std::shared_lock g(connections_mtx); + const auto& index = connections.get(); + if (auto i = index.find(connection_id); i != index.end()) + return i->c; + return {}; + } + void connections_manager::add_supplied_peers(const vector& peers ) { std::lock_guard g(connections_mtx); supplied_peers.insert( peers.begin(), peers.end() ); From d1ff6a54edcc001461ae40c7dba8fecd174774ff Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Tue, 18 Feb 2025 08:06:38 -0600 Subject: [PATCH 4/9] GH-1155 Add p2p-peer-limit option --- plugins/net_plugin/net_plugin.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 47fb0241dc..e8309bb0cf 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -422,6 +422,7 @@ namespace eosio { bool p2p_disable_block_nack = false; bool p2p_accept_votes = true; fc::microseconds p2p_dedup_cache_expire_time_us{}; + uint16_t p2p_peer_limit = 6; chain_id_type chain_id; fc::sha256 node_id; @@ -4336,6 +4337,8 @@ namespace eosio { " p2p.eos.io:9876\n" " p2p.trx.eos.io:9876:trx\n" " p2p.blk.eos.io:9876:blk\n") + ( "p2p-peer-limit", bpo::value()->default_value(6), + "Limit the number of p2p-peer-address to remain connected to. Selects the best peers of p2p-peer-address-list.") ( "p2p-max-nodes-per-host", bpo::value()->default_value(def_max_nodes_per_host), "Maximum number of client nodes from any single IP address") ( "p2p-accept-transactions", bpo::value()->default_value(true), "Allow transactions received over p2p network to be evaluated and relayed if valid.") ( "p2p-disable-block-nack", bpo::value()->default_value(false), @@ -4397,6 +4400,7 @@ namespace eosio { p2p_dedup_cache_expire_time_us = fc::seconds( options.at( "p2p-dedup-cache-expire-time-sec" ).as() ); resp_expected_period = def_resp_expected_wait; max_nodes_per_host = options.at( "p2p-max-nodes-per-host" ).as(); + p2p_peer_limit = options.at("p2p-peer-limit").as(); p2p_accept_transactions = options.at( "p2p-accept-transactions" ).as(); p2p_disable_block_nack = options.at( "p2p-disable-block-nack" ).as(); From 42918824b553844c88793d312a2c20c914b3d1f5 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 21 Feb 2025 08:44:00 -0600 Subject: [PATCH 5/9] GH-1151 Avoid race on add trx. --- plugins/net_plugin/net_plugin.cpp | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index e8309bb0cf..851a98a574 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -230,8 +230,8 @@ namespace eosio { void rm_block(const block_id_type& blkid); bool add_peer_txn( const transaction_id_type& id, const time_point_sec& trx_expires, uint32_t connection_id, - const time_point_sec& now = time_point_sec(time_point::now()) ); - bool have_txn( const transaction_id_type& tid ) const; + const time_point& now ); + bool add_txn( const transaction_id_type& id, const time_point_sec& trx_expires, uint32_t connection_id ); void expire_txns(); void bcast_vote_msg( uint32_t exclude_peer, send_buffer_type msg ); @@ -2736,14 +2736,15 @@ namespace eosio { index.erase(p.first, p.second); } + // returns true if [id, connection_id] is not found bool dispatch_manager::add_peer_txn( const transaction_id_type& id, const time_point_sec& trx_expires, - uint32_t connection_id, const time_point_sec& now ) { + uint32_t connection_id, const time_point& now ) { fc::lock_guard g( local_txns_mtx ); auto tptr = local_txns.get().find( std::make_tuple( std::ref( id ), connection_id ) ); bool added = (tptr == local_txns.end()); if( added ) { // expire at either transaction expiration or configured max expire time whichever is less - time_point_sec expires{now.to_time_point() + my_impl->p2p_dedup_cache_expire_time_us}; + time_point_sec expires{now + my_impl->p2p_dedup_cache_expire_time_us}; expires = std::min( trx_expires, expires ); local_txns.insert( node_transaction_state{ .id = id, @@ -2753,10 +2754,18 @@ namespace eosio { return added; } - bool dispatch_manager::have_txn( const transaction_id_type& tid ) const { + // return true if trx already received, adds to local trxs with connection_id + bool dispatch_manager::add_txn(const transaction_id_type& id, const time_point_sec& trx_expires, uint32_t connection_id ) { fc::lock_guard g( local_txns_mtx ); - const auto tptr = local_txns.get().find( tid ); - return tptr != local_txns.end(); + bool contains = local_txns.get().contains( id ); + // expire at either transaction expiration or configured max expire time whichever is less + time_point_sec expires{fc::time_point::now() + my_impl->p2p_dedup_cache_expire_time_us}; + expires = std::min( trx_expires, expires ); + local_txns.insert( node_transaction_state{ + .id = id, + .expires = expires, + .connection_id = connection_id} ); + return contains; } void dispatch_manager::expire_txns() { @@ -2844,7 +2853,7 @@ namespace eosio { // called from any thread void dispatch_manager::bcast_transaction(const packed_transaction_ptr& trx) { trx_buffer_factory buff_factory; - const fc::time_point_sec now{fc::time_point::now()}; + const fc::time_point now{fc::time_point::now()}; my_impl->connections.for_each_connection( [this, &trx, &now, &buff_factory]( const connection_ptr& cp ) { if( !cp->is_transactions_connection() || !cp->current() ) { return; @@ -3262,10 +3271,8 @@ namespace eosio { } return true; } - bool have_trx = my_impl->dispatcher.have_txn( ptr->id() ); - my_impl->dispatcher.add_peer_txn( ptr->id(), ptr->expiration(), connection_id ); - if( have_trx ) { + if (!my_impl->dispatcher.add_txn( ptr->id(), ptr->expiration(), connection_id )) { peer_dlog( this, "got a duplicate transaction - dropping" ); return true; } From b5aca0c49211e93267d84619b83f3673e94911fa Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 24 Feb 2025 10:06:01 -0600 Subject: [PATCH 6/9] GH-1155 Add start_time, unique_trx_count, and unique_blk_count to connection status --- .../net_plugin/include/eosio/net_plugin/net_plugin.hpp | 5 ++++- plugins/net_plugin/net_plugin.cpp | 9 ++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/plugins/net_plugin/include/eosio/net_plugin/net_plugin.hpp b/plugins/net_plugin/include/eosio/net_plugin/net_plugin.hpp index b6eb399387..39fe38e47a 100644 --- a/plugins/net_plugin/include/eosio/net_plugin/net_plugin.hpp +++ b/plugins/net_plugin/include/eosio/net_plugin/net_plugin.hpp @@ -19,6 +19,9 @@ namespace eosio { bool is_blocks_only = false; bool is_transactions_only = false; time_point last_vote_received; + time_point start_time; + uint32_t unique_trx_count = 0; + uint32_t unique_blk_count = 0; handshake_message last_handshake; }; @@ -105,4 +108,4 @@ namespace eosio { FC_REFLECT( eosio::connection_status, (peer)(remote_ip)(remote_port)(connecting)(syncing) (is_bp_peer)(is_socket_open)(is_blocks_only)(is_transactions_only) - (last_vote_received)(last_handshake) ) + (last_vote_received)(start_time)(unique_trx_count)(unique_blk_count)(last_handshake) ) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 851a98a574..b09138af08 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -927,7 +927,7 @@ namespace eosio { std::string remote_endpoint_ip GUARDED_BY(conn_mtx); boost::asio::ip::address_v6::bytes_type remote_endpoint_ip_array GUARDED_BY(conn_mtx); - std::chrono::nanoseconds connection_start_time{0}; + std::atomic connection_start_time; // block nack support static constexpr uint16_t consecutive_block_nacks_threshold{2}; // stop sending blocks when reached @@ -1372,6 +1372,9 @@ namespace eosio { stat.is_blocks_only = is_blocks_only_connection(); stat.is_transactions_only = is_transactions_only_connection(); stat.last_vote_received = last_vote_received; + stat.start_time = fc::time_point(fc::microseconds(connection_start_time.load().count()/1000)); + stat.unique_trx_count = unique_trxs_rcvd_count; + stat.unique_blk_count = unique_blocks_rcvd_count; fc::lock_guard g( conn_mtx ); stat.peer = peer_addr; stat.remote_ip = log_remote_endpoint_ip; @@ -1485,6 +1488,10 @@ namespace eosio { block_sync_frame_bytes_sent = 0; block_sync_throttling = false; last_vote_received = time_point{}; + unique_votes_rcvd_count = 0; + unique_blocks_rcvd_count = 0; + unique_trxs_rcvd_count = 0; + last_unique_block_received = time_point{}; consecutive_blocks_nacks = 0; last_block_nack = block_id_type{}; From 5a46f8eec971f9fb2f492db85485dcc53906b7a3 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 24 Feb 2025 10:10:05 -0600 Subject: [PATCH 7/9] GH-1155 Add disconnect of connections over peer-limit --- plugins/net_plugin/net_plugin.cpp | 108 +++++++++++++++++++++++++++--- 1 file changed, 97 insertions(+), 11 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index b09138af08..9f2666e850 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -248,6 +248,7 @@ namespace eosio { constexpr auto def_max_clients = 25; // 0 for unlimited clients constexpr auto def_max_nodes_per_host = 1; constexpr auto def_conn_retry_wait = 30; + constexpr auto def_conn_retry_wait_peer_limit_multiplier = 10; constexpr auto def_txn_expire_wait = std::chrono::seconds(3); constexpr auto def_resp_expected_wait = std::chrono::seconds(5); constexpr auto def_sync_fetch_span = 1000; @@ -310,6 +311,7 @@ namespace eosio { mutable std::shared_mutex connections_mtx; connection_details_index connections; chain::flat_set supplied_peers; + std::atomic max_peer_ping_time_ns{std::numeric_limits::max()};; alignas(hardware_destructive_interference_sz) fc::mutex connector_check_timer_mtx; @@ -322,6 +324,7 @@ namespace eosio { fc::microseconds max_cleanup_time; boost::asio::steady_timer::duration connector_period{0}; uint32_t max_client_count{def_max_clients}; + uint16_t peer_limit{6}; std::function update_p2p_connection_metrics; private: // must call with held mutex @@ -332,13 +335,16 @@ namespace eosio { public: size_t number_connections() const; + size_t number_connected_peers() const; void add_supplied_peers(const vector& peers ); + void maybe_disconnect(connection_ptr c) const; // not thread safe, only call on startup void init(std::chrono::milliseconds heartbeat_timeout_ms, fc::microseconds conn_max_cleanup_time, boost::asio::steady_timer::duration conn_period, - uint32_t maximum_client_count); + uint32_t maximum_client_count, + uint16_t peer_limit); std::chrono::milliseconds get_heartbeat_timeout() const { return heartbeat_timeout; } @@ -907,6 +913,8 @@ namespace eosio { block_status_monitor block_status_monitor_; std::atomic last_vote_received; std::atomic unique_votes_rcvd_count{0}; + std::atomic last_unique_block_received; + std::atomic closed_by_peer_limit = false; alignas(hardware_destructive_interference_sz) fc::mutex sync_response_expected_timer_mtx; @@ -1399,6 +1407,7 @@ namespace eosio { peer_dlog( this, "connected" ); socket_open = true; connection_start_time = get_time(); + closed_by_peer_limit = false; start_read_message(); return true; } @@ -3665,6 +3674,8 @@ namespace eosio { // make sure we also get the latency we need if (peer_ping_time_ns == std::numeric_limits::max()) { send_time(); + } else { + my_impl->connections.maybe_disconnect(shared_from_this()); } } @@ -4019,6 +4030,7 @@ namespace eosio { if (fork_db_add_result == fork_db_add_t::appended_to_head || fork_db_add_result == fork_db_add_t::fork_switch) { ++c->unique_blocks_rcvd_count; + c->last_unique_block_received = fc::time_point::now(); fc_dlog(logger, "post process_incoming_block to app thread, block ${n}", ("n", ptr->block_num())); my_impl->producer_plug->process_blocks(); @@ -4352,7 +4364,7 @@ namespace eosio { " p2p.trx.eos.io:9876:trx\n" " p2p.blk.eos.io:9876:blk\n") ( "p2p-peer-limit", bpo::value()->default_value(6), - "Limit the number of p2p-peer-address to remain connected to. Selects the best peers of p2p-peer-address-list.") + "Soft limit on the number of p2p-peer-address to remain connected to. Selects the best peers of p2p-peer-address-list.") ( "p2p-max-nodes-per-host", bpo::value()->default_value(def_max_nodes_per_host), "Maximum number of client nodes from any single IP address") ( "p2p-accept-transactions", bpo::value()->default_value(true), "Allow transactions received over p2p network to be evaluated and relayed if valid.") ( "p2p-disable-block-nack", bpo::value()->default_value(false), @@ -4371,7 +4383,7 @@ namespace eosio { "Tuple of [PublicKey, WIF private key] (may specify multiple times)") ( "max-clients", bpo::value()->default_value(def_max_clients), "Maximum number of clients from which connections are accepted, use 0 for no limit") ( "connection-cleanup-period", bpo::value()->default_value(def_conn_retry_wait), "number of seconds to wait before cleaning up dead connections") - ( "max-cleanup-time-msec", bpo::value()->default_value(10), "max connection cleanup time per cleanup call in milliseconds") + ( "max-cleanup-time-msec", bpo::value()->default_value(50), "max connection cleanup time per cleanup call in milliseconds") ( "p2p-dedup-cache-expire-time-sec", bpo::value()->default_value(10), "Maximum time to track transaction for duplicate optimization") ( "net-threads", bpo::value()->default_value(my->thread_pool_size), "Number of worker threads in net_plugin thread pool" ) @@ -4418,6 +4430,16 @@ namespace eosio { p2p_accept_transactions = options.at( "p2p-accept-transactions" ).as(); p2p_disable_block_nack = options.at( "p2p-disable-block-nack" ).as(); + if (p2p_accept_transactions || chain_plug->accept_votes()) { + if (p2p_accept_transactions && chain_plug->accept_votes()) { + EOS_ASSERT( p2p_peer_limit >= 6, chain::plugin_config_exception, + "p2p-peer-limit with vote processing and trx processing should be >= 6, two connections each reserved for votes and trxs."); + } else { + EOS_ASSERT( p2p_peer_limit >= 4, chain::plugin_config_exception, + "p2p-peer-limit with vote processing or trx processing should be >= 4, two connections reserved for vote or trx processing"); + } + } + use_socket_read_watermark = options.at( "use-socket-read-watermark" ).as(); keepalive_interval = std::chrono::milliseconds( options.at( "p2p-keepalive-interval-ms" ).as() ); EOS_ASSERT( keepalive_interval.count() > 0, chain::plugin_config_exception, @@ -4436,7 +4458,8 @@ namespace eosio { connections.init( std::chrono::milliseconds( options.at("p2p-keepalive-interval-ms").as() * 2 ), fc::milliseconds( options.at("max-cleanup-time-msec").as() ), std::chrono::seconds( options.at("connection-cleanup-period").as() ), - options.at("max-clients").as() ); + options.at("max-clients").as(), + p2p_peer_limit ); if( options.count( "p2p-listen-endpoint" )) { auto p2ps = options.at("p2p-listen-endpoint").as>(); @@ -4727,6 +4750,16 @@ namespace eosio { return connections.size(); } + size_t connections_manager::number_connected_peers() const { + size_t num = 0; + for_each_block_connection([&num](const auto& cc) { + if (cc->socket_is_open() && !cc->incoming()) { + ++num; + } + }); + return num; + } + connection_ptr connections_manager::find_connection(uint32_t connection_id) const { std::shared_lock g(connections_mtx); const auto& index = connections.get(); @@ -4740,15 +4773,48 @@ namespace eosio { supplied_peers.insert( peers.begin(), peers.end() ); } + void connections_manager::maybe_disconnect(connection_ptr c) const { + assert(c); + // if accepting trx then don't disconnect from any peers + if (my_impl->p2p_accept_transactions) return; + // if syncing then other connections are basically idle + if (my_impl->sync_master->syncing_from_peer()) return; + // verify current, not syncing from us + if (!c->current()) return; + // remain connected for a heartbeat to gather info + if (c->connection_start_time.load() > connection::get_time() - get_heartbeat_timeout()) return; + // we have received votes first from this connection + if (c->unique_votes_rcvd_count > 0) return; + // we received block first from this connection recently + if (c->last_unique_block_received.load() > fc::time_point::now() - fc::minutes(10)) return; + // quick check if more connections than limit + if (number_connections() <= peer_limit) return; + // do we have more than the limit currently + if (number_connected_peers() <= peer_limit) return; + + auto peer_ping_time = c->get_peer_ping_time_ns(); + if (peer_ping_time < max_peer_ping_time_ns) { + return; + } + + peer_ilog(c, "Disconnecting from peer that is not one of the best of p2p-peer-address, ping ${p}ms, max ${m}ms", + ("p", peer_ping_time/1000000)("m", max_peer_ping_time_ns.load()/1000000)); + + c->closed_by_peer_limit = true; + c->close(false); + } + // not thread safe, only call on startup void connections_manager::init( std::chrono::milliseconds heartbeat_timeout_ms, fc::microseconds conn_max_cleanup_time, boost::asio::steady_timer::duration conn_period, - uint32_t maximum_client_count ) { + uint32_t maximum_client_count, + uint16_t p2p_peer_limit) { heartbeat_timeout = heartbeat_timeout_ms; max_cleanup_time = conn_max_cleanup_time; connector_period = conn_period; max_client_count = maximum_client_count; + peer_limit = p2p_peer_limit; } fc::microseconds connections_manager::get_connector_period() const { @@ -4958,11 +5024,16 @@ namespace eosio { // called from any thread void connections_manager::connection_monitor(const std::weak_ptr& from_connection) { size_t num_rm = 0, num_clients = 0, num_peers = 0, num_bp_peers = 0; - auto cleanup = [&num_rm, this](vector&& reconnecting, vector&& removing) { - for( auto& c : reconnecting ) { - if (!c->resolve_and_connect()) { - ++num_rm; - removing.push_back(c); + std::multiset ping_times; + auto cleanup = [&num_rm, num_peers, this](vector&& reconnecting, vector&& removing) { + // if num_peers less than or equal the peer_limit then attempt to connect to all configured peers + // the best of the configured peers will be chosen after they connect. + if (num_peers <= peer_limit) { + for( auto& c : reconnecting ) { + if (!c->resolve_and_connect()) { + ++num_rm; + removing.push_back(c); + } } } std::scoped_lock g( connections_mtx ); @@ -4971,6 +5042,14 @@ namespace eosio { index.erase(c->connection_id); } }; + auto record_max_peer_ping_time = [&]() { + if (ping_times.size() > peer_limit) { + auto itr = ping_times.begin(); + std::advance(itr, peer_limit-1); + max_peer_ping_time_ns = *itr; + } + }; + auto peer_limit_reconnect_time = connection::get_time() - (connector_period * def_conn_retry_wait_peer_limit_multiplier); auto max_time = fc::time_point::now().safe_add(max_cleanup_time); std::vector reconnecting, removing; auto from = from_connection.lock(); @@ -5001,16 +5080,23 @@ namespace eosio { if (!c->socket_is_open() && c->state() != connection::connection_state::connecting) { if (!c->incoming()) { --num_peers; - reconnecting.push_back(c); + if (!c->closed_by_peer_limit || (c->connection_start_time.load() < peer_limit_reconnect_time)) { + reconnecting.push_back(c); + } } else { --num_clients; ++num_rm; removing.push_back(c); } } + auto ping_time = c->get_peer_ping_time_ns(); + if (ping_time < std::numeric_limits::max()) { + ping_times.insert(ping_time); + } ++it; } g.unlock(); + record_max_peer_ping_time(); cleanup(std::move(reconnecting), std::move(removing)); if( num_clients > 0 || num_peers > 0 ) { From 1ad869c88e911650573bbfaf7d97739e5a2c12bd Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 24 Feb 2025 11:20:59 -0600 Subject: [PATCH 8/9] GH-1155 Fix de-dup trx logic --- plugins/net_plugin/net_plugin.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 9f2666e850..009365387f 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -2770,7 +2770,7 @@ namespace eosio { return added; } - // return true if trx already received, adds to local trxs with connection_id + // return true if trx not already received, adds to local trxs with connection_id bool dispatch_manager::add_txn(const transaction_id_type& id, const time_point_sec& trx_expires, uint32_t connection_id ) { fc::lock_guard g( local_txns_mtx ); bool contains = local_txns.get().contains( id ); @@ -2781,7 +2781,7 @@ namespace eosio { .id = id, .expires = expires, .connection_id = connection_id} ); - return contains; + return !contains; } void dispatch_manager::expire_txns() { From 85e066bbb2bea26c9f44f265b215bd92a42ad976 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 24 Feb 2025 12:36:32 -0600 Subject: [PATCH 9/9] GH-1155 Update p2p-peer-limit assert and warning --- plugins/net_plugin/net_plugin.cpp | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 009365387f..be8c0e2aa9 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -4364,7 +4364,7 @@ namespace eosio { " p2p.trx.eos.io:9876:trx\n" " p2p.blk.eos.io:9876:blk\n") ( "p2p-peer-limit", bpo::value()->default_value(6), - "Soft limit on the number of p2p-peer-address to remain connected to. Selects the best peers of p2p-peer-address-list.") + "Soft limit on the number of p2p-peer-address to remain connected to. Selects the best peers of p2p-peer-address list.") ( "p2p-max-nodes-per-host", bpo::value()->default_value(def_max_nodes_per_host), "Maximum number of client nodes from any single IP address") ( "p2p-accept-transactions", bpo::value()->default_value(true), "Allow transactions received over p2p network to be evaluated and relayed if valid.") ( "p2p-disable-block-nack", bpo::value()->default_value(false), @@ -4427,19 +4427,10 @@ namespace eosio { resp_expected_period = def_resp_expected_wait; max_nodes_per_host = options.at( "p2p-max-nodes-per-host" ).as(); p2p_peer_limit = options.at("p2p-peer-limit").as(); + EOS_ASSERT( p2p_peer_limit >= 1, chain::plugin_config_exception, "p2p-peer-limit should be >= 1"); p2p_accept_transactions = options.at( "p2p-accept-transactions" ).as(); p2p_disable_block_nack = options.at( "p2p-disable-block-nack" ).as(); - if (p2p_accept_transactions || chain_plug->accept_votes()) { - if (p2p_accept_transactions && chain_plug->accept_votes()) { - EOS_ASSERT( p2p_peer_limit >= 6, chain::plugin_config_exception, - "p2p-peer-limit with vote processing and trx processing should be >= 6, two connections each reserved for votes and trxs."); - } else { - EOS_ASSERT( p2p_peer_limit >= 4, chain::plugin_config_exception, - "p2p-peer-limit with vote processing or trx processing should be >= 4, two connections reserved for vote or trx processing"); - } - } - use_socket_read_watermark = options.at( "use-socket-read-watermark" ).as(); keepalive_interval = std::chrono::milliseconds( options.at( "p2p-keepalive-interval-ms" ).as() ); EOS_ASSERT( keepalive_interval.count() > 0, chain::plugin_config_exception, @@ -4505,6 +4496,11 @@ namespace eosio { } connections.add_supplied_peers(peers); } + + if (peers.size() > p2p_peer_limit && p2p_accept_transactions) { + fc_wlog(logger, "p2p-peer-limit=${n} ignored due to p2p-accept-transactions=true", ("n", p2p_peer_limit)); + } + if( options.count( "agent-name" )) { user_agent_name = options.at( "agent-name" ).as(); EOS_ASSERT( user_agent_name.length() <= max_handshake_str_length, chain::plugin_config_exception,