From ed35e8b1e0e2e75274e2c15f2eedcc25afcc3b05 Mon Sep 17 00:00:00 2001 From: Jose Daniel Hernandez Date: Fri, 8 Dec 2023 13:38:56 -0600 Subject: [PATCH 1/2] network-libp2p: Expose a configuration for the desired number of peers Expose a configuration for the desired number of peers for the connection pool to realize when to try to connect to more peers. This configuration is exposed in the network and is set in the client to be `consensus.min_peers`. --- lib/src/client.rs | 1 + network-libp2p/src/behaviour.rs | 1 + network-libp2p/src/config.rs | 3 +++ .../src/connection_pool/behaviour.rs | 20 +++++++++++-------- network-libp2p/tests/network.rs | 1 + network-libp2p/tests/request_response.rs | 1 + test-utils/src/test_network.rs | 1 + 7 files changed, 20 insertions(+), 8 deletions(-) diff --git a/lib/src/client.rs b/lib/src/client.rs index c666bdebdb..edab96b5cc 100644 --- a/lib/src/client.rs +++ b/lib/src/client.rs @@ -297,6 +297,7 @@ impl ClientInner { false, required_services, tls_config, + config.consensus.min_peers, config.network.autonat_allow_non_global_ips, config.network.only_secure_ws_connections, config.network.allow_loopback_addresses, diff --git a/network-libp2p/src/behaviour.rs b/network-libp2p/src/behaviour.rs index a3d2823393..6e8cf4709b 100644 --- a/network-libp2p/src/behaviour.rs +++ b/network-libp2p/src/behaviour.rs @@ -85,6 +85,7 @@ impl Behaviour { peer_id, config.seeds, config.discovery.required_services, + config.desired_peer_count, ); // Request Response behaviour diff --git a/network-libp2p/src/config.rs b/network-libp2p/src/config.rs index 093b7ee1d7..c426c45093 100644 --- a/network-libp2p/src/config.rs +++ b/network-libp2p/src/config.rs @@ -27,6 +27,7 @@ pub struct Config { pub memory_transport: bool, pub required_services: Services, pub tls: Option, + pub desired_peer_count: usize, pub autonat_allow_non_global_ips: bool, pub only_secure_ws_connections: bool, pub allow_loopback_addresses: bool, @@ -41,6 +42,7 @@ impl Config { memory_transport: bool, required_services: Services, tls_settings: Option, + desired_peer_count: usize, autonat_allow_non_global_ips: bool, only_secure_ws_connections: bool, allow_loopback_addresses: bool, @@ -87,6 +89,7 @@ impl Config { memory_transport, required_services, tls: tls_settings, + desired_peer_count, autonat_allow_non_global_ips, only_secure_ws_connections, allow_loopback_addresses, diff --git a/network-libp2p/src/connection_pool/behaviour.rs b/network-libp2p/src/connection_pool/behaviour.rs index 7a90d66fdc..11eb124a04 100644 --- a/network-libp2p/src/connection_pool/behaviour.rs +++ b/network-libp2p/src/connection_pool/behaviour.rs @@ -44,7 +44,7 @@ struct Limits { #[derive(Clone, Debug)] struct Config { /// Desired count of peers - peer_count_desired: usize, + desired_peer_count: usize, /// Maximum count of peers peer_count_max: usize, /// Maximum peer count per IP @@ -75,7 +75,7 @@ struct IpInfo { impl Default for Config { fn default() -> Self { Self { - peer_count_desired: 12, + desired_peer_count: 12, peer_count_max: 4000, peer_count_per_ip_max: 20, peer_count_per_subnet_max: 20, @@ -310,13 +310,17 @@ impl Behaviour { own_peer_id: PeerId, seeds: Vec, required_services: Services, + desired_peer_count: usize, ) -> Self { let limits = Limits { ip_count: HashMap::new(), ip_subnet_count: HashMap::new(), peer_count: 0, }; - let config = Config::default(); + let config = Config { + desired_peer_count, + ..Default::default() + }; let housekeeping_timer = wasm_timer::Interval::new(config.housekeeping_interval); Self { @@ -356,7 +360,7 @@ impl Behaviour { } } - /// Tries to maintain at least `peer_count_desired` connections. + /// Tries to maintain at least `desired_peer_count` connections. /// /// For this it will try to select peers or seeds to dial in order to /// achieve that many connection. @@ -375,18 +379,18 @@ impl Behaviour { // Make sure seeds and peers are dial-able again to reach desired peers. // Otherwise we might be stuck in this state forever. if self.active - && self.peer_ids.num_connected() < self.config.peer_count_desired + && self.peer_ids.num_connected() < self.config.desired_peer_count && self.peer_ids.num_dialing() + self.addresses.num_dialing() == 0 { self.addresses.reset_down(); self.peer_ids.reset_down(); } - // Try to maintain at least `peer_count_desired` connections. + // Try to maintain at least `desired_peer_count` connections. // Note: when counting dialing IDs we have to account for peer IDs and // addresses (seeds may only be in the `addresses` set). if self.active - && self.peer_ids.num_connected() < self.config.peer_count_desired + && self.peer_ids.num_connected() < self.config.desired_peer_count && self.peer_ids.num_dialing() + self.addresses.num_dialing() < self.config.dialing_count_max { @@ -447,7 +451,7 @@ impl Behaviour { fn choose_peers_to_dial(&self) -> Vec { let num_peers = usize::min( - self.config.peer_count_desired - self.peer_ids.num_connected(), + self.config.desired_peer_count - self.peer_ids.num_connected(), self.config.dialing_count_max - self.peer_ids.num_dialing(), ); let contacts = self.contacts.read(); diff --git a/network-libp2p/tests/network.rs b/network-libp2p/tests/network.rs index 2e4261a2b7..77015f6652 100644 --- a/network-libp2p/tests/network.rs +++ b/network-libp2p/tests/network.rs @@ -62,6 +62,7 @@ fn network_config(address: Multiaddr) -> Config { memory_transport: true, required_services: Services::all(), tls: None, + desired_peer_count: 3, autonat_allow_non_global_ips: true, only_secure_ws_connections: false, allow_loopback_addresses: true, diff --git a/network-libp2p/tests/request_response.rs b/network-libp2p/tests/request_response.rs index 0b3eaee638..2cba9fbdaa 100644 --- a/network-libp2p/tests/request_response.rs +++ b/network-libp2p/tests/request_response.rs @@ -282,6 +282,7 @@ fn network_config(address: Multiaddr) -> Config { memory_transport: true, required_services: Services::all(), tls: None, + desired_peer_count: 3, autonat_allow_non_global_ips: true, only_secure_ws_connections: false, allow_loopback_addresses: true, diff --git a/test-utils/src/test_network.rs b/test-utils/src/test_network.rs index 50047a3217..6109c5f523 100644 --- a/test-utils/src/test_network.rs +++ b/test-utils/src/test_network.rs @@ -70,6 +70,7 @@ impl TestNetwork for Network { true, Services::all(), None, + 3, true, false, true, From 1d388bc43e99b82b8626b2c0f66ab2054b89b738 Mon Sep 17 00:00:00 2001 From: Jose Daniel Hernandez Date: Fri, 19 Jan 2024 17:58:37 -0600 Subject: [PATCH 2/2] network-libp2p: Add some throttling for re-connections Make the `ConnectionState` aware of the desired number of connections such that when doing `housekeeping` it resets the connections marked as down after 1s when the number of connections is below the desired number. This changes how eagerly connections are re-tried when the number of connected peers is less than the desired value. --- .../src/connection_pool/behaviour.rs | 36 +++++++++++-------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/network-libp2p/src/connection_pool/behaviour.rs b/network-libp2p/src/connection_pool/behaviour.rs index 11eb124a04..b8bc6d4a31 100644 --- a/network-libp2p/src/connection_pool/behaviour.rs +++ b/network-libp2p/src/connection_pool/behaviour.rs @@ -108,10 +108,13 @@ struct ConnectionState { max_failures: usize, /// Time after which a connection ID would be removed from IDs marked as down. retry_down_after: Duration, + /// Desired number of connections. When the number of connections is below this number, + /// the `housekeeping` will retry the down nodes after 1s instead of `retry_down_after`. + desired_connections: usize, } impl ConnectionState { - fn new(max_failures: usize, retry_down_after: Duration) -> Self { + fn new(max_failures: usize, retry_down_after: Duration, desired_connections: usize) -> Self { Self { dialing: BTreeSet::new(), connected: BTreeSet::new(), @@ -120,6 +123,7 @@ impl ConnectionState { down: BTreeMap::new(), max_failures, retry_down_after, + desired_connections, } } @@ -220,18 +224,18 @@ impl ConnectionState { } /// Remove all down peers that haven't been dialed in a while from the `down` - /// map to dial them again. + /// map to dial them again. If the number of connections is less than the desired number + /// of connections, this happens for every connection marked as down after 1s, if not then + /// `self.retry_after_down is used`. fn housekeeping(&mut self) { - let retry_down_after = self.retry_down_after; + let retry_down_after = if self.num_connected() < self.desired_connections { + Duration::from_secs(1) + } else { + self.retry_down_after + }; self.down .retain(|_, down_since| down_since.elapsed() < retry_down_after); } - - /// Remove all connection IDs marked as down. - /// This will make an ID dial-able again. - fn reset_down(&mut self) { - self.down.clear() - } } impl std::fmt::Display for ConnectionState { @@ -328,8 +332,8 @@ impl Behaviour { own_peer_id, seeds, required_services, - peer_ids: ConnectionState::new(2, config.retry_down_after), - addresses: ConnectionState::new(4, config.retry_down_after), + peer_ids: ConnectionState::new(2, config.retry_down_after, desired_peer_count), + addresses: ConnectionState::new(4, config.retry_down_after, desired_peer_count), actions: VecDeque::new(), active: false, limits, @@ -376,14 +380,16 @@ impl Behaviour { // If we are active and have less connections than the desired amount // and we are not dialing anyone, it is most likely because we went down // (i.e. we are or were offline). - // Make sure seeds and peers are dial-able again to reach desired peers. - // Otherwise we might be stuck in this state forever. + // Make sure seeds and peers are dial-able again ASAP by just calling + // the addresses and peer IDs housekeeping since it has a mechanism to + // reset the connections marked as down after 1s if the number of connections + // is less than the desired peer count if self.active && self.peer_ids.num_connected() < self.config.desired_peer_count && self.peer_ids.num_dialing() + self.addresses.num_dialing() == 0 { - self.addresses.reset_down(); - self.peer_ids.reset_down(); + self.addresses.housekeeping(); + self.peer_ids.housekeeping(); } // Try to maintain at least `desired_peer_count` connections.