From 0e38a3cc6a4b043ca9ad719b9194ddee2fb15d5d Mon Sep 17 00:00:00 2001 From: Odd Stranne Date: Tue, 3 Oct 2023 11:34:59 +0200 Subject: [PATCH 1/2] Use cool down period if TCP accept fails --- CHANGELOG.md | 3 +++ src/exponential_backoff.rs | 37 +++++++++++++++++++++++++++++++++++++ src/lib.rs | 1 + src/tcp2udp.rs | 15 ++++++++++++++- 4 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 src/exponential_backoff.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index d8f0532..7a18e77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,9 @@ Line wrap the file at 100 chars. Th ## [Unreleased] +### Changed +- Use cool down period if TCP accept fails. This avoids excessive CPU usage e.g. when there are no + free file descriptors available to be allocated. ## [0.3.0] - 2023-02-28 diff --git a/src/exponential_backoff.rs b/src/exponential_backoff.rs new file mode 100644 index 0000000..9de4f29 --- /dev/null +++ b/src/exponential_backoff.rs @@ -0,0 +1,37 @@ +use std::cmp; +use std::time::Duration; + +/// Simple exponential backoff +pub struct ExponentialBackoff { + start_delay: Duration, + max_delay: Duration, + current_delay: Duration, +} + +impl ExponentialBackoff { + /// Creates a new exponential backoff instance starting with delay + /// `start_delay` and maxing out at `max_delay`. + pub fn new(start_delay: Duration, max_delay: Duration) -> Self { + Self { + start_delay, + max_delay, + current_delay: start_delay, + } + } + + /// Resets the exponential backoff so that the next delay is the start delay again. + pub fn reset(&mut self) { + self.current_delay = self.start_delay; + } + + /// Returns the next delay. This is twice as long as the last returned delay, + /// up until `max_delay` is reached. + pub fn next_delay(&mut self) -> Duration { + let delay = self.current_delay; + + let next_delay = self.current_delay * 2; + self.current_delay = cmp::min(next_delay, self.max_delay); + + delay + } +} diff --git a/src/lib.rs b/src/lib.rs index ad39936..6e95d3d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -86,6 +86,7 @@ pub mod udp2tcp; pub use udp2tcp::Udp2Tcp; +mod exponential_backoff; mod forward_traffic; mod logging; mod tcp_options; diff --git a/src/tcp2udp.rs b/src/tcp2udp.rs index b158c67..49dac39 100644 --- a/src/tcp2udp.rs +++ b/src/tcp2udp.rs @@ -1,6 +1,7 @@ //! Primitives for listening on TCP and forwarding the data in incoming connections //! to UDP. +use crate::exponential_backoff::ExponentialBackoff; use crate::logging::Redact; use err_context::{BoxedErrorExt as _, ErrorExt as _, ResultExt as _}; use std::convert::Infallible; @@ -9,6 +10,7 @@ use std::io; use std::net::{IpAddr, SocketAddr}; use std::time::Duration; use tokio::net::{TcpListener, TcpSocket, TcpStream, UdpSocket}; +use tokio::time::sleep; #[derive(Debug)] #[cfg_attr(feature = "clap", derive(clap::Parser))] @@ -149,6 +151,8 @@ async fn process_tcp_listener( tcp_recv_timeout: Option, tcp_nodelay: bool, ) -> ! { + let mut cooldown = + ExponentialBackoff::new(Duration::from_millis(50), Duration::from_millis(5000)); loop { match tcp_listener.accept().await { Ok((tcp_stream, tcp_peer_addr)) => { @@ -169,8 +173,17 @@ async fn process_tcp_listener( log::error!("Error: {}", error.display("\nCaused by: ")); } }); + cooldown.reset(); + } + Err(error) => { + log::error!("Error when accepting incoming TCP connection: {}", error); + + // If the process runs out of file descriptors, it will fail to accept a socket. + // But that socket will also remain in the queue, so it will fail again immediately. + // This will busy loop consuming the CPU and filling any logs. To prevent this, + // delay between failed socket accept operations. + sleep(cooldown.next_delay()).await; } - Err(error) => log::error!("Error when accepting incoming TCP connection: {}", error), } } } From 62e96944ab780529bafc2743dd2bf41ae51875cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20F=C3=A4rnstrand?= Date: Tue, 3 Oct 2023 17:09:59 +0200 Subject: [PATCH 2/2] Add unit tests to ExponentialBackoff --- src/exponential_backoff.rs | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/exponential_backoff.rs b/src/exponential_backoff.rs index 9de4f29..fb9cf43 100644 --- a/src/exponential_backoff.rs +++ b/src/exponential_backoff.rs @@ -35,3 +35,29 @@ impl ExponentialBackoff { delay } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn correct_delays() { + let mut backoff = + ExponentialBackoff::new(Duration::from_millis(60), Duration::from_millis(300)); + assert_eq!(backoff.next_delay(), Duration::from_millis(60)); + assert_eq!(backoff.next_delay(), Duration::from_millis(120)); + assert_eq!(backoff.next_delay(), Duration::from_millis(240)); + assert_eq!(backoff.next_delay(), Duration::from_millis(300)); + assert_eq!(backoff.next_delay(), Duration::from_millis(300)); + } + + #[test] + fn reset() { + let mut backoff = + ExponentialBackoff::new(Duration::from_millis(60), Duration::from_millis(300)); + assert_eq!(backoff.next_delay(), Duration::from_millis(60)); + backoff.reset(); + assert_eq!(backoff.next_delay(), Duration::from_millis(60)); + assert_eq!(backoff.next_delay(), Duration::from_millis(120)); + } +}