diff --git a/deploy/config/edge-docker.toml b/deploy/config/edge-docker.toml index d712aed..8feab1e 100644 --- a/deploy/config/edge-docker.toml +++ b/deploy/config/edge-docker.toml @@ -23,6 +23,7 @@ handshake_timeout_secs = 5 max_connections_per_ip = 10 rate_limit_pps = 5.0 rate_limit_burst = 10.0 +relay_idle_timeout_secs = 300 [ban] ban_duration_secs = 3600 diff --git a/deploy/config/edge-http.toml b/deploy/config/edge-http.toml index 016a5d8..ff4b7f9 100644 --- a/deploy/config/edge-http.toml +++ b/deploy/config/edge-http.toml @@ -16,6 +16,7 @@ handshake_timeout_secs = 5 max_connections_per_ip = 10 rate_limit_pps = 5.0 rate_limit_burst = 10.0 +relay_idle_timeout_secs = 300 [ban] ban_duration_secs = 3600 diff --git a/deploy/config/edge.toml b/deploy/config/edge.toml index 711c19e..24d1231 100644 --- a/deploy/config/edge.toml +++ b/deploy/config/edge.toml @@ -26,6 +26,7 @@ handshake_timeout_secs = 5 max_connections_per_ip = 10 rate_limit_pps = 5.0 rate_limit_burst = 10.0 +relay_idle_timeout_secs = 300 [ban] ban_duration_secs = 3600 diff --git a/docs/configuration.md b/docs/configuration.md index 3ff65c5..c3cd4bf 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -31,6 +31,7 @@ handshake_timeout_secs = 5 # таймаут чтения (slowloris-защ max_connections_per_ip = 10 # одновременные соединения с одного IP rate_limit_pps = 5.0 # лимит коннектов/сек с одного IP rate_limit_burst = 10.0 # burst +relay_idle_timeout_secs = 300 # простой relay до разрыва, 1..86400 [ban] ban_duration_secs = 3600 @@ -84,6 +85,15 @@ max_header_bytes = 512 Все секции опциональны — при отсутствии используются дефолты из таблицы выше. Для ingress PROXY v2 укажи сети именно прокси в `trusted_cidrs`; включение без доверенной сети отклоняется при чтении конфига. + +`trusted_cidrs` — это точные адреса/CIDR самих прокси, а не «весь DC»: IP из +заголовка используется для per-IP лимитов, репутации и банов, поэтому любой хост +внутри `trusted_cidrs` может назвать себя кем угодно. Заголовок с адресом из +`trusted_cidrs` (прокси не может быть клиентом за самим собой) или с +unspecified/loopback/link-local/multicast/broadcast адресом отклоняется, и +клиентом считается transport peer — событие пишется в debug один раз на +соединение. Адреса из `trusted_cidrs` никогда не банятся локально, в Redis и XDP; +попытка бана подавляется с предупреждением в логе. `send_proxy_protocol` отправляет заголовок на Paper backend; прежнее имя `proxy_protocol` внутри `[protocol.minecraft]` остаётся допустимым alias. @@ -97,6 +107,7 @@ max_header_bytes = 512 | `JWT_EXPIRATION_SECS` | нет | `86400` | Время жизни токена | | `API_PASSWORD` | да | — | Пароль для `/api/v1/auth/login`, минимум 16 символов; `changeme` запрещён | | `CORS_ORIGIN` | нет | — (CORS выключен) | `*` — разрешить все источники; список через запятую; пусто/не задано — запретить cross-origin | +| `MANAGER_TRUSTED_PROXIES` | нет | — (никому не доверяем) | Сети reverse-прокси через запятую (`10.0.0.0/8,2001:db8::/32`). `X-Forwarded-For` учитывается только для запросов, пришедших с адреса из этих сетей; иначе клиентом считается TCP peer. Пусто/не задано — заголовок игнорируется полностью. Опечатка или пустая запись в списке не даёт менеджеру стартовать | Manager всегда слушает `0.0.0.0:8080`. @@ -150,6 +161,10 @@ clang -O2 -g -target bpf -c xdp/core/universal_filter.c -o /tmp/universal_filter `whitelist6_map`) при attach и обходит state machine, throttle и баны. - **CIDR-баны**: XDP матчит по LPM-трие, userspace — по диапазонам адресов, поэтому бан `/24` или `/64` покрывает всю подсеть. +- **Оценка числа источников** в префиксе: XDP ведёт 64-битный bitmap хешей source-адресов, + userspace считает popcount. Благодаря этому spoof-gate (не банить `/24`, если в нём лишь + пара адресов) работает и на XDP-пути. Счётчики префиксов сбрасываются при каждом чтении, + то есть окно = интервал опроса монитора. Покрытие семейств адресов: diff --git a/src/config/sections/edge.rs b/src/config/sections/edge.rs index 95b8411..79541b5 100644 --- a/src/config/sections/edge.rs +++ b/src/config/sections/edge.rs @@ -21,7 +21,6 @@ impl Default for BindConfig { } } } - fn default_bind_address() -> String { "0.0.0.0".to_string() } @@ -43,7 +42,6 @@ impl Default for BackendConfig { } } } - fn default_upstreams() -> Vec { vec!["127.0.0.1:25566".to_string()] } @@ -60,7 +58,6 @@ impl Default for WorkerConfig { Self { count: 4 } } } - fn default_worker_count() -> usize { 4 } @@ -76,6 +73,11 @@ pub struct LimitsConfig { pub rate_limit_pps: f64, #[serde(default = "default_rate_limit_burst")] pub rate_limit_burst: f64, + #[serde( + default = "default_relay_idle_timeout", + deserialize_with = "deserialize_relay_idle_timeout" + )] + pub relay_idle_timeout_secs: u64, } impl Default for LimitsConfig { @@ -85,10 +87,10 @@ impl Default for LimitsConfig { max_connections_per_ip: default_max_connections_per_ip(), rate_limit_pps: default_rate_limit_pps(), rate_limit_burst: default_rate_limit_burst(), + relay_idle_timeout_secs: default_relay_idle_timeout(), } } } - fn default_handshake_timeout() -> u64 { 5 } @@ -101,6 +103,22 @@ fn default_rate_limit_pps() -> f64 { fn default_rate_limit_burst() -> f64 { 10.0 } +fn default_relay_idle_timeout() -> u64 { + 300 +} + +/// Deserializes and validates `limits.relay_idle_timeout_secs`. +/// # Errors +/// Fails outside `1..=86_400`; zero would cut every relayed session immediately. +fn deserialize_relay_idle_timeout<'de, D: Deserializer<'de>>(deserializer: D) -> Result { + let secs = u64::deserialize(deserializer)?; + if (1..=86_400).contains(&secs) { + return Ok(secs); + } + Err(serde::de::Error::custom( + "limits.relay_idle_timeout_secs must be between 1 and 86400", + )) +} /// `[ban]` section: default ban duration. #[derive(Debug, Clone, Deserialize)] @@ -116,7 +134,6 @@ impl Default for BanConfig { } } } - fn default_ban_duration() -> u64 { 3600 } @@ -153,11 +170,11 @@ impl TrustedProxyCidr { /// Returns whether `ip` belongs to this network. #[must_use] pub fn contains(&self, ip: IpAddr) -> bool { - let mask4 = mask_v4(self.prefix_len); - let mask6 = mask_v6(self.prefix_len); + // Masks are family-specific: a prefix above 32 would underflow the v4 shift. + let prefix = self.prefix_len; match (self.network, ip) { - (IpAddr::V4(network), IpAddr::V4(ip)) => u32::from(network) & mask4 == u32::from(ip) & mask4, - (IpAddr::V6(network), IpAddr::V6(ip)) => u128::from(network) & mask6 == u128::from(ip) & mask6, + (IpAddr::V4(net), IpAddr::V4(ip)) => u32::from(net) & mask_v4(prefix) == u32::from(ip) & mask_v4(prefix), + (IpAddr::V6(net), IpAddr::V6(ip)) => u128::from(net) & mask_v6(prefix) == u128::from(ip) & mask_v6(prefix), _ => false, } } @@ -167,18 +184,12 @@ impl FromStr for TrustedProxyCidr { type Err = String; fn from_str(value: &str) -> Result { - let (address, prefix) = match value.split_once('/') { - Some((address, prefix)) => (address, Some(prefix)), - None => (value, None), - }; + let (address, prefix) = value.split_once('/').map_or((value, None), |(a, p)| (a, Some(p))); let network = IpAddr::from_str(address).map_err(|_| format!("invalid trusted proxy CIDR: {value}"))?; let max_prefix = if network.is_ipv4() { 32 } else { 128 }; - let prefix_len = match prefix { - Some(prefix) => prefix - .parse::() - .map_err(|_| format!("invalid trusted proxy CIDR prefix: {value}"))?, - None => max_prefix, - }; + let prefix_len = prefix + .map_or(Ok(max_prefix), |prefix| prefix.parse::()) + .map_err(|_| format!("invalid trusted proxy CIDR prefix: {value}"))?; if prefix_len > max_prefix { return Err(format!("trusted proxy CIDR prefix is too large: {value}")); } @@ -231,7 +242,6 @@ impl Default for ProxyProtocolConfig { } } } - fn default_proxy_protocol_timeout_ms() -> u64 { 750 } diff --git a/src/engine/listener.rs b/src/engine/listener.rs index 7bc85e2..8b9e260 100644 --- a/src/engine/listener.rs +++ b/src/engine/listener.rs @@ -3,8 +3,10 @@ use crate::config::Config; use crate::engine::tunnel::Gateway; use crate::traffic::hook::TrafficHook; +use anyhow::Context; use socket2::{Domain, Socket, Type}; use std::io::ErrorKind; +use std::net::{IpAddr, SocketAddr}; use std::sync::Arc; use std::time::Duration; use tokio::net::TcpListener; @@ -119,7 +121,7 @@ pub async fn run( hook: Arc, shutdown: watch::Receiver, ) -> anyhow::Result<()> { - let addr: std::net::SocketAddr = format!("{}:{}", config.bind.address, config.bind.port).parse()?; + let addr = parse_listen_addr(&config.bind.address, config.bind.port)?; let workers = config.workers.count.max(1); let mut handles = Vec::with_capacity(workers); @@ -139,11 +141,39 @@ pub async fn run( Ok(()) } -fn build_listener(addr: std::net::SocketAddr) -> anyhow::Result { - let socket = Socket::new(Domain::IPV4, Type::STREAM, None)?; +/// Parses the configured listen address into a socket address. +/// +/// The address must be an IP literal: resolving a host name at startup would +/// make the bound family depend on resolver state, and the listener needs the +/// family up front to pick the correct socket domain. +/// +/// # Errors +/// Fails when `address` is not a valid IPv4 or IPv6 literal. +pub fn parse_listen_addr(address: &str, port: u16) -> anyhow::Result { + let ip: IpAddr = address + .parse() + .with_context(|| format!("bind.address must be an IP address, got {address:?}"))?; + Ok(SocketAddr::new(ip, port)) +} + +/// Binds a TCP listener on `addr` with `SO_REUSEPORT` and `SO_REUSEADDR`. +/// +/// The socket domain follows the address family, so IPv6 listen addresses work +/// as well as IPv4 ones. An IPv6 wildcard socket is switched to dual-stack mode +/// (`IPV6_V6ONLY=0`), which keeps `bind.address = "::"` accepting IPv4 clients +/// through IPv4-mapped addresses. +/// +/// # Errors +/// Fails when the socket cannot be created, configured or bound. +pub fn build_listener(addr: SocketAddr) -> anyhow::Result { + let domain = if addr.is_ipv4() { Domain::IPV4 } else { Domain::IPV6 }; + let socket = Socket::new(domain, Type::STREAM, None)?; socket.set_reuse_port(true)?; socket.set_reuse_address(true)?; socket.set_nonblocking(true)?; + if addr.is_ipv6() && addr.ip().is_unspecified() { + socket.set_only_v6(false)?; + } socket.bind(&addr.into())?; socket.listen(65535)?; Ok(TcpListener::from_std(socket.into())?) diff --git a/src/engine/tunnel.rs b/src/engine/tunnel.rs index a44b203..a65f380 100644 --- a/src/engine/tunnel.rs +++ b/src/engine/tunnel.rs @@ -17,9 +17,33 @@ use std::sync::Arc; use std::sync::Mutex; use std::sync::atomic::{AtomicU64, AtomicUsize}; use tokio::sync::Mutex as TokioMutex; +use tokio::sync::Semaphore; mod gateway; #[cfg(feature = "protocol-minecraft")] mod proxy; +#[cfg(feature = "protocol-minecraft")] +pub use proxy::{ClientIpRejection, validate_client_ip}; + +/// Multiplier applied to the per-IP limit when sizing the global connection +/// budget: `4 x workers x max_connections_per_ip` in-flight connections. +const CONNECTION_BUDGET_FACTOR: usize = 4; +/// Ceiling of the global budget, so a large worker count cannot exhaust the +/// file-descriptor table of the host. +const CONNECTION_BUDGET_MAX: usize = 100_000; + +/// Returns the in-flight connection budget of one gateway instance. +/// +/// The budget lives on the gateway instead of in a process-wide static so an +/// embedded or test gateway gets its own limiter and configuration changes are +/// visible without restarting the process. +fn connection_budget_capacity(config: &Config) -> usize { + let workers = config.workers.count.max(1); + let per_ip = config.limits.max_connections_per_ip.max(1) as usize; + workers + .saturating_mul(per_ip) + .saturating_mul(CONNECTION_BUDGET_FACTOR) + .clamp(1, CONNECTION_BUDGET_MAX) +} /// Shared edge engine state handed to every accept loop by [`Arc`]. pub struct Gateway { @@ -35,6 +59,8 @@ pub struct Gateway { pub allowed_1s: Arc, pub registry: Arc, pub subnet_tracker: Option>, + budget: Arc, + budget_rejections: AtomicU64, upstream_cursor: AtomicUsize, } diff --git a/src/engine/tunnel/gateway.rs b/src/engine/tunnel/gateway.rs index ebf6d34..4bbcdf8 100644 --- a/src/engine/tunnel/gateway.rs +++ b/src/engine/tunnel/gateway.rs @@ -10,8 +10,6 @@ mod challenge; mod pipeline; mod relay; -pub use relay::RELAY_IDLE_TIMEOUT; - #[cfg(feature = "protocol-minecraft")] use super::proxy; use super::{Gateway, GatewayOutcome}; @@ -31,6 +29,7 @@ use std::sync::atomic::AtomicU64; use std::sync::{Arc, Mutex}; use tokio::net::TcpStream; use tokio::sync::Mutex as TokioMutex; +use tokio::sync::Semaphore; impl Gateway { #[allow(clippy::too_many_arguments)] @@ -48,6 +47,7 @@ impl Gateway { subnet_tracker: Option>, ) -> Self { let max_connections = config.limits.max_connections_per_ip as usize; + let budget = Arc::new(Semaphore::new(super::connection_budget_capacity(&config))); Self { config, rate_limiter, @@ -61,6 +61,8 @@ impl Gateway { allowed_1s, registry, subnet_tracker, + budget, + budget_rejections: AtomicU64::new(0), upstream_cursor: std::sync::atomic::AtomicUsize::new(0), } } diff --git a/src/engine/tunnel/gateway/admission.rs b/src/engine/tunnel/gateway/admission.rs index 92d6227..493cdc0 100644 --- a/src/engine/tunnel/gateway/admission.rs +++ b/src/engine/tunnel/gateway/admission.rs @@ -21,6 +21,17 @@ impl Gateway { metrics::CONNECTIONS_TOTAL.with_label_values(&["blocked"]).inc(); self.reputation.record_bad(ip); if self.reputation.score(ip) < -40 { + // Defence in depth: a trusted proxy address must never enter the ban + // path, even when reputation escalation is reached. Bans are + // replicated to Redis and XDP on every node, so one wrong identity + // here could take the ingress proxy out of the whole cluster. + if self.is_trusted_proxy(ip) { + tracing::warn!( + %ip, + "suppressed ban for an address inside proxy_protocol.trusted_cidrs" + ); + return; + } let duration_secs = Duration::from_secs(self.config.ban.ban_duration_secs); self.blacklist.add(ip, duration_secs, "low_reputation"); if !self.config.proxy_protocol.ingress_enabled { @@ -31,6 +42,15 @@ impl Gateway { } } + /// Returns whether `ip` belongs to a configured trusted proxy network. + fn is_trusted_proxy(&self, ip: IpAddr) -> bool { + self.config + .proxy_protocol + .trusted_cidrs + .iter() + .any(|cidr| cidr.contains(ip)) + } + /// Mirrors a ban into the XDP map, if the kernel filter is attached. fn xdp_ban(&self, ip: IpAddr, duration_secs: u64) { let Some(xdp) = &self.xdp else { diff --git a/src/engine/tunnel/gateway/pipeline.rs b/src/engine/tunnel/gateway/pipeline.rs index 2205556..cd7af79 100644 --- a/src/engine/tunnel/gateway/pipeline.rs +++ b/src/engine/tunnel/gateway/pipeline.rs @@ -1,48 +1,16 @@ //! Filter order and upstream establishment for one accepted connection. use super::Gateway; -use super::RELAY_IDLE_TIMEOUT; use crate::metrics; -use std::sync::OnceLock; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::Ordering; use std::time::Duration; use tokio::io::AsyncWriteExt; use tokio::net::TcpStream; -use tokio::sync::Semaphore; -/// Multiplier applied to the per-IP limit when sizing the global connection -/// budget: `4 x workers x max_connections_per_ip` in-flight connections. -const GLOBAL_BUDGET_FACTOR: usize = 4; -/// Ceiling of the global budget, so a large worker count cannot exhaust the -/// file-descriptor table of the host. -const GLOBAL_BUDGET_MAX: usize = 100_000; /// One in every `BUDGET_LOG_EVERY` budget rejections is logged. const BUDGET_LOG_EVERY: u64 = 128; -/// Process-wide in-flight connection budget. -/// -/// Initialised on first use from the worker count and the per-IP limit of the -/// gateway configuration. A process-wide value is intentional: the budget caps -/// descriptors and tasks for the whole edge node, and there is exactly one -/// gateway per process in production. -static GLOBAL_BUDGET: OnceLock = OnceLock::new(); -/// Total connections rejected by the global budget, used for log rate limiting. -static BUDGET_REJECTIONS: AtomicU64 = AtomicU64::new(0); - impl Gateway { - /// Returns the process-wide in-flight connection budget. - fn global_budget(&self) -> &'static Semaphore { - GLOBAL_BUDGET.get_or_init(|| { - let workers = self.config.workers.count.max(1); - let per_ip = self.config.limits.max_connections_per_ip.max(1) as usize; - let capacity = workers - .saturating_mul(per_ip) - .saturating_mul(GLOBAL_BUDGET_FACTOR) - .clamp(1, GLOBAL_BUDGET_MAX); - Semaphore::new(capacity) - }) - } - /// Runs the filter stages, PoW gate, protocol handshake and relay. /// /// # Errors @@ -86,11 +54,11 @@ impl Gateway { let _global_permit = if trusted { None } else { - match self.global_budget().try_acquire() { + match self.budget.try_acquire() { Ok(permit) => Some(permit), Err(_) => { metrics::CONNECTIONS_TOTAL.with_label_values(&["overloaded"]).inc(); - let seen = BUDGET_REJECTIONS.fetch_add(1, Ordering::Relaxed); + let seen = self.budget_rejections.fetch_add(1, Ordering::Relaxed); if seen % BUDGET_LOG_EVERY == 0 { tracing::warn!( "global connection budget exhausted after {seen} rejections, dropping {peer_ip}" @@ -145,7 +113,7 @@ impl Gateway { handshake_budget, ) .await?; - self.relay_with_idle(client, backend, RELAY_IDLE_TIMEOUT).await + self.relay_with_idle(client, backend, self.relay_idle_timeout()).await } /// Opens the upstream connection under the handshake timeout. diff --git a/src/engine/tunnel/gateway/relay.rs b/src/engine/tunnel/gateway/relay.rs index 9ecd994..9541f1c 100644 --- a/src/engine/tunnel/gateway/relay.rs +++ b/src/engine/tunnel/gateway/relay.rs @@ -8,17 +8,17 @@ use tokio::net::TcpStream; const RELAY_BUFFER_BYTES: usize = 16 * 1024; -/// Idle budget of the client/backend relay. -/// -/// Minecraft keeps one connection alive for a whole session and a legitimately -/// silent gap (menus, chunk loading, AFK timeouts) can last minutes, so 300s is -/// the safe default: it never cuts a real session and still reaps dead peers. -/// The value is a constant because `[limits]` is owned by a parallel change; it -/// is plumbed as a parameter through [`Gateway::relay_with_idle`] so a config -/// field can be wired in without touching the relay itself. -pub const RELAY_IDLE_TIMEOUT: Duration = Duration::from_secs(300); - impl Gateway { + /// Returns the configured idle budget of the client/backend relay. + /// + /// `limits.relay_idle_timeout_secs` defaults to 300s: Minecraft keeps one + /// connection alive for a whole session and a legitimately silent gap + /// (menus, chunk loading, AFK timeouts) can last minutes, so a short budget + /// would cut real sessions while a missing one would pin descriptors. + pub fn relay_idle_timeout(&self) -> Duration { + Duration::from_secs(self.config.limits.relay_idle_timeout_secs) + } + /// Relays bytes in both directions until either side closes, an I/O error /// occurs or a direction stays silent for `idle`. /// diff --git a/src/engine/tunnel/proxy.rs b/src/engine/tunnel/proxy.rs index d681845..b9f2048 100644 --- a/src/engine/tunnel/proxy.rs +++ b/src/engine/tunnel/proxy.rs @@ -2,19 +2,85 @@ //! parsing of headers injected by trusted ingress peers. use crate::config::ProxyProtocolConfig; -use std::net::SocketAddr; +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::time::Duration; use tokio::net::TcpStream; mod inbound; pub(super) use inbound::read_v2_header; +/// Reason a PROXY header client address is rejected as an identity. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ClientIpRejection { + /// `0.0.0.0` or `::`: no host behind the header. + Unspecified, + /// The header claims a loopback address, which is never a real client. + Loopback, + /// The header claims a link-local address (`169.254.0.0/16`, `fe80::/10`). + LinkLocal, + /// The header claims a multicast address. + Multicast, + /// The header claims the limited IPv4 broadcast address. + Broadcast, + /// The header claims an address of the proxy network itself. + TrustedProxy, +} + +/// Validates the client IP claimed in an inbound PROXY v2 header. +/// +/// A trusted proxy is allowed to state who the client is, but it must not be +/// able to name an address that can never be a real remote client (host-local, +/// link-local, multicast, broadcast) or another trusted proxy. Without this +/// check a single compromised or misconfigured proxy could ban arbitrary +/// addresses cluster-wide or poison their reputation. +/// +/// # Errors +/// Returns the reason `ip` is not usable as a client identity. +pub fn validate_client_ip(config: &ProxyProtocolConfig, ip: IpAddr) -> Result<(), ClientIpRejection> { + // Collapse IPv4-mapped IPv6 first: the ban tables do the same, so leaving + // `::ffff:127.0.0.1` as v6 would bypass the loopback and trusted-network + // checks and then be normalized into a ban for `127.0.0.1`. + let ip = match ip { + IpAddr::V6(v6) => v6.to_ipv4_mapped().map_or(ip, IpAddr::V4), + IpAddr::V4(_) => ip, + }; + if ip.is_unspecified() { + return Err(ClientIpRejection::Unspecified); + } + if ip.is_loopback() { + return Err(ClientIpRejection::Loopback); + } + let link_local = match ip { + IpAddr::V4(v4) => v4.is_link_local(), + IpAddr::V6(v6) => v6.is_unicast_link_local(), + }; + if link_local { + return Err(ClientIpRejection::LinkLocal); + } + if ip.is_multicast() { + return Err(ClientIpRejection::Multicast); + } + if ip == IpAddr::V4(Ipv4Addr::BROADCAST) { + return Err(ClientIpRejection::Broadcast); + } + if config.trusted_cidrs.iter().any(|cidr| cidr.contains(ip)) { + return Err(ClientIpRejection::TrustedProxy); + } + Ok(()) +} + /// Resolves the effective client address from an inbound PROXY v2 header. /// /// Only peers inside `config.trusted_cidrs` may inject a header; anyone else /// is rejected instead of being believed. The whole header must arrive within /// `config.ingress_timeout_ms` and fit in `config.max_header_bytes`. /// +/// A header whose client address fails [`validate_client_ip`] is not fatal: +/// the transport peer becomes the client identity and the reason is logged once +/// per connection at debug level. Failing closed would drop every connection +/// behind a misbehaving proxy, while trusting the claimed address would hand +/// the ban path to the proxy. +/// /// # Errors /// Fails when the peer is not trusted, on timeout and on any header parse error. pub(super) async fn resolve_client_addr( @@ -25,12 +91,24 @@ pub(super) async fn resolve_client_addr( if !config.trusted_cidrs.iter().any(|cidr| cidr.contains(peer_addr.ip())) { anyhow::bail!("untrusted peer attempted inbound PROXY protocol"); } - read_v2_header( + let claimed = read_v2_header( client, Duration::from_millis(config.ingress_timeout_ms), config.max_header_bytes, ) - .await + .await?; + match validate_client_ip(config, claimed.ip()) { + Ok(()) => Ok(claimed), + Err(reason) => { + tracing::debug!( + peer = %peer_addr, + claimed = %claimed, + reason = ?reason, + "ignoring PROXY header client address, using the transport peer" + ); + Ok(peer_addr) + }, + } } /// Encodes a PROXY protocol v2 TCP header for the backend. diff --git a/tests/config_parse.rs b/tests/config_parse.rs index 74b2b4a..6fd9ce1 100644 --- a/tests/config_parse.rs +++ b/tests/config_parse.rs @@ -238,6 +238,30 @@ fn rejects_rate_limit_burst_below_one() { } } +#[test] +fn relay_idle_timeout_defaults_to_300_and_accepts_the_full_range() { + let config = Config::parse_str("").expect("empty config must parse"); + assert_eq!(config.limits.relay_idle_timeout_secs, 300); + + for secs in [1u64, 60, 300, 86_400] { + let toml = format!("[limits]\nrelay_idle_timeout_secs = {secs}\n"); + let config = Config::parse_str(&toml).expect("in-range idle timeout must parse"); + assert_eq!(config.limits.relay_idle_timeout_secs, secs); + } +} + +#[test] +fn rejects_out_of_range_relay_idle_timeout() { + for secs in [0u64, 86_401] { + let toml = format!("[limits]\nrelay_idle_timeout_secs = {secs}\n"); + let err = Config::parse_str(&toml).expect_err("idle timeout outside 1..=86400 must be rejected"); + assert!( + err.to_string().contains("limits.relay_idle_timeout_secs"), + "idle timeout '{secs}' must be rejected" + ); + } +} + #[test] fn rejects_out_of_range_ban_duration() { let err = Config::parse_str("[ban]\nban_duration_secs = 0\n").expect_err("zero ban duration must be rejected"); diff --git a/tests/connection_limits.rs b/tests/connection_limits.rs index 33601dd..55bd6dc 100644 --- a/tests/connection_limits.rs +++ b/tests/connection_limits.rs @@ -3,7 +3,7 @@ use rampart::config::Config; use rampart::engine::challenge::DifficultyAdjuster; -use rampart::engine::listener::AcceptBackoff; +use rampart::engine::listener::{AcceptBackoff, build_listener, parse_listen_addr}; use rampart::engine::tunnel::Gateway; use rampart::filter::blacklist::Blacklist; use rampart::filter::rate_limit::RateLimiter; @@ -11,7 +11,7 @@ use rampart::protocol::ProtocolRegistry; use rampart::traffic::reputation::IpReputation; use std::collections::HashSet; use std::io; -use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; use std::sync::Arc; use std::sync::Mutex; use std::sync::atomic::AtomicU64; @@ -263,6 +263,45 @@ async fn accept_backoff_sleeps_instead_of_spinning() { ); } +/// The listen address is parsed as an IP literal and the socket domain follows +/// the address family, so IPv6 loopback binds instead of forcing IPv4. +#[tokio::test] +async fn binds_ipv4_and_ipv6_loopback() { + let v4 = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0); + let listener = build_listener(v4).expect("bind IPv4 loopback"); + assert!(listener.local_addr().expect("local addr").is_ipv4()); + + let v6 = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 0); + let listener = build_listener(v6).expect("bind IPv6 loopback"); + assert!(listener.local_addr().expect("local addr").is_ipv6()); +} + +/// `::` must parse as a wildcard IPv6 listen address instead of being +/// concatenated into the invalid `":::port"` form. +#[test] +fn parses_ipv6_wildcard_listen_address() { + let addr = parse_listen_addr("::", 25565).expect("IPv6 wildcard must parse"); + assert_eq!(addr, SocketAddr::new(IpAddr::V6(Ipv6Addr::UNSPECIFIED), 25565)); + assert!(addr.is_ipv6()); +} + +/// The relay idle budget is configurable through `limits` instead of being a +/// hardcoded constant. +#[test] +fn relay_idle_timeout_comes_from_limits_config() { + let mut config = Config::default(); + config.limits.relay_idle_timeout_secs = 1; + let gateway = test_gateway(config, HashSet::new()); + assert_eq!(gateway.relay_idle_timeout(), Duration::from_secs(1)); + + let default_gateway = test_gateway(Config::default(), HashSet::new()); + assert_eq!( + default_gateway.relay_idle_timeout(), + Duration::from_secs(300), + "the default idle budget is 300s" + ); +} + /// A silent pair is closed by the relay idle timeout instead of pinning two /// descriptors forever. #[tokio::test] diff --git a/tests/proxy_identity.rs b/tests/proxy_identity.rs new file mode 100644 index 0000000..bb441fb --- /dev/null +++ b/tests/proxy_identity.rs @@ -0,0 +1,239 @@ +//! Client identity behind a trusted PROXY protocol proxy: header address +//! validation, fallback to the transport peer and suppression of bans for +//! addresses inside `proxy_protocol.trusted_cidrs`. + +#![cfg(feature = "protocol-minecraft")] + +use rampart::config::{Config, ProxyProtocolConfig, TrustedProxyCidr}; +use rampart::engine::challenge::DifficultyAdjuster; +use rampart::engine::tunnel::{ClientIpRejection, Gateway, GatewayOutcome, validate_client_ip}; +use rampart::filter::blacklist::Blacklist; +use rampart::filter::rate_limit::RateLimiter; +use rampart::protocol::ProtocolRegistry; +use rampart::traffic::reputation::IpReputation; +use std::collections::HashSet; +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::atomic::AtomicU64; +use tokio::io::AsyncWriteExt; +use tokio::net::{TcpListener, TcpStream}; + +/// Builds a gateway with no XDP/ClickHouse writers and an empty protocol +/// registry, so every connection stops at the admission filters. +fn test_gateway(config: Config, whitelist: HashSet) -> Arc { + Arc::new(Gateway::new( + Arc::new(config.clone()), + Arc::new(RateLimiter::new( + config.limits.rate_limit_pps, + config.limits.rate_limit_burst, + )), + Arc::new(Blacklist::new()), + Arc::new(Mutex::new(DifficultyAdjuster::default())), + Arc::new(whitelist), + Arc::new(IpReputation::new()), + None, + None, + Arc::new(AtomicU64::new(0)), + Arc::new(ProtocolRegistry::new()), + None, + )) +} + +/// Encodes a PROXY protocol v2 TCP-over-IPv4 header. +fn proxy_v2_header(source: Ipv4Addr, destination: Ipv4Addr) -> Vec { + const SIGNATURE: [u8; 12] = *b"\r\n\r\n\0\r\nQUIT\n"; + let mut header = Vec::with_capacity(28); + header.extend_from_slice(&SIGNATURE); + header.push(0x21); + header.push(0x11); + header.extend_from_slice(&12u16.to_be_bytes()); + header.extend_from_slice(&source.octets()); + header.extend_from_slice(&destination.octets()); + header.extend_from_slice(&40_000u16.to_be_bytes()); + header.extend_from_slice(&25565u16.to_be_bytes()); + header +} + +/// Runs one connection through the gateway after sending a PROXY v2 header that +/// claims `claimed` as the client address. The transport peer passed to the +/// gateway is `peer`, which lets the test model a trusted proxy explicitly. +async fn handle_with_header(gateway: &Gateway, peer: SocketAddr, claimed: Ipv4Addr) -> GatewayOutcome { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind loopback"); + let local = listener.local_addr().expect("local addr"); + let (client, accepted) = tokio::join!(TcpStream::connect(local), listener.accept()); + let mut client = client.expect("connect"); + let server = accepted.expect("accept").0; + let destination = match local.ip() { + IpAddr::V4(ip) => ip, + IpAddr::V6(_) => Ipv4Addr::LOCALHOST, + }; + client + .write_all(&proxy_v2_header(claimed, destination)) + .await + .expect("write PROXY header"); + let outcome = gateway.handle(server, peer).await; + drop(client); + outcome +} + +/// Runs one connection without a PROXY header. +async fn handle_plain(gateway: &Gateway, peer: SocketAddr) -> GatewayOutcome { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind loopback"); + let local = listener.local_addr().expect("local addr"); + let (_client, server) = tokio::join!(TcpStream::connect(local), listener.accept()); + gateway.handle(server.expect("accept").0, peer).await +} + +/// Parses a trusted proxy network for the test config. +fn cidr(value: &str) -> TrustedProxyCidr { + value.parse().expect("valid trusted proxy CIDR") +} + +/// A trusted proxy may state a public client address, and that address is the +/// one used for admission decisions and bans. +#[tokio::test] +async fn trusted_proxy_header_ip_is_used_for_identity_and_bans() { + let mut config = Config::default(); + config.limits.rate_limit_pps = 0.0; + config.limits.rate_limit_burst = 0.0; + config.proxy_protocol.ingress_enabled = true; + config.proxy_protocol.trusted_cidrs = vec![cidr("127.0.0.1/32")]; + let gateway = test_gateway(config, HashSet::new()); + + let claimed = Ipv4Addr::new(198, 51, 100, 7); + let peer = SocketAddr::from((Ipv4Addr::LOCALHOST, 51_100)); + for _ in 0..5 { + let outcome = handle_with_header(&gateway, peer, claimed).await; + assert!(outcome.result.is_ok(), "a rate-limited connection is dropped silently"); + assert_eq!( + outcome.client_ip, + Some(IpAddr::V4(claimed)), + "the header IP, not the proxy IP, is the client identity" + ); + } + assert!( + gateway.blacklist.is_blocked(IpAddr::V4(claimed)), + "repeated rate-limit hits must ban the header IP" + ); + assert!( + !gateway.blacklist.is_blocked(peer.ip()), + "the trusted proxy itself must stay unbanned" + ); +} + +/// Header addresses that cannot be a real client are rejected and the +/// transport peer becomes the identity. +#[test] +fn rejects_header_addresses_that_cannot_be_a_client() { + let config = ProxyProtocolConfig { + trusted_cidrs: vec![cidr("10.0.0.0/8")], + ..ProxyProtocolConfig::default() + }; + let cases = [ + ("0.0.0.0", ClientIpRejection::Unspecified), + ("::", ClientIpRejection::Unspecified), + ("127.0.0.1", ClientIpRejection::Loopback), + ("::1", ClientIpRejection::Loopback), + ("169.254.10.1", ClientIpRejection::LinkLocal), + ("fe80::1", ClientIpRejection::LinkLocal), + ("224.0.0.1", ClientIpRejection::Multicast), + ("ff02::1", ClientIpRejection::Multicast), + ("255.255.255.255", ClientIpRejection::Broadcast), + ("::ffff:127.0.0.1", ClientIpRejection::Loopback), + ("::ffff:169.254.10.1", ClientIpRejection::LinkLocal), + ]; + for (address, expected) in cases { + let ip = address.parse().expect("valid test address"); + assert_eq!(validate_client_ip(&config, ip), Err(expected), "address {address}"); + } +} + +/// A proxy must not be able to claim an address of the proxy network itself. +#[test] +fn rejects_header_address_inside_trusted_cidrs() { + let config = ProxyProtocolConfig { + trusted_cidrs: vec![cidr("198.51.100.0/24"), cidr("2001:db8:2::/48")], + ..ProxyProtocolConfig::default() + }; + assert_eq!( + validate_client_ip(&config, "198.51.100.7".parse().expect("ipv4")), + Err(ClientIpRejection::TrustedProxy) + ); + assert_eq!( + validate_client_ip(&config, "2001:db8:2::7".parse().expect("ipv6")), + Err(ClientIpRejection::TrustedProxy) + ); + assert_eq!( + validate_client_ip(&config, "::ffff:198.51.100.7".parse().expect("mapped ipv4")), + Err(ClientIpRejection::TrustedProxy), + "an IPv4-mapped address must match the IPv4 proxy network" + ); + assert!( + validate_client_ip(&config, "203.0.113.7".parse().expect("ipv4")).is_ok(), + "an address outside the proxy networks is a valid client identity" + ); +} + +/// An invalid header address falls back to the transport peer, and the peer +/// (itself a trusted proxy) is never banned. +#[tokio::test] +async fn invalid_header_ip_falls_back_to_peer_and_suppresses_bans() { + let mut config = Config::default(); + config.limits.rate_limit_pps = 0.0; + config.limits.rate_limit_burst = 0.0; + config.proxy_protocol.ingress_enabled = true; + config.proxy_protocol.trusted_cidrs = vec![cidr("127.0.0.1/32")]; + let gateway = test_gateway(config, HashSet::new()); + + let peer = SocketAddr::from((Ipv4Addr::LOCALHOST, 51_101)); + for _ in 0..8 { + let outcome = handle_with_header(&gateway, peer, Ipv4Addr::LOCALHOST).await; + assert!(outcome.result.is_ok()); + assert_eq!( + outcome.client_ip, + Some(peer.ip()), + "a rejected header IP must fall back to the transport peer" + ); + } + assert!( + gateway.blacklist.is_empty(), + "a trusted proxy address must never be banned" + ); +} + +/// A header claiming an address inside `trusted_cidrs` is rejected, and the +/// fallback peer (also a trusted proxy) is not banned either. +#[tokio::test] +async fn header_inside_trusted_cidrs_is_rejected_and_never_banned() { + let mut config = Config::default(); + config.limits.rate_limit_pps = 0.0; + config.limits.rate_limit_burst = 0.0; + config.proxy_protocol.ingress_enabled = true; + config.proxy_protocol.trusted_cidrs = vec![cidr("127.0.0.1/32"), cidr("198.51.100.0/24")]; + let gateway = test_gateway(config, HashSet::new()); + + let claimed = Ipv4Addr::new(198, 51, 100, 7); + let peer = SocketAddr::from((Ipv4Addr::LOCALHOST, 51_102)); + for _ in 0..5 { + let outcome = handle_with_header(&gateway, peer, claimed).await; + assert_eq!( + outcome.client_ip, + Some(peer.ip()), + "a proxy must not claim another proxy address as the client" + ); + } + assert!( + gateway.blacklist.is_empty(), + "neither the claimed proxy address nor the fallback peer may be banned" + ); +} + +/// Plain traffic without PROXY protocol keeps using the transport peer. +#[tokio::test] +async fn plain_connection_uses_the_transport_peer() { + let gateway = test_gateway(Config::default(), HashSet::new()); + let peer = SocketAddr::from(([198, 51, 100, 20], 51_103)); + let outcome = handle_plain(&gateway, peer).await; + assert_eq!(outcome.client_ip, Some(peer.ip())); +}