fix(edge): IPv6 bind, configurable relay idle, per-gateway budget, PROXY identity

- bind.address is parsed as IpAddr (no ":::port"), the socket domain follows
  the family, and "::" binds dual-stack. IPv6 was impossible before.
- limits.relay_idle_timeout_secs (default 300, 1..=86400) replaces the
  hardcoded relay constant; the deploy examples document it.
- The connection budget is an Arc<Semaphore> owned by Gateway instead of a
  process-wide OnceLock, so tests and multiple gateways stay independent.
- PROXY header identity is validated: unspecified, loopback, link-local,
  multicast, broadcast and addresses inside trusted_cidrs are rejected, and
  IPv4-mapped IPv6 is normalised first. An invalid header falls back to the
  transport peer. Bans targeting trusted_cidrs are suppressed, closing the
  ban-amplification path through a trusted load balancer.
- TrustedProxyCidr::contains no longer overflows its mask for IPv6 prefixes
  shorter than /96.
This commit is contained in:
Loki 2026-11-16 12:36:19 +01:00
parent 9baae25acb
commit 324ef12b27
Signed by: boba
GPG key ID: 6CDE87BC9A778D97
15 changed files with 530 additions and 76 deletions

View file

@ -23,6 +23,7 @@ handshake_timeout_secs = 5
max_connections_per_ip = 10 max_connections_per_ip = 10
rate_limit_pps = 5.0 rate_limit_pps = 5.0
rate_limit_burst = 10.0 rate_limit_burst = 10.0
relay_idle_timeout_secs = 300
[ban] [ban]
ban_duration_secs = 3600 ban_duration_secs = 3600

View file

@ -16,6 +16,7 @@ handshake_timeout_secs = 5
max_connections_per_ip = 10 max_connections_per_ip = 10
rate_limit_pps = 5.0 rate_limit_pps = 5.0
rate_limit_burst = 10.0 rate_limit_burst = 10.0
relay_idle_timeout_secs = 300
[ban] [ban]
ban_duration_secs = 3600 ban_duration_secs = 3600

View file

@ -26,6 +26,7 @@ handshake_timeout_secs = 5
max_connections_per_ip = 10 max_connections_per_ip = 10
rate_limit_pps = 5.0 rate_limit_pps = 5.0
rate_limit_burst = 10.0 rate_limit_burst = 10.0
relay_idle_timeout_secs = 300
[ban] [ban]
ban_duration_secs = 3600 ban_duration_secs = 3600

View file

@ -31,6 +31,7 @@ handshake_timeout_secs = 5 # таймаут чтения (slowloris-защ
max_connections_per_ip = 10 # одновременные соединения с одного IP max_connections_per_ip = 10 # одновременные соединения с одного IP
rate_limit_pps = 5.0 # лимит коннектов/сек с одного IP rate_limit_pps = 5.0 # лимит коннектов/сек с одного IP
rate_limit_burst = 10.0 # burst rate_limit_burst = 10.0 # burst
relay_idle_timeout_secs = 300 # простой relay до разрыва, 1..86400
[ban] [ban]
ban_duration_secs = 3600 ban_duration_secs = 3600
@ -84,6 +85,15 @@ max_header_bytes = 512
Все секции опциональны — при отсутствии используются дефолты из таблицы выше. Все секции опциональны — при отсутствии используются дефолты из таблицы выше.
Для ingress PROXY v2 укажи сети именно прокси в `trusted_cidrs`; включение без Для ingress PROXY v2 укажи сети именно прокси в `trusted_cidrs`; включение без
доверенной сети отклоняется при чтении конфига. доверенной сети отклоняется при чтении конфига.
`trusted_cidrs` — это точные адреса/CIDR самих прокси, а не «весь DC»: IP из
заголовка используется для per-IP лимитов, репутации и банов, поэтому любой хост
внутри `trusted_cidrs` может назвать себя кем угодно. Заголовок с адресом из
`trusted_cidrs` (прокси не может быть клиентом за самим собой) или с
unspecified/loopback/link-local/multicast/broadcast адресом отклоняется, и
клиентом считается transport peer — событие пишется в debug один раз на
соединение. Адреса из `trusted_cidrs` никогда не банятся локально, в Redis и XDP;
попытка бана подавляется с предупреждением в логе.
`send_proxy_protocol` отправляет заголовок на Paper backend; прежнее имя `send_proxy_protocol` отправляет заголовок на Paper backend; прежнее имя
`proxy_protocol` внутри `[protocol.minecraft]` остаётся допустимым alias. `proxy_protocol` внутри `[protocol.minecraft]` остаётся допустимым alias.
@ -97,6 +107,7 @@ max_header_bytes = 512
| `JWT_EXPIRATION_SECS` | нет | `86400` | Время жизни токена | | `JWT_EXPIRATION_SECS` | нет | `86400` | Время жизни токена |
| `API_PASSWORD` | да | — | Пароль для `/api/v1/auth/login`, минимум 16 символов; `changeme` запрещён | | `API_PASSWORD` | да | — | Пароль для `/api/v1/auth/login`, минимум 16 символов; `changeme` запрещён |
| `CORS_ORIGIN` | нет | — (CORS выключен) | `*` — разрешить все источники; список через запятую; пусто/не задано — запретить cross-origin | | `CORS_ORIGIN` | нет | — (CORS выключен) | `*` — разрешить все источники; список через запятую; пусто/не задано — запретить cross-origin |
| `MANAGER_TRUSTED_PROXIES` | нет | — (никому не доверяем) | Сети reverse-прокси через запятую (`10.0.0.0/8,2001:db8::/32`). `X-Forwarded-For` учитывается только для запросов, пришедших с адреса из этих сетей; иначе клиентом считается TCP peer. Пусто/не задано — заголовок игнорируется полностью. Опечатка или пустая запись в списке не даёт менеджеру стартовать |
Manager всегда слушает `0.0.0.0:8080`. Manager всегда слушает `0.0.0.0:8080`.
@ -150,6 +161,10 @@ clang -O2 -g -target bpf -c xdp/core/universal_filter.c -o /tmp/universal_filter
`whitelist6_map`) при attach и обходит state machine, throttle и баны. `whitelist6_map`) при attach и обходит state machine, throttle и баны.
- **CIDR-баны**: XDP матчит по LPM-трие, userspace — по диапазонам адресов, поэтому бан - **CIDR-баны**: XDP матчит по LPM-трие, userspace — по диапазонам адресов, поэтому бан
`/24` или `/64` покрывает всю подсеть. `/24` или `/64` покрывает всю подсеть.
- **Оценка числа источников** в префиксе: XDP ведёт 64-битный bitmap хешей source-адресов,
userspace считает popcount. Благодаря этому spoof-gate (не банить `/24`, если в нём лишь
пара адресов) работает и на XDP-пути. Счётчики префиксов сбрасываются при каждом чтении,
то есть окно = интервал опроса монитора.
Покрытие семейств адресов: Покрытие семейств адресов:

View file

@ -21,7 +21,6 @@ impl Default for BindConfig {
} }
} }
} }
fn default_bind_address() -> String { fn default_bind_address() -> String {
"0.0.0.0".to_string() "0.0.0.0".to_string()
} }
@ -43,7 +42,6 @@ impl Default for BackendConfig {
} }
} }
} }
fn default_upstreams() -> Vec<String> { fn default_upstreams() -> Vec<String> {
vec!["127.0.0.1:25566".to_string()] vec!["127.0.0.1:25566".to_string()]
} }
@ -60,7 +58,6 @@ impl Default for WorkerConfig {
Self { count: 4 } Self { count: 4 }
} }
} }
fn default_worker_count() -> usize { fn default_worker_count() -> usize {
4 4
} }
@ -76,6 +73,11 @@ pub struct LimitsConfig {
pub rate_limit_pps: f64, pub rate_limit_pps: f64,
#[serde(default = "default_rate_limit_burst")] #[serde(default = "default_rate_limit_burst")]
pub rate_limit_burst: f64, pub rate_limit_burst: f64,
#[serde(
default = "default_relay_idle_timeout",
deserialize_with = "deserialize_relay_idle_timeout"
)]
pub relay_idle_timeout_secs: u64,
} }
impl Default for LimitsConfig { impl Default for LimitsConfig {
@ -85,10 +87,10 @@ impl Default for LimitsConfig {
max_connections_per_ip: default_max_connections_per_ip(), max_connections_per_ip: default_max_connections_per_ip(),
rate_limit_pps: default_rate_limit_pps(), rate_limit_pps: default_rate_limit_pps(),
rate_limit_burst: default_rate_limit_burst(), rate_limit_burst: default_rate_limit_burst(),
relay_idle_timeout_secs: default_relay_idle_timeout(),
} }
} }
} }
fn default_handshake_timeout() -> u64 { fn default_handshake_timeout() -> u64 {
5 5
} }
@ -101,6 +103,22 @@ fn default_rate_limit_pps() -> f64 {
fn default_rate_limit_burst() -> f64 { fn default_rate_limit_burst() -> f64 {
10.0 10.0
} }
fn default_relay_idle_timeout() -> u64 {
300
}
/// Deserializes and validates `limits.relay_idle_timeout_secs`.
/// # Errors
/// Fails outside `1..=86_400`; zero would cut every relayed session immediately.
fn deserialize_relay_idle_timeout<'de, D: Deserializer<'de>>(deserializer: D) -> Result<u64, D::Error> {
let secs = u64::deserialize(deserializer)?;
if (1..=86_400).contains(&secs) {
return Ok(secs);
}
Err(serde::de::Error::custom(
"limits.relay_idle_timeout_secs must be between 1 and 86400",
))
}
/// `[ban]` section: default ban duration. /// `[ban]` section: default ban duration.
#[derive(Debug, Clone, Deserialize)] #[derive(Debug, Clone, Deserialize)]
@ -116,7 +134,6 @@ impl Default for BanConfig {
} }
} }
} }
fn default_ban_duration() -> u64 { fn default_ban_duration() -> u64 {
3600 3600
} }
@ -153,11 +170,11 @@ impl TrustedProxyCidr {
/// Returns whether `ip` belongs to this network. /// Returns whether `ip` belongs to this network.
#[must_use] #[must_use]
pub fn contains(&self, ip: IpAddr) -> bool { pub fn contains(&self, ip: IpAddr) -> bool {
let mask4 = mask_v4(self.prefix_len); // Masks are family-specific: a prefix above 32 would underflow the v4 shift.
let mask6 = mask_v6(self.prefix_len); let prefix = self.prefix_len;
match (self.network, ip) { match (self.network, ip) {
(IpAddr::V4(network), IpAddr::V4(ip)) => u32::from(network) & mask4 == u32::from(ip) & mask4, (IpAddr::V4(net), IpAddr::V4(ip)) => u32::from(net) & mask_v4(prefix) == u32::from(ip) & mask_v4(prefix),
(IpAddr::V6(network), IpAddr::V6(ip)) => u128::from(network) & mask6 == u128::from(ip) & mask6, (IpAddr::V6(net), IpAddr::V6(ip)) => u128::from(net) & mask_v6(prefix) == u128::from(ip) & mask_v6(prefix),
_ => false, _ => false,
} }
} }
@ -167,18 +184,12 @@ impl FromStr for TrustedProxyCidr {
type Err = String; type Err = String;
fn from_str(value: &str) -> Result<Self, Self::Err> { fn from_str(value: &str) -> Result<Self, Self::Err> {
let (address, prefix) = match value.split_once('/') { let (address, prefix) = value.split_once('/').map_or((value, None), |(a, p)| (a, Some(p)));
Some((address, prefix)) => (address, Some(prefix)),
None => (value, None),
};
let network = IpAddr::from_str(address).map_err(|_| format!("invalid trusted proxy CIDR: {value}"))?; let network = IpAddr::from_str(address).map_err(|_| format!("invalid trusted proxy CIDR: {value}"))?;
let max_prefix = if network.is_ipv4() { 32 } else { 128 }; let max_prefix = if network.is_ipv4() { 32 } else { 128 };
let prefix_len = match prefix { let prefix_len = prefix
Some(prefix) => prefix .map_or(Ok(max_prefix), |prefix| prefix.parse::<u8>())
.parse::<u8>() .map_err(|_| format!("invalid trusted proxy CIDR prefix: {value}"))?;
.map_err(|_| format!("invalid trusted proxy CIDR prefix: {value}"))?,
None => max_prefix,
};
if prefix_len > max_prefix { if prefix_len > max_prefix {
return Err(format!("trusted proxy CIDR prefix is too large: {value}")); return Err(format!("trusted proxy CIDR prefix is too large: {value}"));
} }
@ -231,7 +242,6 @@ impl Default for ProxyProtocolConfig {
} }
} }
} }
fn default_proxy_protocol_timeout_ms() -> u64 { fn default_proxy_protocol_timeout_ms() -> u64 {
750 750
} }

View file

@ -3,8 +3,10 @@
use crate::config::Config; use crate::config::Config;
use crate::engine::tunnel::Gateway; use crate::engine::tunnel::Gateway;
use crate::traffic::hook::TrafficHook; use crate::traffic::hook::TrafficHook;
use anyhow::Context;
use socket2::{Domain, Socket, Type}; use socket2::{Domain, Socket, Type};
use std::io::ErrorKind; use std::io::ErrorKind;
use std::net::{IpAddr, SocketAddr};
use std::sync::Arc; use std::sync::Arc;
use std::time::Duration; use std::time::Duration;
use tokio::net::TcpListener; use tokio::net::TcpListener;
@ -119,7 +121,7 @@ pub async fn run(
hook: Arc<TrafficHook>, hook: Arc<TrafficHook>,
shutdown: watch::Receiver<bool>, shutdown: watch::Receiver<bool>,
) -> anyhow::Result<()> { ) -> anyhow::Result<()> {
let addr: std::net::SocketAddr = format!("{}:{}", config.bind.address, config.bind.port).parse()?; let addr = parse_listen_addr(&config.bind.address, config.bind.port)?;
let workers = config.workers.count.max(1); let workers = config.workers.count.max(1);
let mut handles = Vec::with_capacity(workers); let mut handles = Vec::with_capacity(workers);
@ -139,11 +141,39 @@ pub async fn run(
Ok(()) Ok(())
} }
fn build_listener(addr: std::net::SocketAddr) -> anyhow::Result<TcpListener> { /// Parses the configured listen address into a socket address.
let socket = Socket::new(Domain::IPV4, Type::STREAM, None)?; ///
/// The address must be an IP literal: resolving a host name at startup would
/// make the bound family depend on resolver state, and the listener needs the
/// family up front to pick the correct socket domain.
///
/// # Errors
/// Fails when `address` is not a valid IPv4 or IPv6 literal.
pub fn parse_listen_addr(address: &str, port: u16) -> anyhow::Result<SocketAddr> {
let ip: IpAddr = address
.parse()
.with_context(|| format!("bind.address must be an IP address, got {address:?}"))?;
Ok(SocketAddr::new(ip, port))
}
/// Binds a TCP listener on `addr` with `SO_REUSEPORT` and `SO_REUSEADDR`.
///
/// The socket domain follows the address family, so IPv6 listen addresses work
/// as well as IPv4 ones. An IPv6 wildcard socket is switched to dual-stack mode
/// (`IPV6_V6ONLY=0`), which keeps `bind.address = "::"` accepting IPv4 clients
/// through IPv4-mapped addresses.
///
/// # Errors
/// Fails when the socket cannot be created, configured or bound.
pub fn build_listener(addr: SocketAddr) -> anyhow::Result<TcpListener> {
let domain = if addr.is_ipv4() { Domain::IPV4 } else { Domain::IPV6 };
let socket = Socket::new(domain, Type::STREAM, None)?;
socket.set_reuse_port(true)?; socket.set_reuse_port(true)?;
socket.set_reuse_address(true)?; socket.set_reuse_address(true)?;
socket.set_nonblocking(true)?; socket.set_nonblocking(true)?;
if addr.is_ipv6() && addr.ip().is_unspecified() {
socket.set_only_v6(false)?;
}
socket.bind(&addr.into())?; socket.bind(&addr.into())?;
socket.listen(65535)?; socket.listen(65535)?;
Ok(TcpListener::from_std(socket.into())?) Ok(TcpListener::from_std(socket.into())?)

View file

@ -17,9 +17,33 @@ use std::sync::Arc;
use std::sync::Mutex; use std::sync::Mutex;
use std::sync::atomic::{AtomicU64, AtomicUsize}; use std::sync::atomic::{AtomicU64, AtomicUsize};
use tokio::sync::Mutex as TokioMutex; use tokio::sync::Mutex as TokioMutex;
use tokio::sync::Semaphore;
mod gateway; mod gateway;
#[cfg(feature = "protocol-minecraft")] #[cfg(feature = "protocol-minecraft")]
mod proxy; mod proxy;
#[cfg(feature = "protocol-minecraft")]
pub use proxy::{ClientIpRejection, validate_client_ip};
/// Multiplier applied to the per-IP limit when sizing the global connection
/// budget: `4 x workers x max_connections_per_ip` in-flight connections.
const CONNECTION_BUDGET_FACTOR: usize = 4;
/// Ceiling of the global budget, so a large worker count cannot exhaust the
/// file-descriptor table of the host.
const CONNECTION_BUDGET_MAX: usize = 100_000;
/// Returns the in-flight connection budget of one gateway instance.
///
/// The budget lives on the gateway instead of in a process-wide static so an
/// embedded or test gateway gets its own limiter and configuration changes are
/// visible without restarting the process.
fn connection_budget_capacity(config: &Config) -> usize {
let workers = config.workers.count.max(1);
let per_ip = config.limits.max_connections_per_ip.max(1) as usize;
workers
.saturating_mul(per_ip)
.saturating_mul(CONNECTION_BUDGET_FACTOR)
.clamp(1, CONNECTION_BUDGET_MAX)
}
/// Shared edge engine state handed to every accept loop by [`Arc`]. /// Shared edge engine state handed to every accept loop by [`Arc`].
pub struct Gateway { pub struct Gateway {
@ -35,6 +59,8 @@ pub struct Gateway {
pub allowed_1s: Arc<AtomicU64>, pub allowed_1s: Arc<AtomicU64>,
pub registry: Arc<ProtocolRegistry>, pub registry: Arc<ProtocolRegistry>,
pub subnet_tracker: Option<Arc<SubnetTracker>>, pub subnet_tracker: Option<Arc<SubnetTracker>>,
budget: Arc<Semaphore>,
budget_rejections: AtomicU64,
upstream_cursor: AtomicUsize, upstream_cursor: AtomicUsize,
} }

View file

@ -10,8 +10,6 @@ mod challenge;
mod pipeline; mod pipeline;
mod relay; mod relay;
pub use relay::RELAY_IDLE_TIMEOUT;
#[cfg(feature = "protocol-minecraft")] #[cfg(feature = "protocol-minecraft")]
use super::proxy; use super::proxy;
use super::{Gateway, GatewayOutcome}; use super::{Gateway, GatewayOutcome};
@ -31,6 +29,7 @@ use std::sync::atomic::AtomicU64;
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
use tokio::net::TcpStream; use tokio::net::TcpStream;
use tokio::sync::Mutex as TokioMutex; use tokio::sync::Mutex as TokioMutex;
use tokio::sync::Semaphore;
impl Gateway { impl Gateway {
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
@ -48,6 +47,7 @@ impl Gateway {
subnet_tracker: Option<Arc<SubnetTracker>>, subnet_tracker: Option<Arc<SubnetTracker>>,
) -> Self { ) -> Self {
let max_connections = config.limits.max_connections_per_ip as usize; let max_connections = config.limits.max_connections_per_ip as usize;
let budget = Arc::new(Semaphore::new(super::connection_budget_capacity(&config)));
Self { Self {
config, config,
rate_limiter, rate_limiter,
@ -61,6 +61,8 @@ impl Gateway {
allowed_1s, allowed_1s,
registry, registry,
subnet_tracker, subnet_tracker,
budget,
budget_rejections: AtomicU64::new(0),
upstream_cursor: std::sync::atomic::AtomicUsize::new(0), upstream_cursor: std::sync::atomic::AtomicUsize::new(0),
} }
} }

View file

@ -21,6 +21,17 @@ impl Gateway {
metrics::CONNECTIONS_TOTAL.with_label_values(&["blocked"]).inc(); metrics::CONNECTIONS_TOTAL.with_label_values(&["blocked"]).inc();
self.reputation.record_bad(ip); self.reputation.record_bad(ip);
if self.reputation.score(ip) < -40 { if self.reputation.score(ip) < -40 {
// Defence in depth: a trusted proxy address must never enter the ban
// path, even when reputation escalation is reached. Bans are
// replicated to Redis and XDP on every node, so one wrong identity
// here could take the ingress proxy out of the whole cluster.
if self.is_trusted_proxy(ip) {
tracing::warn!(
%ip,
"suppressed ban for an address inside proxy_protocol.trusted_cidrs"
);
return;
}
let duration_secs = Duration::from_secs(self.config.ban.ban_duration_secs); let duration_secs = Duration::from_secs(self.config.ban.ban_duration_secs);
self.blacklist.add(ip, duration_secs, "low_reputation"); self.blacklist.add(ip, duration_secs, "low_reputation");
if !self.config.proxy_protocol.ingress_enabled { if !self.config.proxy_protocol.ingress_enabled {
@ -31,6 +42,15 @@ impl Gateway {
} }
} }
/// Returns whether `ip` belongs to a configured trusted proxy network.
fn is_trusted_proxy(&self, ip: IpAddr) -> bool {
self.config
.proxy_protocol
.trusted_cidrs
.iter()
.any(|cidr| cidr.contains(ip))
}
/// Mirrors a ban into the XDP map, if the kernel filter is attached. /// Mirrors a ban into the XDP map, if the kernel filter is attached.
fn xdp_ban(&self, ip: IpAddr, duration_secs: u64) { fn xdp_ban(&self, ip: IpAddr, duration_secs: u64) {
let Some(xdp) = &self.xdp else { let Some(xdp) = &self.xdp else {

View file

@ -1,48 +1,16 @@
//! Filter order and upstream establishment for one accepted connection. //! Filter order and upstream establishment for one accepted connection.
use super::Gateway; use super::Gateway;
use super::RELAY_IDLE_TIMEOUT;
use crate::metrics; use crate::metrics;
use std::sync::OnceLock; use std::sync::atomic::Ordering;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Duration; use std::time::Duration;
use tokio::io::AsyncWriteExt; use tokio::io::AsyncWriteExt;
use tokio::net::TcpStream; use tokio::net::TcpStream;
use tokio::sync::Semaphore;
/// Multiplier applied to the per-IP limit when sizing the global connection
/// budget: `4 x workers x max_connections_per_ip` in-flight connections.
const GLOBAL_BUDGET_FACTOR: usize = 4;
/// Ceiling of the global budget, so a large worker count cannot exhaust the
/// file-descriptor table of the host.
const GLOBAL_BUDGET_MAX: usize = 100_000;
/// One in every `BUDGET_LOG_EVERY` budget rejections is logged. /// One in every `BUDGET_LOG_EVERY` budget rejections is logged.
const BUDGET_LOG_EVERY: u64 = 128; const BUDGET_LOG_EVERY: u64 = 128;
/// Process-wide in-flight connection budget.
///
/// Initialised on first use from the worker count and the per-IP limit of the
/// gateway configuration. A process-wide value is intentional: the budget caps
/// descriptors and tasks for the whole edge node, and there is exactly one
/// gateway per process in production.
static GLOBAL_BUDGET: OnceLock<Semaphore> = OnceLock::new();
/// Total connections rejected by the global budget, used for log rate limiting.
static BUDGET_REJECTIONS: AtomicU64 = AtomicU64::new(0);
impl Gateway { impl Gateway {
/// Returns the process-wide in-flight connection budget.
fn global_budget(&self) -> &'static Semaphore {
GLOBAL_BUDGET.get_or_init(|| {
let workers = self.config.workers.count.max(1);
let per_ip = self.config.limits.max_connections_per_ip.max(1) as usize;
let capacity = workers
.saturating_mul(per_ip)
.saturating_mul(GLOBAL_BUDGET_FACTOR)
.clamp(1, GLOBAL_BUDGET_MAX);
Semaphore::new(capacity)
})
}
/// Runs the filter stages, PoW gate, protocol handshake and relay. /// Runs the filter stages, PoW gate, protocol handshake and relay.
/// ///
/// # Errors /// # Errors
@ -86,11 +54,11 @@ impl Gateway {
let _global_permit = if trusted { let _global_permit = if trusted {
None None
} else { } else {
match self.global_budget().try_acquire() { match self.budget.try_acquire() {
Ok(permit) => Some(permit), Ok(permit) => Some(permit),
Err(_) => { Err(_) => {
metrics::CONNECTIONS_TOTAL.with_label_values(&["overloaded"]).inc(); metrics::CONNECTIONS_TOTAL.with_label_values(&["overloaded"]).inc();
let seen = BUDGET_REJECTIONS.fetch_add(1, Ordering::Relaxed); let seen = self.budget_rejections.fetch_add(1, Ordering::Relaxed);
if seen % BUDGET_LOG_EVERY == 0 { if seen % BUDGET_LOG_EVERY == 0 {
tracing::warn!( tracing::warn!(
"global connection budget exhausted after {seen} rejections, dropping {peer_ip}" "global connection budget exhausted after {seen} rejections, dropping {peer_ip}"
@ -145,7 +113,7 @@ impl Gateway {
handshake_budget, handshake_budget,
) )
.await?; .await?;
self.relay_with_idle(client, backend, RELAY_IDLE_TIMEOUT).await self.relay_with_idle(client, backend, self.relay_idle_timeout()).await
} }
/// Opens the upstream connection under the handshake timeout. /// Opens the upstream connection under the handshake timeout.

View file

@ -8,17 +8,17 @@ use tokio::net::TcpStream;
const RELAY_BUFFER_BYTES: usize = 16 * 1024; const RELAY_BUFFER_BYTES: usize = 16 * 1024;
/// Idle budget of the client/backend relay.
///
/// Minecraft keeps one connection alive for a whole session and a legitimately
/// silent gap (menus, chunk loading, AFK timeouts) can last minutes, so 300s is
/// the safe default: it never cuts a real session and still reaps dead peers.
/// The value is a constant because `[limits]` is owned by a parallel change; it
/// is plumbed as a parameter through [`Gateway::relay_with_idle`] so a config
/// field can be wired in without touching the relay itself.
pub const RELAY_IDLE_TIMEOUT: Duration = Duration::from_secs(300);
impl Gateway { impl Gateway {
/// Returns the configured idle budget of the client/backend relay.
///
/// `limits.relay_idle_timeout_secs` defaults to 300s: Minecraft keeps one
/// connection alive for a whole session and a legitimately silent gap
/// (menus, chunk loading, AFK timeouts) can last minutes, so a short budget
/// would cut real sessions while a missing one would pin descriptors.
pub fn relay_idle_timeout(&self) -> Duration {
Duration::from_secs(self.config.limits.relay_idle_timeout_secs)
}
/// Relays bytes in both directions until either side closes, an I/O error /// Relays bytes in both directions until either side closes, an I/O error
/// occurs or a direction stays silent for `idle`. /// occurs or a direction stays silent for `idle`.
/// ///

View file

@ -2,19 +2,85 @@
//! parsing of headers injected by trusted ingress peers. //! parsing of headers injected by trusted ingress peers.
use crate::config::ProxyProtocolConfig; use crate::config::ProxyProtocolConfig;
use std::net::SocketAddr; use std::net::{IpAddr, Ipv4Addr, SocketAddr};
use std::time::Duration; use std::time::Duration;
use tokio::net::TcpStream; use tokio::net::TcpStream;
mod inbound; mod inbound;
pub(super) use inbound::read_v2_header; pub(super) use inbound::read_v2_header;
/// Reason a PROXY header client address is rejected as an identity.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ClientIpRejection {
/// `0.0.0.0` or `::`: no host behind the header.
Unspecified,
/// The header claims a loopback address, which is never a real client.
Loopback,
/// The header claims a link-local address (`169.254.0.0/16`, `fe80::/10`).
LinkLocal,
/// The header claims a multicast address.
Multicast,
/// The header claims the limited IPv4 broadcast address.
Broadcast,
/// The header claims an address of the proxy network itself.
TrustedProxy,
}
/// Validates the client IP claimed in an inbound PROXY v2 header.
///
/// A trusted proxy is allowed to state who the client is, but it must not be
/// able to name an address that can never be a real remote client (host-local,
/// link-local, multicast, broadcast) or another trusted proxy. Without this
/// check a single compromised or misconfigured proxy could ban arbitrary
/// addresses cluster-wide or poison their reputation.
///
/// # Errors
/// Returns the reason `ip` is not usable as a client identity.
pub fn validate_client_ip(config: &ProxyProtocolConfig, ip: IpAddr) -> Result<(), ClientIpRejection> {
// Collapse IPv4-mapped IPv6 first: the ban tables do the same, so leaving
// `::ffff:127.0.0.1` as v6 would bypass the loopback and trusted-network
// checks and then be normalized into a ban for `127.0.0.1`.
let ip = match ip {
IpAddr::V6(v6) => v6.to_ipv4_mapped().map_or(ip, IpAddr::V4),
IpAddr::V4(_) => ip,
};
if ip.is_unspecified() {
return Err(ClientIpRejection::Unspecified);
}
if ip.is_loopback() {
return Err(ClientIpRejection::Loopback);
}
let link_local = match ip {
IpAddr::V4(v4) => v4.is_link_local(),
IpAddr::V6(v6) => v6.is_unicast_link_local(),
};
if link_local {
return Err(ClientIpRejection::LinkLocal);
}
if ip.is_multicast() {
return Err(ClientIpRejection::Multicast);
}
if ip == IpAddr::V4(Ipv4Addr::BROADCAST) {
return Err(ClientIpRejection::Broadcast);
}
if config.trusted_cidrs.iter().any(|cidr| cidr.contains(ip)) {
return Err(ClientIpRejection::TrustedProxy);
}
Ok(())
}
/// Resolves the effective client address from an inbound PROXY v2 header. /// Resolves the effective client address from an inbound PROXY v2 header.
/// ///
/// Only peers inside `config.trusted_cidrs` may inject a header; anyone else /// Only peers inside `config.trusted_cidrs` may inject a header; anyone else
/// is rejected instead of being believed. The whole header must arrive within /// is rejected instead of being believed. The whole header must arrive within
/// `config.ingress_timeout_ms` and fit in `config.max_header_bytes`. /// `config.ingress_timeout_ms` and fit in `config.max_header_bytes`.
/// ///
/// A header whose client address fails [`validate_client_ip`] is not fatal:
/// the transport peer becomes the client identity and the reason is logged once
/// per connection at debug level. Failing closed would drop every connection
/// behind a misbehaving proxy, while trusting the claimed address would hand
/// the ban path to the proxy.
///
/// # Errors /// # Errors
/// Fails when the peer is not trusted, on timeout and on any header parse error. /// Fails when the peer is not trusted, on timeout and on any header parse error.
pub(super) async fn resolve_client_addr( pub(super) async fn resolve_client_addr(
@ -25,12 +91,24 @@ pub(super) async fn resolve_client_addr(
if !config.trusted_cidrs.iter().any(|cidr| cidr.contains(peer_addr.ip())) { if !config.trusted_cidrs.iter().any(|cidr| cidr.contains(peer_addr.ip())) {
anyhow::bail!("untrusted peer attempted inbound PROXY protocol"); anyhow::bail!("untrusted peer attempted inbound PROXY protocol");
} }
read_v2_header( let claimed = read_v2_header(
client, client,
Duration::from_millis(config.ingress_timeout_ms), Duration::from_millis(config.ingress_timeout_ms),
config.max_header_bytes, config.max_header_bytes,
) )
.await .await?;
match validate_client_ip(config, claimed.ip()) {
Ok(()) => Ok(claimed),
Err(reason) => {
tracing::debug!(
peer = %peer_addr,
claimed = %claimed,
reason = ?reason,
"ignoring PROXY header client address, using the transport peer"
);
Ok(peer_addr)
},
}
} }
/// Encodes a PROXY protocol v2 TCP header for the backend. /// Encodes a PROXY protocol v2 TCP header for the backend.

View file

@ -238,6 +238,30 @@ fn rejects_rate_limit_burst_below_one() {
} }
} }
#[test]
fn relay_idle_timeout_defaults_to_300_and_accepts_the_full_range() {
let config = Config::parse_str("").expect("empty config must parse");
assert_eq!(config.limits.relay_idle_timeout_secs, 300);
for secs in [1u64, 60, 300, 86_400] {
let toml = format!("[limits]\nrelay_idle_timeout_secs = {secs}\n");
let config = Config::parse_str(&toml).expect("in-range idle timeout must parse");
assert_eq!(config.limits.relay_idle_timeout_secs, secs);
}
}
#[test]
fn rejects_out_of_range_relay_idle_timeout() {
for secs in [0u64, 86_401] {
let toml = format!("[limits]\nrelay_idle_timeout_secs = {secs}\n");
let err = Config::parse_str(&toml).expect_err("idle timeout outside 1..=86400 must be rejected");
assert!(
err.to_string().contains("limits.relay_idle_timeout_secs"),
"idle timeout '{secs}' must be rejected"
);
}
}
#[test] #[test]
fn rejects_out_of_range_ban_duration() { fn rejects_out_of_range_ban_duration() {
let err = Config::parse_str("[ban]\nban_duration_secs = 0\n").expect_err("zero ban duration must be rejected"); let err = Config::parse_str("[ban]\nban_duration_secs = 0\n").expect_err("zero ban duration must be rejected");

View file

@ -3,7 +3,7 @@
use rampart::config::Config; use rampart::config::Config;
use rampart::engine::challenge::DifficultyAdjuster; use rampart::engine::challenge::DifficultyAdjuster;
use rampart::engine::listener::AcceptBackoff; use rampart::engine::listener::{AcceptBackoff, build_listener, parse_listen_addr};
use rampart::engine::tunnel::Gateway; use rampart::engine::tunnel::Gateway;
use rampart::filter::blacklist::Blacklist; use rampart::filter::blacklist::Blacklist;
use rampart::filter::rate_limit::RateLimiter; use rampart::filter::rate_limit::RateLimiter;
@ -11,7 +11,7 @@ use rampart::protocol::ProtocolRegistry;
use rampart::traffic::reputation::IpReputation; use rampart::traffic::reputation::IpReputation;
use std::collections::HashSet; use std::collections::HashSet;
use std::io; use std::io;
use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
use std::sync::Arc; use std::sync::Arc;
use std::sync::Mutex; use std::sync::Mutex;
use std::sync::atomic::AtomicU64; use std::sync::atomic::AtomicU64;
@ -263,6 +263,45 @@ async fn accept_backoff_sleeps_instead_of_spinning() {
); );
} }
/// The listen address is parsed as an IP literal and the socket domain follows
/// the address family, so IPv6 loopback binds instead of forcing IPv4.
#[tokio::test]
async fn binds_ipv4_and_ipv6_loopback() {
let v4 = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0);
let listener = build_listener(v4).expect("bind IPv4 loopback");
assert!(listener.local_addr().expect("local addr").is_ipv4());
let v6 = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 0);
let listener = build_listener(v6).expect("bind IPv6 loopback");
assert!(listener.local_addr().expect("local addr").is_ipv6());
}
/// `::` must parse as a wildcard IPv6 listen address instead of being
/// concatenated into the invalid `":::port"` form.
#[test]
fn parses_ipv6_wildcard_listen_address() {
let addr = parse_listen_addr("::", 25565).expect("IPv6 wildcard must parse");
assert_eq!(addr, SocketAddr::new(IpAddr::V6(Ipv6Addr::UNSPECIFIED), 25565));
assert!(addr.is_ipv6());
}
/// The relay idle budget is configurable through `limits` instead of being a
/// hardcoded constant.
#[test]
fn relay_idle_timeout_comes_from_limits_config() {
let mut config = Config::default();
config.limits.relay_idle_timeout_secs = 1;
let gateway = test_gateway(config, HashSet::new());
assert_eq!(gateway.relay_idle_timeout(), Duration::from_secs(1));
let default_gateway = test_gateway(Config::default(), HashSet::new());
assert_eq!(
default_gateway.relay_idle_timeout(),
Duration::from_secs(300),
"the default idle budget is 300s"
);
}
/// A silent pair is closed by the relay idle timeout instead of pinning two /// A silent pair is closed by the relay idle timeout instead of pinning two
/// descriptors forever. /// descriptors forever.
#[tokio::test] #[tokio::test]

239
tests/proxy_identity.rs Normal file
View file

@ -0,0 +1,239 @@
//! Client identity behind a trusted PROXY protocol proxy: header address
//! validation, fallback to the transport peer and suppression of bans for
//! addresses inside `proxy_protocol.trusted_cidrs`.
#![cfg(feature = "protocol-minecraft")]
use rampart::config::{Config, ProxyProtocolConfig, TrustedProxyCidr};
use rampart::engine::challenge::DifficultyAdjuster;
use rampart::engine::tunnel::{ClientIpRejection, Gateway, GatewayOutcome, validate_client_ip};
use rampart::filter::blacklist::Blacklist;
use rampart::filter::rate_limit::RateLimiter;
use rampart::protocol::ProtocolRegistry;
use rampart::traffic::reputation::IpReputation;
use std::collections::HashSet;
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
use std::sync::Arc;
use std::sync::Mutex;
use std::sync::atomic::AtomicU64;
use tokio::io::AsyncWriteExt;
use tokio::net::{TcpListener, TcpStream};
/// Builds a gateway with no XDP/ClickHouse writers and an empty protocol
/// registry, so every connection stops at the admission filters.
fn test_gateway(config: Config, whitelist: HashSet<IpAddr>) -> Arc<Gateway> {
Arc::new(Gateway::new(
Arc::new(config.clone()),
Arc::new(RateLimiter::new(
config.limits.rate_limit_pps,
config.limits.rate_limit_burst,
)),
Arc::new(Blacklist::new()),
Arc::new(Mutex::new(DifficultyAdjuster::default())),
Arc::new(whitelist),
Arc::new(IpReputation::new()),
None,
None,
Arc::new(AtomicU64::new(0)),
Arc::new(ProtocolRegistry::new()),
None,
))
}
/// Encodes a PROXY protocol v2 TCP-over-IPv4 header.
fn proxy_v2_header(source: Ipv4Addr, destination: Ipv4Addr) -> Vec<u8> {
const SIGNATURE: [u8; 12] = *b"\r\n\r\n\0\r\nQUIT\n";
let mut header = Vec::with_capacity(28);
header.extend_from_slice(&SIGNATURE);
header.push(0x21);
header.push(0x11);
header.extend_from_slice(&12u16.to_be_bytes());
header.extend_from_slice(&source.octets());
header.extend_from_slice(&destination.octets());
header.extend_from_slice(&40_000u16.to_be_bytes());
header.extend_from_slice(&25565u16.to_be_bytes());
header
}
/// Runs one connection through the gateway after sending a PROXY v2 header that
/// claims `claimed` as the client address. The transport peer passed to the
/// gateway is `peer`, which lets the test model a trusted proxy explicitly.
async fn handle_with_header(gateway: &Gateway, peer: SocketAddr, claimed: Ipv4Addr) -> GatewayOutcome {
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind loopback");
let local = listener.local_addr().expect("local addr");
let (client, accepted) = tokio::join!(TcpStream::connect(local), listener.accept());
let mut client = client.expect("connect");
let server = accepted.expect("accept").0;
let destination = match local.ip() {
IpAddr::V4(ip) => ip,
IpAddr::V6(_) => Ipv4Addr::LOCALHOST,
};
client
.write_all(&proxy_v2_header(claimed, destination))
.await
.expect("write PROXY header");
let outcome = gateway.handle(server, peer).await;
drop(client);
outcome
}
/// Runs one connection without a PROXY header.
async fn handle_plain(gateway: &Gateway, peer: SocketAddr) -> GatewayOutcome {
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind loopback");
let local = listener.local_addr().expect("local addr");
let (_client, server) = tokio::join!(TcpStream::connect(local), listener.accept());
gateway.handle(server.expect("accept").0, peer).await
}
/// Parses a trusted proxy network for the test config.
fn cidr(value: &str) -> TrustedProxyCidr {
value.parse().expect("valid trusted proxy CIDR")
}
/// A trusted proxy may state a public client address, and that address is the
/// one used for admission decisions and bans.
#[tokio::test]
async fn trusted_proxy_header_ip_is_used_for_identity_and_bans() {
let mut config = Config::default();
config.limits.rate_limit_pps = 0.0;
config.limits.rate_limit_burst = 0.0;
config.proxy_protocol.ingress_enabled = true;
config.proxy_protocol.trusted_cidrs = vec![cidr("127.0.0.1/32")];
let gateway = test_gateway(config, HashSet::new());
let claimed = Ipv4Addr::new(198, 51, 100, 7);
let peer = SocketAddr::from((Ipv4Addr::LOCALHOST, 51_100));
for _ in 0..5 {
let outcome = handle_with_header(&gateway, peer, claimed).await;
assert!(outcome.result.is_ok(), "a rate-limited connection is dropped silently");
assert_eq!(
outcome.client_ip,
Some(IpAddr::V4(claimed)),
"the header IP, not the proxy IP, is the client identity"
);
}
assert!(
gateway.blacklist.is_blocked(IpAddr::V4(claimed)),
"repeated rate-limit hits must ban the header IP"
);
assert!(
!gateway.blacklist.is_blocked(peer.ip()),
"the trusted proxy itself must stay unbanned"
);
}
/// Header addresses that cannot be a real client are rejected and the
/// transport peer becomes the identity.
#[test]
fn rejects_header_addresses_that_cannot_be_a_client() {
let config = ProxyProtocolConfig {
trusted_cidrs: vec![cidr("10.0.0.0/8")],
..ProxyProtocolConfig::default()
};
let cases = [
("0.0.0.0", ClientIpRejection::Unspecified),
("::", ClientIpRejection::Unspecified),
("127.0.0.1", ClientIpRejection::Loopback),
("::1", ClientIpRejection::Loopback),
("169.254.10.1", ClientIpRejection::LinkLocal),
("fe80::1", ClientIpRejection::LinkLocal),
("224.0.0.1", ClientIpRejection::Multicast),
("ff02::1", ClientIpRejection::Multicast),
("255.255.255.255", ClientIpRejection::Broadcast),
("::ffff:127.0.0.1", ClientIpRejection::Loopback),
("::ffff:169.254.10.1", ClientIpRejection::LinkLocal),
];
for (address, expected) in cases {
let ip = address.parse().expect("valid test address");
assert_eq!(validate_client_ip(&config, ip), Err(expected), "address {address}");
}
}
/// A proxy must not be able to claim an address of the proxy network itself.
#[test]
fn rejects_header_address_inside_trusted_cidrs() {
let config = ProxyProtocolConfig {
trusted_cidrs: vec![cidr("198.51.100.0/24"), cidr("2001:db8:2::/48")],
..ProxyProtocolConfig::default()
};
assert_eq!(
validate_client_ip(&config, "198.51.100.7".parse().expect("ipv4")),
Err(ClientIpRejection::TrustedProxy)
);
assert_eq!(
validate_client_ip(&config, "2001:db8:2::7".parse().expect("ipv6")),
Err(ClientIpRejection::TrustedProxy)
);
assert_eq!(
validate_client_ip(&config, "::ffff:198.51.100.7".parse().expect("mapped ipv4")),
Err(ClientIpRejection::TrustedProxy),
"an IPv4-mapped address must match the IPv4 proxy network"
);
assert!(
validate_client_ip(&config, "203.0.113.7".parse().expect("ipv4")).is_ok(),
"an address outside the proxy networks is a valid client identity"
);
}
/// An invalid header address falls back to the transport peer, and the peer
/// (itself a trusted proxy) is never banned.
#[tokio::test]
async fn invalid_header_ip_falls_back_to_peer_and_suppresses_bans() {
let mut config = Config::default();
config.limits.rate_limit_pps = 0.0;
config.limits.rate_limit_burst = 0.0;
config.proxy_protocol.ingress_enabled = true;
config.proxy_protocol.trusted_cidrs = vec![cidr("127.0.0.1/32")];
let gateway = test_gateway(config, HashSet::new());
let peer = SocketAddr::from((Ipv4Addr::LOCALHOST, 51_101));
for _ in 0..8 {
let outcome = handle_with_header(&gateway, peer, Ipv4Addr::LOCALHOST).await;
assert!(outcome.result.is_ok());
assert_eq!(
outcome.client_ip,
Some(peer.ip()),
"a rejected header IP must fall back to the transport peer"
);
}
assert!(
gateway.blacklist.is_empty(),
"a trusted proxy address must never be banned"
);
}
/// A header claiming an address inside `trusted_cidrs` is rejected, and the
/// fallback peer (also a trusted proxy) is not banned either.
#[tokio::test]
async fn header_inside_trusted_cidrs_is_rejected_and_never_banned() {
let mut config = Config::default();
config.limits.rate_limit_pps = 0.0;
config.limits.rate_limit_burst = 0.0;
config.proxy_protocol.ingress_enabled = true;
config.proxy_protocol.trusted_cidrs = vec![cidr("127.0.0.1/32"), cidr("198.51.100.0/24")];
let gateway = test_gateway(config, HashSet::new());
let claimed = Ipv4Addr::new(198, 51, 100, 7);
let peer = SocketAddr::from((Ipv4Addr::LOCALHOST, 51_102));
for _ in 0..5 {
let outcome = handle_with_header(&gateway, peer, claimed).await;
assert_eq!(
outcome.client_ip,
Some(peer.ip()),
"a proxy must not claim another proxy address as the client"
);
}
assert!(
gateway.blacklist.is_empty(),
"neither the claimed proxy address nor the fallback peer may be banned"
);
}
/// Plain traffic without PROXY protocol keeps using the transport peer.
#[tokio::test]
async fn plain_connection_uses_the_transport_peer() {
let gateway = test_gateway(Config::default(), HashSet::new());
let peer = SocketAddr::from(([198, 51, 100, 20], 51_103));
let outcome = handle_plain(&gateway, peer).await;
assert_eq!(outcome.client_ip, Some(peer.ip()));
}