fix(core): close auth bypass, hangs and memory leaks found by review

Security:
- An empty API_PASSWORD passed validation (only the literal "changeme" was
  rejected) and verify_password("", "") returned true, so a login with an
  empty password minted an admin token. Require >= 16 characters and >= 32
  non-whitespace bytes for JWT_SECRET; verify rejects an empty stored secret.
- syn_challenge_enabled required a real challenge_secret_hex: the public
  placeholder from config.h is now rejected in every spelling.
- docker-compose: Redis/ClickHouse/Grafana are loopback-only and every
  password uses ${VAR:?} instead of a default. An exposed Redis was a way to
  lift bans and publish unbans to the whole fleet with one redis-cli.
- Empty CORS_ORIGIN no longer means "allow any origin".
- ban_cidr / userspace blacklist validate and mask prefixes; CIDR bans are
  stored as ranges, so a /24 or /64 ban actually covers the subnet (before,
  only the network address was blocked).

Availability:
- ClickHouse client has connect/request timeouts and no mutex held across
  await: a hung analytics backend can no longer freeze the ban path. Events
  go through a bounded buffer with a drop counter.
- Redis uses a ConnectionManager with per-command timeouts; pubsub and
  reconcile reset the backoff after success; publish cannot block shutdown.
- Ban durations from pubsub are clamped and use checked_add; the sync task
  survives panics instead of silently dying.
- Blacklist TOCTOU (remove_if) and replace_redis no longer empties the table
  during a Redis reconnect.
- RateLimiter and reputation have hard caps with incremental eviction instead
  of an unbounded DashMap and a stop-the-world retain.
- The attack detector scores before updating its baseline and needs several
  quiet windows to leave UnderAttack, so a sustained flood keeps banning new
  offenders instead of ending after one tick.
- accept() backs off on EMFILE/ENFILE, shutdown Err is treated as shutdown,
  a global semaphore bounds in-flight connections, rate limiting runs before
  PoW, whitelisted IPs are never banned, and connect/handshake/relay have
  timeouts with an idle budget.
- PoW uses the configured difficulty as a floor, the clamp is enforced, and
  nonce framing survives split segments and coalesced handshake bytes.
- Subnet tracker memory is capped (prefixes and per-prefix sources) and the
  monitor bans with the configured TTL.

CI and docs:
- CI runs the test suite with --no-default-features --features protocol-http;
  make ebpf fails on errors; the secrets gate catches weak compose defaults.
- doctor skips protected endpoints without a token and exits non-zero on
  failure; a drain timeout returns exit code 1.
- TODO §2b records the review findings, what is fixed and what is deferred;
  configuration/testing docs describe the new XDP and secret semantics.
This commit is contained in:
Loki 2026-11-16 12:06:23 +01:00
parent 14dcaccdbf
commit c1a076a4a3
Signed by: boba
GPG key ID: 6CDE87BC9A778D97
57 changed files with 4193 additions and 1171 deletions

View file

@ -132,6 +132,9 @@ pub async fn run() -> anyhow::Result<RunExit> {
let subnet_tracker = start_subnet_tracker(&config);
monitor::spawn(
&config.detect.prefix,
Duration::from_secs(config.ban.ban_duration_secs),
blacklist.clone(),
!config.proxy_protocol.ingress_enabled,
subnet_tracker.clone(),
xdp_filter.clone(),
shutdown_rx.clone(),
@ -190,7 +193,9 @@ pub async fn run() -> anyhow::Result<RunExit> {
tracing::info!("Rampart edge starting on {}:{}", config.bind.address, config.bind.port);
tracing::info!("Upstreams: {:?}", config.backend.upstreams);
let adjuster = Arc::new(Mutex::new(DifficultyAdjuster::default()));
// The configured difficulty is the floor and the starting point; the
// adjuster only raises it under load.
let adjuster = Arc::new(Mutex::new(DifficultyAdjuster::with_floor(config.pow.difficulty)));
let gateway = Arc::new(Gateway::new(
config.clone(),
rate_limiter,

View file

@ -18,11 +18,14 @@ pub enum RunExit {
impl RunExit {
/// Process exit code for this outcome.
///
/// A forced drain timeout loses connections that did not finish in time, so
/// it must not look like a clean shutdown to supervisors and scripts.
#[must_use]
pub const fn code(self) -> i32 {
match self {
Self::Ok => 0,
Self::DrainTimeout => 0,
Self::DrainTimeout => 1,
}
}
}
@ -53,15 +56,3 @@ async fn wait_for_signal() {
_ = term.recv() => {}
}
}
#[cfg(test)]
mod tests {
use super::RunExit;
#[test]
fn run_exit_codes_match_previous_process_exit() {
// Both the clean path and the former drain timeout (`process::exit(0)`) exited with code 0.
assert_eq!(RunExit::Ok.code(), 0);
assert_eq!(RunExit::DrainTimeout.code(), 0);
}
}

View file

@ -1,17 +1,14 @@
//! Rampart manager daemon: HTTP API in front of the redis state store.
use axum::{
Router,
http::HeaderValue,
middleware,
Router, middleware,
routing::{get, post},
};
use dashmap::DashMap;
#[cfg(feature = "store-redis")]
use rampart::manager::sync;
use rampart::manager::{AppState, api, auth};
use rampart::manager::{AppState, api, auth, security};
use std::sync::Arc;
use tower_http::cors::CorsLayer;
use tracing_subscriber::EnvFilter;
#[tokio::main]
@ -32,9 +29,7 @@ async fn main() -> anyhow::Result<()> {
let redis_client = redis::Client::open(redis_url)?;
let jwt_secret = std::env::var("JWT_SECRET").map_err(|_| anyhow::anyhow!("JWT_SECRET must be set"))?;
if jwt_secret.len() < 32 {
return Err(anyhow::anyhow!("JWT_SECRET must be at least 32 bytes"));
}
security::validate_jwt_secret(&jwt_secret)?;
let jwt_audience = std::env::var("JWT_AUDIENCE").unwrap_or_else(|_| "rampart".to_string());
let jwt_expiration = std::env::var("JWT_EXPIRATION_SECS")
.unwrap_or_else(|_| "86400".to_string())
@ -42,9 +37,7 @@ async fn main() -> anyhow::Result<()> {
.map_err(|_| anyhow::anyhow!("JWT_EXPIRATION_SECS must be a valid u64"))?;
let api_password = std::env::var("API_PASSWORD").map_err(|_| anyhow::anyhow!("API_PASSWORD must be set"))?;
if api_password == "changeme" {
return Err(anyhow::anyhow!("API_PASSWORD must not be the default 'changeme'"));
}
security::validate_api_password(&api_password)?;
let state = Arc::new(AppState {
#[cfg(feature = "store-redis")]
@ -78,17 +71,13 @@ async fn main() -> anyhow::Result<()> {
.route("/api/v1/nodes", get(api::inventory::nodes::list_nodes));
let protected = protected.route_layer(middleware::from_fn_with_state(state.clone(), auth::auth_middleware));
let cors = match std::env::var("CORS_ORIGIN") {
Ok(origin) if origin.is_empty() || origin == "*" => CorsLayer::new().allow_origin(tower_http::cors::Any),
Ok(origin) => CorsLayer::new().allow_origin(HeaderValue::from_str(&origin)?),
Err(_) => CorsLayer::new().allow_origin(HeaderValue::from_static("http://localhost:5173")),
};
let cors_policy = security::CorsPolicy::parse(std::env::var("CORS_ORIGIN").ok().as_deref())?;
tracing::info!("CORS policy: {cors_policy:?}");
let app = Router::new()
.merge(public)
.merge(protected)
.layer(cors)
.with_state(state);
let mut app = Router::new().merge(public).merge(protected).with_state(state);
if let Some(cors) = cors_policy.cors_layer() {
app = app.layer(cors);
}
let addr = "0.0.0.0:8080";
tracing::info!("Manager API listening on {addr}");

View file

@ -14,7 +14,7 @@ pub async fn run(key: Option<String>, value: Option<String>) -> anyhow::Result<(
println!();
println!("Example keys:");
println!(" workers.count");
println!(" limits.rate_limit_login_pps");
println!(" limits.rate_limit_pps");
println!(" limits.max_connections_per_ip");
Ok(())
},

View file

@ -2,6 +2,11 @@
use crate::xdp::{EnvironmentReport, FilesystemProbe, SystemProbe};
/// Environment variable holding a ready-to-use manager bearer token.
const TOKEN_ENV: &str = "RAMPART_TOKEN";
/// Environment variable holding the manager API password used to log in.
const PASSWORD_ENV: &str = "RAMPART_API_PASSWORD";
/// Finds the interface that owns the default route in `/proc/net/route`
/// (destination `00000000`). Works without root.
fn default_route_interface(probe: &impl SystemProbe) -> Option<String> {
@ -12,6 +17,51 @@ fn default_route_interface(probe: &impl SystemProbe) -> Option<String> {
})
}
/// Obtains a bearer token for the protected manager endpoints.
///
/// `RAMPART_TOKEN` is used as is; otherwise `RAMPART_API_PASSWORD` is exchanged
/// for a token via `/api/v1/auth/login`. Protected endpoints are never called
/// anonymously, so `None` means the caller has to be told to provide a
/// credential instead of seeing a misleading failure.
async fn manager_token(client: &reqwest::Client, manager_url: &str) -> Option<String> {
if let Ok(token) = std::env::var(TOKEN_ENV) {
if !token.trim().is_empty() {
return Some(token);
}
}
let password = std::env::var(PASSWORD_ENV).ok().filter(|value| !value.is_empty())?;
let response = client
.post(format!("{manager_url}/api/v1/auth/login"))
.json(&serde_json::json!({"password": password}))
.send()
.await
.ok()?;
if !response.status().is_success() {
println!("[WARN] Manager login rejected ({})", response.status());
return None;
}
let body = response.json::<serde_json::Value>().await.ok()?;
body["token"].as_str().map(str::to_owned)
}
/// Probes one protected manager endpoint with a bearer token.
async fn check_protected(client: &reqwest::Client, manager_url: &str, path: &str, label: &str, token: Option<&str>) {
let Some(token) = token else {
println!("[SKIP] {label}: set {TOKEN_ENV} or {PASSWORD_ENV} to check it");
return;
};
match client
.get(format!("{manager_url}{path}"))
.bearer_auth(token)
.send()
.await
{
Ok(resp) if resp.status().is_success() => println!("[OK] {label}"),
Ok(resp) => println!("[WARN] {label} unavailable ({})", resp.status()),
Err(e) => println!("[WARN] {label} unavailable: {e}"),
}
}
/// Network/XDP section: what the environment exposes without root.
fn print_network_section(all_ok: &mut bool) {
println!("\nNetwork / XDP");
@ -60,6 +110,10 @@ fn print_network_section(all_ok: &mut bool) {
}
/// Runs every diagnostic and prints the report.
///
/// # Errors
/// Returns an error when any check failed, so scripts and CI see a non-zero
/// exit status instead of a report that only looks alarming.
pub async fn run() -> anyhow::Result<()> {
println!("Rampart Diagnostics");
println!("=====================\n");
@ -67,8 +121,9 @@ pub async fn run() -> anyhow::Result<()> {
let mut all_ok = true;
let manager_url = std::env::var("RAMPART_MANAGER").unwrap_or_else(|_| "http://localhost:8080".to_string());
let client = reqwest::Client::new();
match reqwest::get(format!("{manager_url}/api/v1/health")).await {
match client.get(format!("{manager_url}/api/v1/health")).send().await {
Ok(resp) if resp.status().is_success() => {
println!("[OK] Manager API");
},
@ -78,32 +133,31 @@ pub async fn run() -> anyhow::Result<()> {
},
}
match reqwest::get(format!("{manager_url}/api/v1/blacklist")).await {
Ok(resp) if resp.status().is_success() => {
println!("[OK] Blacklist API");
},
_ => {
println!("[WARN] Blacklist API unavailable");
},
}
match reqwest::get(format!("{manager_url}/api/v1/servers")).await {
Ok(resp) if resp.status().is_success() => {
println!("[OK] Servers API");
},
_ => {
println!("[WARN] Servers API unavailable");
},
}
let token = manager_token(&client, &manager_url).await;
check_protected(
&client,
&manager_url,
"/api/v1/blacklist",
"Blacklist API",
token.as_deref(),
)
.await;
check_protected(
&client,
&manager_url,
"/api/v1/servers",
"Servers API",
token.as_deref(),
)
.await;
print_network_section(&mut all_ok);
println!();
if all_ok {
println!("All checks passed.");
} else {
if !all_ok {
println!("Some checks failed. Run with --verbose for details.");
anyhow::bail!("doctor: one or more checks failed");
}
println!("All checks passed.");
Ok(())
}

View file

@ -19,6 +19,14 @@ pub use sections::{
use serde::Deserialize;
use std::fs;
/// Upper bound of `ban.ban_duration_secs`: one year.
const MAX_BAN_DURATION_SECS: u64 = 31_536_000;
/// Upper bound of `pow.difficulty`.
const MAX_POW_DIFFICULTY: u8 = 10;
/// Placeholder `xdp.challenge_secret_hex` values baked into the XDP defaults
/// (`xdp/core/config.h` and `XdpGlobals::default()`; both spellings exist).
const PLACEHOLDER_CHALLENGE_SECRETS: [&str; 2] = ["a5a5a5a5a5a5a5a5", "a5a55a5a5a5aa5a5"];
#[derive(Debug, Clone, Default, Deserialize)]
pub struct Config {
#[serde(default)]
@ -90,9 +98,38 @@ impl Config {
if self.detect.prefix.enabled && self.detect.prefix.syn_threshold == 0 {
anyhow::bail!("detect.prefix.syn_threshold must be positive when enabled");
}
if self.xdp.syn_challenge_enabled {
let secret = self.xdp.challenge_secret_hex.as_deref().ok_or_else(|| {
anyhow::anyhow!(
"xdp.syn_challenge_enabled requires xdp.challenge_secret_hex; \
generate one with `openssl rand -hex 8`"
)
})?;
if is_placeholder_challenge_secret(secret) {
anyhow::bail!(
"xdp.challenge_secret_hex is the built-in placeholder; \
generate a random secret with `openssl rand -hex 8`"
);
}
}
if self.limits.max_connections_per_ip == 0 {
anyhow::bail!("limits.max_connections_per_ip must be positive");
}
if !self.limits.rate_limit_pps.is_finite() || self.limits.rate_limit_pps <= 0.0 {
anyhow::bail!("limits.rate_limit_pps must be a finite number greater than 0");
}
if !self.limits.rate_limit_burst.is_finite() || self.limits.rate_limit_burst < 1.0 {
anyhow::bail!("limits.rate_limit_burst must be a finite number at least 1");
}
if self.ban.ban_duration_secs == 0 || self.ban.ban_duration_secs > MAX_BAN_DURATION_SECS {
anyhow::bail!("ban.ban_duration_secs must be between 1 and {MAX_BAN_DURATION_SECS}");
}
if self.pow.difficulty == 0 || self.pow.difficulty > MAX_POW_DIFFICULTY {
anyhow::bail!("pow.difficulty must be between 1 and {MAX_POW_DIFFICULTY}");
}
if self.workers.count == 0 {
anyhow::bail!("workers.count must be at least 1: otherwise no accept loops are started");
}
if self.proxy_protocol.ingress_enabled {
if self.proxy_protocol.trusted_cidrs.is_empty() {
anyhow::bail!("proxy_protocol.ingress_enabled requires at least one trusted_cidrs entry");
@ -138,109 +175,17 @@ impl Config {
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn empty_config_uses_defaults() {
let config = Config::parse_str("").expect("empty config should parse");
assert_eq!(config.bind.address, "0.0.0.0");
assert_eq!(config.bind.port, 25565);
assert_eq!(config.backend.upstreams, vec!["127.0.0.1:25566"]);
assert!(!config.pow.enabled);
assert!(!config.xdp.enabled);
}
#[test]
fn full_config_parses() {
let config = Config::parse_str(
r#"
[bind]
address = "10.0.0.1"
port = 443
[backend]
upstreams = ["192.168.1.10:8080", "192.168.1.11:8080"]
[pow]
enabled = true
difficulty = 6
whitelist = ["203.0.113.7"]
"#,
)
.expect("config should parse");
assert_eq!(config.bind.port, 443);
assert_eq!(config.backend.upstreams.len(), 2);
assert!(config.pow.enabled);
assert_eq!(config.pow.difficulty, 6);
}
#[test]
fn empty_upstreams_rejected() {
let result = Config::parse_str(
r#"
[backend]
upstreams = []
"#,
);
assert!(result.is_err());
}
#[test]
fn invalid_upstream_rejected() {
let result = Config::parse_str(
r#"
[backend]
upstreams = ["not-an-addr"]
"#,
);
assert!(result.is_err());
}
#[test]
fn invalid_whitelist_entry_rejected() {
let result = Config::parse_str("whitelist = [\"999.999.1.1\"]");
assert!(result.is_err());
}
#[test]
fn prefix_detect_defaults_disabled() {
let config = Config::parse_str("").expect("empty config should parse");
assert!(!config.detect.prefix.enabled);
assert_eq!(config.detect.prefix.syn_threshold, 500);
assert_eq!(config.detect.prefix.window_secs, 10);
assert_eq!(config.detect.prefix.min_unique_sources, 16);
}
#[test]
fn prefix_detect_section_parses() {
let config = Config::parse_str(
r#"
[detect.prefix]
enabled = true
syn_threshold = 1000
window_secs = 5
min_unique_sources = 8
"#,
)
.expect("detect.prefix section should parse");
assert!(config.detect.prefix.enabled);
assert_eq!(config.detect.prefix.syn_threshold, 1000);
assert_eq!(config.detect.prefix.window_secs, 5);
assert_eq!(config.detect.prefix.min_unique_sources, 8);
}
#[test]
fn prefix_detect_zero_threshold_rejected_when_enabled() {
let result = Config::parse_str(
r#"
[detect.prefix]
enabled = true
syn_threshold = 0
"#,
);
assert!(result.is_err());
}
/// Returns whether `secret` spells a built-in placeholder challenge secret.
///
/// Hex digits are compared case-insensitively, with an optional `0x` prefix and
/// `_`, `:` and `-` separators ignored, so every spelling of the placeholder is
/// caught.
fn is_placeholder_challenge_secret(secret: &str) -> bool {
let normalized = secret
.trim()
.trim_start_matches("0x")
.trim_start_matches("0X")
.replace(['_', ':', '-'], "")
.to_ascii_lowercase();
PLACEHOLDER_CHALLENGE_SECRETS.contains(&normalized.as_str())
}

View file

@ -4,10 +4,20 @@ use crate::metrics;
use std::collections::VecDeque;
use std::time::Instant;
/// Lowest usable difficulty: 0 disables the PoW gate before the adjuster runs.
pub const MIN_DIFFICULTY: u8 = 1;
/// Highest adaptive difficulty. Every extra hex digit quadruples solver work.
pub const MAX_DIFFICULTY: u8 = 12;
/// Difficulty the adjuster climbs to under the heaviest observed load.
const DEFAULT_MAX_DIFFICULTY: u8 = 10;
/// Adapts the PoW difficulty to the current connection rate.
///
/// The rate is counted over a sliding one-second window and the resulting
/// difficulty is clamped to `[min, max]`.
/// difficulty is clamped to `[min, max]`. The lower bound is also the floor and
/// the starting difficulty, so a configured value takes effect immediately.
pub struct DifficultyAdjuster {
window: VecDeque<Instant>,
min: u8,
@ -22,17 +32,36 @@ impl Default for DifficultyAdjuster {
}
impl DifficultyAdjuster {
/// Creates an adjuster clamped to `min.max(4)` and `max.min(10)`.
/// Creates an adjuster with the explicit `[min, max]` range.
///
/// Inverted bounds are swapped, and both ends are clamped to
/// [`MIN_DIFFICULTY`]..=[`MAX_DIFFICULTY`]. The lower bound doubles as the
/// starting difficulty.
#[must_use]
pub fn new(min: u8, max: u8) -> Self {
let (min, max) = if min > max { (max, min) } else { (min, max) };
let min = min.clamp(MIN_DIFFICULTY, MAX_DIFFICULTY);
let max = max.clamp(min, MAX_DIFFICULTY);
Self {
window: VecDeque::new(),
min: min.max(4),
max: max.min(10),
current: min.max(4),
min,
max,
current: min,
}
}
/// Creates an adjuster whose floor and starting point is the configured PoW
/// difficulty.
///
/// The ceiling stays at the default maximum, or rises to the configured
/// value when that value is higher. Values outside
/// [`MIN_DIFFICULTY`]..=[`MAX_DIFFICULTY`] are clamped.
#[must_use]
pub fn with_floor(difficulty: u8) -> Self {
let floor = difficulty.clamp(MIN_DIFFICULTY, MAX_DIFFICULTY);
Self::new(floor, floor.max(DEFAULT_MAX_DIFFICULTY))
}
/// Records one connection and recomputes the difficulty for the window.
pub fn record_connection(&mut self) {
let now = Instant::now();
@ -64,33 +93,18 @@ impl DifficultyAdjuster {
self.current
}
fn compute_difficulty(&self) -> u8 {
/// Rate step before it is clamped to the configured range.
fn rate_difficulty(&self) -> u8 {
match self.window.len() {
cps if cps > 500 => self.max.max(self.min),
cps if cps > 500 => MAX_DIFFICULTY,
cps if cps > 200 => 8,
cps if cps > 50 => 6,
_ => self.min,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn adjuster_raises_difficulty_under_load() {
let mut adjuster = DifficultyAdjuster::new(4, 10);
for _ in 0..600 {
adjuster.record_connection();
}
assert_eq!(adjuster.current_difficulty(), 10);
}
#[test]
fn adjuster_stays_minimal_when_idle() {
let mut adjuster = DifficultyAdjuster::new(4, 10);
adjuster.record_connection();
assert_eq!(adjuster.current_difficulty(), 4);
fn compute_difficulty(&self) -> u8 {
// The rate ladder is global; the configured range is authoritative.
self.rate_difficulty().clamp(self.min, self.max)
}
}

View file

@ -4,5 +4,5 @@
mod difficulty;
mod pow;
pub use difficulty::DifficultyAdjuster;
pub use difficulty::{DifficultyAdjuster, MAX_DIFFICULTY, MIN_DIFFICULTY};
pub use pow::{Challenge, enforce, solve};

View file

@ -1,3 +1,5 @@
//! Universal SHA-256 hashcash challenge generation, solver and verifier.
use rand::Rng;
use sha2::{Digest, Sha256};
use std::net::IpAddr;
@ -5,11 +7,15 @@ use std::time::{Duration, Instant};
use subtle::ConstantTimeEq;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpStream;
use tokio::time::timeout;
const CHALLENGE_TTL_SECS: u64 = 30;
const MAX_NONCE_LEN: usize = 64;
const ALLOWED_HEX_PREFIX: [u8; 4] = *b"0123";
/// Maximum time a peer gets to read the challenge and return a nonce.
const NONCE_TIMEOUT: Duration = Duration::from_secs(10);
/// Active PoW challenge issued to one connection.
pub struct Challenge {
token: [u8; 32],
@ -32,7 +38,9 @@ impl Challenge {
}
/// Builds a challenge with an explicit token.
///
/// Intended for deterministic tests, where a random token makes the outcome
/// hard to predict.
#[must_use]
pub fn with_token(token: [u8; 32], difficulty: u8) -> Self {
Self {
@ -104,8 +112,14 @@ pub fn solve(challenge: &str, difficulty: u8) -> Option<String> {
/// Runs the textual PoW gate on `stream`: issues a challenge and checks the
/// answer.
///
/// Wire format: the server writes the hex challenge followed by `\n`; the
/// client answers with the nonce followed by `\n`. A reply terminated by EOF is
/// also accepted for compatibility with clients that close their write half
/// after sending the nonce. Anything the client sent after the delimiter stays
/// in the socket buffer and is seen by the protocol handshake that runs next.
///
/// # Errors
/// Only I/O failures are reported; a wrong solution is `Ok(false)`.
/// Only I/O failures are reported; a wrong or missing solution is `Ok(false)`.
pub async fn enforce(stream: &mut TcpStream, peer_ip: IpAddr, difficulty: u8) -> anyhow::Result<bool> {
if difficulty == 0 {
tracing::debug!("pow: difficulty 0, skipping for {peer_ip}");
@ -114,22 +128,21 @@ pub async fn enforce(stream: &mut TcpStream, peer_ip: IpAddr, difficulty: u8) ->
let mut challenge = Challenge::generate(difficulty);
let line = format!("{}\n", challenge.challenge_string());
stream.write_all(line.as_bytes()).await?;
timeout(NONCE_TIMEOUT, stream.write_all(line.as_bytes())).await??;
let mut buf = [0u8; MAX_NONCE_LEN + 1];
let n = tokio::time::timeout(Duration::from_secs(10), stream.read(&mut buf)).await??;
if n == 0 {
tracing::debug!("pow: no response from {peer_ip}");
let nonce = match timeout(NONCE_TIMEOUT, read_nonce_line(stream)).await {
Ok(result) => result?,
Err(_) => {
tracing::debug!("pow: nonce read timed out for {peer_ip}");
return Ok(false);
},
};
let Some(nonce) = nonce else {
tracing::debug!("pow: invalid or missing nonce from {peer_ip}");
return Ok(false);
}
};
let nonce = std::str::from_utf8(&buf[..n.min(MAX_NONCE_LEN)]).unwrap_or("").trim();
if nonce.is_empty() || nonce.len() > MAX_NONCE_LEN {
tracing::debug!("pow: invalid nonce from {peer_ip}");
return Ok(false);
}
let valid = challenge.verify(nonce);
let valid = challenge.verify(&nonce);
tracing::debug!(
"pow: verification {} for {peer_ip}",
if valid { "passed" } else { "failed" }
@ -137,54 +150,43 @@ pub async fn enforce(stream: &mut TcpStream, peer_ip: IpAddr, difficulty: u8) ->
Ok(valid)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn solve_then_verify_roundtrip() {
let mut challenge = Challenge::generate(2);
let nonce = solve(&challenge.challenge_string(), 2).expect("solver should find a nonce");
assert!(challenge.verify(&nonce), "verifier must accept solver output");
}
#[test]
fn verify_rejects_wrong_nonce() {
let mut challenge = Challenge::generate(4);
let challenge_str = challenge.challenge_string();
// Search for a nonce that an independently computed SHA-256 provably rejects,
// so the test stays deterministic.
let mut wrong_nonce = None;
for i in 0..10_000u32 {
let candidate = format!("bad-{i}");
let hex_hash = hex::encode(Sha256::digest(format!("{challenge_str}{candidate}").as_bytes()));
if !hex_hash
.as_bytes()
.iter()
.take(4)
.all(|c| ALLOWED_HEX_PREFIX.contains(c))
{
wrong_nonce = Some(candidate);
break;
}
/// Reads a newline-terminated nonce without consuming the bytes that follow the
/// delimiter.
///
/// `peek` inspects the buffer first; only the nonce and its `\n` are consumed, so
/// a client that coalesces the nonce and the handshake into one TCP segment is
/// not truncated. A nonce split across segments is accumulated until the
/// delimiter arrives or the connection reports EOF.
///
/// Returns `None` for an empty, oversized or non-UTF-8 reply.
async fn read_nonce_line(stream: &mut TcpStream) -> anyhow::Result<Option<String>> {
let mut buf = [0u8; MAX_NONCE_LEN + 1];
let mut nonce = Vec::with_capacity(MAX_NONCE_LEN);
loop {
let n = stream.peek(&mut buf).await?;
if n == 0 {
// EOF without a delimiter: accept the bytes received so far.
break;
}
let wrong_nonce = wrong_nonce.expect("a failing nonce must exist among candidates");
assert!(!challenge.verify(&wrong_nonce));
if let Some(pos) = buf[..n].iter().position(|&b| b == b'\n') {
if nonce.len() + pos > MAX_NONCE_LEN {
return Ok(None);
}
nonce.extend_from_slice(&buf[..pos]);
// Consume exactly the nonce line; the handshake bytes stay buffered.
stream.read_exact(&mut buf[..=pos]).await?;
break;
}
if nonce.len() + n > MAX_NONCE_LEN {
return Ok(None);
}
stream.read_exact(&mut buf[..n]).await?;
nonce.extend_from_slice(&buf[..n]);
}
#[test]
fn verify_rejects_replay() {
let mut challenge = Challenge::generate(2);
let nonce = solve(&challenge.challenge_string(), 2).expect("solved");
assert!(challenge.verify(&nonce));
assert!(!challenge.verify(&nonce), "challenge must be single-use");
}
#[test]
fn verify_rejects_oversized_nonce() {
let mut challenge = Challenge::generate(2);
let big_nonce = "0".repeat(MAX_NONCE_LEN + 1);
assert!(!challenge.verify(&big_nonce));
let text = std::str::from_utf8(&nonce).unwrap_or("").trim();
if text.is_empty() || text.len() > MAX_NONCE_LEN {
return Ok(None);
}
Ok(Some(text.to_string()))
}

View file

@ -4,10 +4,107 @@ use crate::config::Config;
use crate::engine::tunnel::Gateway;
use crate::traffic::hook::TrafficHook;
use socket2::{Domain, Socket, Type};
use std::io::ErrorKind;
use std::sync::Arc;
use std::time::Duration;
use tokio::net::TcpListener;
use tokio::sync::watch;
const ACCEPT_BACKOFF_MIN: Duration = Duration::from_millis(50);
const ACCEPT_BACKOFF_MAX: Duration = Duration::from_millis(500);
/// Consecutive accept failures between full-rate ERROR logs.
///
/// Every intervening failure is still reported at `debug`, so the log stays
/// diagnosable without amplifying a spin into a disk-filling storm.
const ACCEPT_ERROR_LOG_EVERY: u32 = 32;
/// Linux errno values that mean "the listener is temporarily unable to accept".
///
/// `libc` is an optional dependency of the `xdp` feature, so the values are
/// spelled out here; they are ABI constants on the platforms Rampart targets.
#[cfg(target_os = "linux")]
mod errno {
pub const ENOMEM: i32 = 12;
pub const ENFILE: i32 = 23;
pub const EMFILE: i32 = 24;
pub const ENOBUFS: i32 = 105;
}
/// Exponential backoff for transient `accept()` failures.
///
/// File-descriptor exhaustion (`EMFILE`/`ENFILE`) makes `accept()` fail
/// synchronously and forever: a plain `continue` turns every worker into a
/// 100% CPU spin that also floods the log. Each transient failure therefore
/// yields a mandatory sleep, growing from `ACCEPT_BACKOFF_MIN` to
/// `ACCEPT_BACKOFF_MAX` and resetting after the next successful accept.
pub struct AcceptBackoff {
delay: Duration,
failures: u32,
}
impl Default for AcceptBackoff {
fn default() -> Self {
Self::new()
}
}
impl AcceptBackoff {
#[must_use]
pub fn new() -> Self {
Self {
delay: Duration::ZERO,
failures: 0,
}
}
/// Returns how long the accept loop must sleep before retrying `error`.
///
/// `None` means the error is not a capacity or readiness condition
/// (`ECONNABORTED`, `EPROTO`, ...): the connection was abandoned by the peer
/// or the kernel, so an immediate retry is correct.
pub fn delay(&mut self, error: &std::io::Error) -> Option<Duration> {
if !is_transient(error) {
return None;
}
self.failures = self.failures.saturating_add(1);
self.delay = if self.delay.is_zero() {
ACCEPT_BACKOFF_MIN
} else {
(self.delay * 2).min(ACCEPT_BACKOFF_MAX)
};
Some(self.delay)
}
/// Clears the accumulated backoff after a successful accept.
pub fn reset(&mut self) {
self.delay = Duration::ZERO;
self.failures = 0;
}
/// Reports whether the current failure deserves a full-rate ERROR log.
#[must_use]
pub fn should_log(&self) -> bool {
self.failures == 1 || self.failures % ACCEPT_ERROR_LOG_EVERY == 0
}
}
fn is_transient(error: &std::io::Error) -> bool {
if matches!(
error.kind(),
ErrorKind::WouldBlock | ErrorKind::Interrupted | ErrorKind::OutOfMemory
) {
return true;
}
#[cfg(target_os = "linux")]
if matches!(
error.raw_os_error(),
Some(errno::EMFILE | errno::ENFILE | errno::ENOBUFS | errno::ENOMEM)
) {
return true;
}
false
}
/// Starts `workers.count` accept loops on a single address.
///
/// Each worker binds its own socket with `SO_REUSEPORT` and `SO_REUSEADDR`, so
@ -58,22 +155,45 @@ async fn accept_loop(
hook: Arc<TrafficHook>,
mut shutdown: watch::Receiver<bool>,
) {
let mut backoff = AcceptBackoff::new();
loop {
tokio::select! {
biased;
_ = shutdown.changed() => {
if *shutdown.borrow() {
tracing::info!("shutdown signal received, stopping accept loop");
return;
changed = shutdown.changed() => {
match changed {
Ok(()) if *shutdown.borrow() => {
tracing::info!("shutdown signal received, stopping accept loop");
return;
},
// An update that does not raise the flag: keep serving.
Ok(()) => {},
// The sender is gone. `changed()` would return `Err` immediately
// and forever, so treat it as shutdown instead of spinning.
Err(_) => {
tracing::info!("shutdown channel closed, stopping accept loop");
return;
},
}
}
result = listener.accept() => {
let (stream, peer_addr) = match result {
Ok(conn) => conn,
Ok(conn) => {
backoff.reset();
conn
},
Err(e) => {
tracing::error!("accept error: {e}");
if let Some(delay) = backoff.delay(&e) {
if backoff.should_log() {
tracing::error!("accept error, backing off {delay:?}: {e}");
} else {
tracing::debug!("accept error, backing off {delay:?}: {e}");
}
tokio::time::sleep(delay).await;
} else {
tracing::debug!("accept error: {e}");
}
continue;
}
},
};
let gateway = Arc::clone(&gateway);
let hook = Arc::clone(&hook);

View file

@ -6,6 +6,7 @@
use crate::config::DetectPrefixConfig;
use crate::engine::subnet_tracker::{SubnetTracker, TrackerSource};
use crate::filter::blacklist::Blacklist;
use crate::metrics;
#[cfg(feature = "xdp")]
use crate::traffic::prefix::XdpPrefixStats;
@ -30,6 +31,22 @@ impl Source {
Source::Tracker(tracker) => tracker.snapshot().await,
}
}
/// Whether the source reports distinct source addresses per prefix.
fn counts_unique_sources(&self) -> bool {
match self {
#[cfg(feature = "xdp")]
Source::Xdp(_) => false,
Source::Tracker(_) => true,
}
}
}
/// Verdict enforcement settings shared by every snapshot tick.
struct BanPolicy {
blacklist: Arc<Blacklist>,
ttl: Duration,
trusted: bool,
}
/// Spawns the background subnet monitor when `[detect.prefix] enabled`.
@ -37,8 +54,16 @@ impl Source {
/// Source priority: the XDP map when a filter is loaded, otherwise the userspace
/// aggregator. Without either source the monitor stays disabled and logs a
/// warning.
///
/// `ban_ttl` comes from `[ban].ban_duration_secs`. `trusted_peer` is false when
/// the observed address comes from an unverified ingress path (inbound PROXY
/// protocol), where banning it could punish the transport proxy instead of the
/// real client.
pub fn spawn(
detect: &DetectPrefixConfig,
ban_ttl: Duration,
blacklist: Arc<Blacklist>,
trusted_peer: bool,
tracker: Option<Arc<SubnetTracker>>,
xdp: Option<Arc<Mutex<XdpFilter>>>,
mut shutdown: watch::Receiver<bool>,
@ -52,6 +77,12 @@ pub fn spawn(
};
let detector = SubnetDetector::from_config(detect);
let window = Duration::from_secs(detect.window_secs.max(1));
let policy = BanPolicy {
blacklist,
ttl: ban_ttl,
trusted: trusted_peer,
};
let extreme_threshold = detect.syn_threshold.saturating_mul(4);
tokio::spawn(async move {
let mut tick = tokio::time::interval(window);
@ -66,7 +97,10 @@ pub fn spawn(
}
_ = tick.tick() => {
match source.snapshot().await {
Ok(snaps) => apply_verdicts(&detector.evaluate(&snaps), &source).await,
Ok(snaps) => {
let verdicts = detector.evaluate(&snaps);
apply_verdicts(&verdicts, &source, &policy, extreme_threshold);
},
Err(e) => tracing::debug!("prefix stats unavailable: {e}"),
}
}
@ -85,7 +119,7 @@ fn build_source(tracker: Option<Arc<SubnetTracker>>, xdp: Option<Arc<Mutex<XdpFi
tracker.map(|t| Source::Tracker(TrackerSource { tracker: t }))
}
async fn apply_verdicts(verdicts: &[PrefixVerdict], source: &Source) {
fn apply_verdicts(verdicts: &[PrefixVerdict], source: &Source, policy: &BanPolicy, extreme_threshold: u64) {
for pv in verdicts {
metrics::SUBNET_VERDICTS
.with_label_values(&[verdict_name(pv.verdict)])
@ -97,35 +131,87 @@ async fn apply_verdicts(verdicts: &[PrefixVerdict], source: &Source) {
verdict = verdict_name(pv.verdict),
"subnet verdict"
);
if pv.verdict == SubnetVerdict::Block {
enforce_block(source, pv).await;
if pv.verdict == SubnetVerdict::Challenge
&& !source.counts_unique_sources()
&& pv.snapshot.syn_count >= extreme_threshold
{
// XDP does not count distinct sources, so the spoof gate can never
// confirm `Block` on that path; make the cap visible instead of silent.
tracing::info!(
prefix = %pv.prefix,
syn = pv.snapshot.syn_count,
"spoof gate suppressed block: XDP has no source diversity data; escalating at L7 only"
);
}
}
match source {
#[cfg(feature = "xdp")]
Source::Xdp(xdp) if policy.trusted => ban_xdp_verdicts(xdp, verdicts, policy.ttl),
#[cfg(feature = "xdp")]
Source::Xdp(_) => {
if verdicts.iter().any(|pv| pv.verdict == SubnetVerdict::Block) {
tracing::warn!("subnet kernel bans suppressed: peer identity comes from an untrusted ingress path");
}
},
Source::Tracker(_) => {
enforce_tracker_verdicts(verdicts, &policy.blacklist, policy.ttl, policy.trusted);
},
}
}
/// Applies userspace `Block` verdicts to the shared blacklist.
///
/// Returns the number of banned prefixes. When `trusted` is false the observed
/// addresses belong to an unverified ingress path, so the verdict is only logged
/// at warn level and no ban is installed.
///
/// The ban is inserted as a CIDR range, so every address inside the detected
/// `/24` or `/64` is blocked, not just its network address.
pub fn enforce_tracker_verdicts(
verdicts: &[PrefixVerdict],
blacklist: &Blacklist,
ttl: Duration,
trusted: bool,
) -> usize {
let mut banned = 0;
for pv in verdicts.iter().filter(|pv| pv.verdict == SubnetVerdict::Block) {
if !trusted {
tracing::warn!(
prefix = %pv.prefix,
"subnet block verdict on an untrusted peer path; not adding to the blacklist"
);
continue;
}
blacklist.add_cidr(pv.prefix, prefix_len(pv.prefix), ttl, "subnet_attack");
banned += 1;
tracing::warn!(
prefix = %pv.prefix,
ttl_secs = ttl.as_secs(),
"subnet blocked in the userspace blacklist"
);
}
banned
}
#[cfg(feature = "xdp")]
fn ban_xdp_verdicts(xdp: &XdpPrefixStats, verdicts: &[PrefixVerdict], ttl: Duration) {
for pv in verdicts.iter().filter(|pv| pv.verdict == SubnetVerdict::Block) {
let result = {
let guard = xdp.filter_lock();
guard.ban_cidr(pv.prefix, prefix_len(pv.prefix), ttl.as_secs())
};
match result {
Ok(()) => tracing::warn!(
prefix = %pv.prefix,
ttl_secs = ttl.as_secs(),
"subnet blocked in XDP"
),
Err(e) => tracing::warn!("subnet block failed for {}: {e}", pv.prefix),
}
}
}
async fn enforce_block(source: &Source, pv: &PrefixVerdict) {
match source {
#[cfg(feature = "xdp")]
Source::Xdp(xdp) => {
let result = {
let guard = xdp.filter_lock();
guard.ban_cidr(pv.prefix, prefix_len(pv.prefix), 3600)
};
if let Err(e) = result {
tracing::warn!("subnet block failed for {}: {e}", pv.prefix);
}
},
Source::Tracker(_) => {
tracing::warn!(
prefix = %pv.prefix,
"block verdict without kernel enforcement; escalate at L7"
);
},
}
}
/// Prefix length matching the detection granularity (v4 /24, v6 /64).
#[cfg(feature = "xdp")]
fn prefix_len(prefix: IpAddr) -> u8 {
match prefix {
IpAddr::V4(_) => 24,

View file

@ -4,25 +4,49 @@
//! Fallback path of the subnet detector when XDP is disabled: it feeds
//! [`crate::traffic::prefix::SubnetDetector`] with the same snapshot shape the
//! kernel map produces.
//!
//! Memory is bounded on purpose. Both the prefix table and the per-prefix
//! source set have hard caps, so a source rotating through a /48 (or a /64 with
//! millions of addresses) cannot grow the tracker without limit during one
//! detection window; over-cap events are counted and reported.
use crate::traffic::prefix::{PrefixSnapshot, PrefixStatsSource, prefix_of};
use dashmap::DashMap;
use std::collections::{HashMap, HashSet};
use std::net::IpAddr;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Instant;
/// Upper bound on tracked prefixes per window.
pub const DEFAULT_MAX_PREFIXES: usize = 4096;
/// Upper bound on distinct sources remembered per prefix.
///
/// 256 is well above the default `min_unique_sources = 16` spoof gate while
/// keeping the worst-case table at a few megabytes.
pub const DEFAULT_MAX_SOURCES_PER_PREFIX: usize = 256;
struct WindowState {
syn_count: u64,
sources: std::collections::HashSet<IpAddr>,
sources: HashSet<IpAddr>,
started: Instant,
}
#[derive(Debug, Clone, Copy)]
struct Limits {
max_prefixes: usize,
max_sources: usize,
}
/// Aggregates new connections per prefix.
///
/// `take_snapshot` drains the accumulated state, so every call closes exactly
/// one detection window.
pub struct SubnetTracker {
windows: DashMap<IpAddr, WindowState>,
windows: Mutex<HashMap<IpAddr, WindowState>>,
limits: Limits,
dropped_prefixes: AtomicU64,
dropped_sources: AtomicU64,
}
impl Default for SubnetTracker {
@ -34,47 +58,108 @@ impl Default for SubnetTracker {
impl SubnetTracker {
#[must_use]
pub fn new() -> Self {
Self::with_limits(DEFAULT_MAX_PREFIXES, DEFAULT_MAX_SOURCES_PER_PREFIX)
}
/// Creates a tracker with explicit caps.
///
/// Both limits are clamped to at least one entry. The constructor mainly
/// lets tests exercise the caps without building thousands of prefixes.
#[must_use]
pub fn with_limits(max_prefixes: usize, max_sources_per_prefix: usize) -> Self {
Self {
windows: DashMap::new(),
windows: Mutex::new(HashMap::new()),
limits: Limits {
max_prefixes: max_prefixes.max(1),
max_sources: max_sources_per_prefix.max(1),
},
dropped_prefixes: AtomicU64::new(0),
dropped_sources: AtomicU64::new(0),
}
}
/// Records a new inbound connection (userspace equivalent of a SYN).
///
/// Call it only after the source passed the blacklist check, so banned
/// sources cannot allocate tracker memory.
pub fn record_connection(&self, ip: IpAddr) {
let prefix = prefix_of(ip);
let mut state = self.windows.entry(prefix).or_insert_with(|| WindowState {
syn_count: 0,
sources: std::collections::HashSet::new(),
started: Instant::now(),
});
state.syn_count = state.syn_count.saturating_add(1);
state.sources.insert(ip);
let mut windows = self.windows.lock().expect("subnet tracker lock poisoned");
if let Some(state) = windows.get_mut(&prefix) {
state.syn_count = state.syn_count.saturating_add(1);
Self::remember_source(state, ip, self.limits.max_sources, &self.dropped_sources);
return;
}
if windows.len() >= self.limits.max_prefixes {
self.dropped_prefixes.fetch_add(1, Ordering::Relaxed);
return;
}
let mut sources = HashSet::new();
sources.insert(ip);
windows.insert(
prefix,
WindowState {
syn_count: 1,
sources,
started: Instant::now(),
},
);
}
/// Takes and resets the aggregates of the current window.
#[must_use]
pub fn take_snapshot(&self) -> Vec<(IpAddr, PrefixSnapshot)> {
let mut out = Vec::new();
// Drain every key: the next window starts from an empty map, so memory does not
// grow when an attacker spoofs many distinct prefixes.
self.windows.retain(|prefix, state| {
if state.syn_count == 0 {
return false;
}
out.push((
*prefix,
PrefixSnapshot {
// Userspace only sees established connections, so `pkt_count` mirrors the same
// flow as `syn_count`.
pkt_count: state.syn_count,
syn_count: state.syn_count,
last_seen_ns: state.started.elapsed().as_nanos() as u64,
unique_sources: state.sources.len() as u64,
},
));
false
});
out
let windows = std::mem::take(&mut *self.windows.lock().expect("subnet tracker lock poisoned"));
let dropped_prefixes = self.dropped_prefixes.load(Ordering::Relaxed);
let dropped_sources = self.dropped_sources.load(Ordering::Relaxed);
if dropped_prefixes > 0 || dropped_sources > 0 {
tracing::debug!(
dropped_prefixes,
dropped_sources,
"subnet tracker caps reached; aggregates are bounded"
);
}
windows
.into_iter()
.filter(|(_, state)| state.syn_count > 0)
.map(|(prefix, state)| {
(
prefix,
PrefixSnapshot {
// Userspace only sees established connections, so `pkt_count` mirrors the
// same flow as `syn_count`.
pkt_count: state.syn_count,
syn_count: state.syn_count,
last_seen_ns: state.started.elapsed().as_nanos() as u64,
unique_sources: state.sources.len() as u64,
},
)
})
.collect()
}
/// Number of connections from new prefixes dropped because the prefix cap
/// was reached.
#[must_use]
pub fn dropped_prefixes(&self) -> u64 {
self.dropped_prefixes.load(Ordering::Relaxed)
}
/// Number of connections from new source addresses dropped because the
/// per-prefix source cap was reached.
#[must_use]
pub fn dropped_sources(&self) -> u64 {
self.dropped_sources.load(Ordering::Relaxed)
}
fn remember_source(state: &mut WindowState, ip: IpAddr, cap: usize, dropped: &AtomicU64) {
if state.sources.len() < cap {
state.sources.insert(ip);
return;
}
if !state.sources.contains(&ip) {
dropped.fetch_add(1, Ordering::Relaxed);
}
}
}
@ -88,48 +173,3 @@ impl PrefixStatsSource for TrackerSource {
Ok(self.tracker.take_snapshot())
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::net::{Ipv4Addr, Ipv6Addr};
fn v4(octets: [u8; 4]) -> IpAddr {
IpAddr::V4(Ipv4Addr::from(octets))
}
#[test]
fn groups_connections_by_prefix() {
let tracker = SubnetTracker::new();
tracker.record_connection(v4([203, 0, 113, 1]));
tracker.record_connection(v4([203, 0, 113, 2]));
tracker.record_connection(v4([198, 51, 100, 9]));
let snaps = tracker.take_snapshot();
assert_eq!(snaps.len(), 2);
let by_prefix: std::collections::HashMap<IpAddr, PrefixSnapshot> = snaps.into_iter().collect();
assert_eq!(by_prefix[&v4([203, 0, 113, 0])].syn_count, 2);
assert_eq!(by_prefix[&v4([203, 0, 113, 0])].unique_sources, 2);
assert_eq!(by_prefix[&v4([198, 51, 100, 0])].syn_count, 1);
}
#[test]
fn snapshot_drains_window() {
let tracker = SubnetTracker::new();
tracker.record_connection(v4([10, 0, 0, 1]));
assert_eq!(tracker.take_snapshot().len(), 1);
assert!(tracker.take_snapshot().is_empty());
}
#[test]
fn mapped_addresses_share_v4_prefix() {
let tracker = SubnetTracker::new();
tracker.record_connection(v4([203, 0, 113, 5]));
// ::ffff:203.0.113.6
let mapped = Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xcb00, 0x7106);
tracker.record_connection(IpAddr::V6(mapped));
let snaps = tracker.take_snapshot();
assert_eq!(snaps.len(), 1);
assert_eq!(snaps[0].0, v4([203, 0, 113, 0]));
assert_eq!(snaps[0].1.unique_sources, 2);
}
}

View file

@ -1,7 +1,16 @@
//! Per-connection pipeline of the edge engine: filters, PoW gate, protocol
//! handshake and upstream relay.
//!
//! The module root owns gateway construction and the entry point; the stages
//! live in `pipeline` (filters, PoW, handshake, upstream), `admission`
//! (reputation escalation, bans and events) and `relay` (idle-bounded relay).
mod admission;
mod challenge;
mod pipeline;
mod relay;
pub use relay::RELAY_IDLE_TIMEOUT;
#[cfg(feature = "protocol-minecraft")]
use super::proxy;
@ -12,17 +21,14 @@ use crate::engine::subnet::tracker::SubnetTracker;
use crate::filter::blacklist::Blacklist;
use crate::filter::connection_limit::ConnectionLimiter;
use crate::filter::rate_limit::RateLimiter;
use crate::metrics;
use crate::protocol::ProtocolRegistry;
use crate::store::clickhouse::{ClickHouseEvent, ClickHouseWriter};
use crate::store::clickhouse::ClickHouseWriter;
use crate::traffic::reputation::IpReputation;
use crate::xdp::XdpFilter;
use std::collections::HashSet;
use std::net::IpAddr;
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
use std::sync::atomic::AtomicU64;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use tokio::io::AsyncWriteExt;
use tokio::net::TcpStream;
use tokio::sync::Mutex as TokioMutex;
@ -55,7 +61,7 @@ impl Gateway {
allowed_1s,
registry,
subnet_tracker,
upstream_cursor: AtomicUsize::new(0),
upstream_cursor: std::sync::atomic::AtomicUsize::new(0),
}
}
@ -69,25 +75,7 @@ impl Gateway {
let client_addr = if self.config.proxy_protocol.ingress_enabled {
#[cfg(feature = "protocol-minecraft")]
{
if !self
.config
.proxy_protocol
.trusted_cidrs
.iter()
.any(|cidr| cidr.contains(peer_addr.ip()))
{
return GatewayOutcome {
client_ip: None,
result: Err(anyhow::anyhow!("untrusted peer attempted inbound PROXY protocol")),
};
}
match proxy::read_v2_header(
&mut client,
Duration::from_millis(self.config.proxy_protocol.ingress_timeout_ms),
self.config.proxy_protocol.max_header_bytes,
)
.await
{
match proxy::resolve_client_addr(&self.config.proxy_protocol, &mut client, peer_addr).await {
Ok(client_addr) => client_addr,
Err(error) => {
return GatewayOutcome {
@ -114,122 +102,4 @@ impl Gateway {
result,
}
}
async fn handle_client(
&self,
mut client: TcpStream,
peer_addr: std::net::SocketAddr,
client_addr: std::net::SocketAddr,
) -> anyhow::Result<()> {
let peer_ip = client_addr.ip();
if let Some(tracker) = &self.subnet_tracker {
tracker.record_connection(peer_ip);
}
if self.blacklist.is_blocked(peer_ip) {
metrics::CONNECTIONS_TOTAL.with_label_values(&["blocked"]).inc();
return Ok(());
}
let Some(_connection_permit) = self.connection_limiter.try_acquire(peer_ip) else {
self.block_rate_limit(peer_ip).await;
return Ok(());
};
if !self.pass_pow_gate(&mut client, peer_ip).await? {
return Ok(());
}
if !self.rate_limiter.check(peer_ip) {
self.block_rate_limit(peer_ip).await;
return Ok(());
}
let handler = self.registry.primary()?;
let handshake = match handler.handshake(&mut client).await {
Ok(handshake) => handshake,
Err(e) => {
tracing::debug!("handshake rejected from {peer_addr}: {e}");
metrics::CONNECTIONS_TOTAL.with_label_values(&["blocked"]).inc();
return Ok(());
},
};
metrics::CONNECTIONS_TOTAL.with_label_values(&["allowed"]).inc();
self.allowed_1s.fetch_add(1, Ordering::Relaxed);
self.reputation.record_good(peer_ip);
let mut backend = TcpStream::connect(self.next_upstream(&handshake.upstream.addr)).await?;
#[cfg(feature = "protocol-minecraft")]
if self.config.protocol.minecraft.enabled && self.config.protocol.minecraft.send_proxy_protocol {
let source = client_addr;
let destination = client.local_addr()?;
backend.write_all(&proxy::protocol_v2(source, destination)?).await?;
}
if !handshake.initial_backend_data.is_empty() {
backend.write_all(&handshake.initial_backend_data).await?;
}
tokio::io::copy_bidirectional(&mut client, &mut backend).await?;
Ok(())
}
fn next_upstream<'a>(&'a self, negotiated: &'a str) -> &'a str {
let upstreams = &self.config.backend.upstreams;
if upstreams.len() > 1 {
let idx = self.upstream_cursor.fetch_add(1, Ordering::Relaxed) % upstreams.len();
return upstreams[idx].as_str();
}
if let Some(first) = upstreams.first() {
return first.as_str();
}
negotiated
}
async fn block_rate_limit(&self, ip: IpAddr) {
metrics::RATE_LIMIT_HITS.with_label_values(&["hit"]).inc();
metrics::CONNECTIONS_TOTAL.with_label_values(&["blocked"]).inc();
self.reputation.record_bad(ip);
if self.reputation.score(ip) < -40 {
let duration_secs = Duration::from_secs(self.config.ban.ban_duration_secs);
self.blacklist.add(ip, duration_secs, "low_reputation");
if !self.config.proxy_protocol.ingress_enabled {
self.xdp_ban(ip, duration_secs.as_secs());
}
self.push_event("block", ip, "low_reputation").await;
tracing::info!("low reputation ban {ip}: rate-limit abuse");
}
}
fn xdp_ban(&self, ip: IpAddr, duration_secs: u64) {
let Some(xdp) = &self.xdp else {
return;
};
let prefix_len = if ip.is_ipv4() { 32 } else { 128 };
match xdp
.lock()
.expect("xdp lock poisoned")
.ban_cidr(ip, prefix_len, duration_secs)
{
Ok(()) => tracing::debug!("xdp ban {ip} for {duration_secs}s"),
Err(e) => tracing::warn!("xdp ban failed for {ip}: {e}"),
}
}
async fn push_event(&self, event_type: &str, ip: IpAddr, data_string: &str) {
let Some(writer) = &self.clickhouse else {
return;
};
let event = ClickHouseEvent {
timestamp: chrono::Utc::now(),
event_type: event_type.to_string(),
ip: ip.to_string(),
data_float: 0.0,
data_int: 0,
data_string: data_string.to_string(),
};
if let Err(e) = writer.lock().await.push(event).await {
tracing::debug!("clickhouse push error: {e}");
}
}
}

View file

@ -0,0 +1,67 @@
//! Reputation escalation, bans and telemetry for dropped connections.
use super::Gateway;
use crate::metrics;
use crate::store::clickhouse::ClickHouseEvent;
use std::net::IpAddr;
use std::time::Duration;
impl Gateway {
/// Accounts for a connection dropped by an admission filter and escalates to
/// a ban when the source reputation is exhausted.
///
/// Whitelisted peers are exempt: a ban is replicated to Redis and XDP on
/// every node, which would lock a trusted operator out of the whole cluster.
pub(super) async fn block_rate_limit(&self, ip: IpAddr) {
if self.whitelist.contains(&ip) {
metrics::RATE_LIMIT_HITS.with_label_values(&["trusted"]).inc();
return;
}
metrics::RATE_LIMIT_HITS.with_label_values(&["hit"]).inc();
metrics::CONNECTIONS_TOTAL.with_label_values(&["blocked"]).inc();
self.reputation.record_bad(ip);
if self.reputation.score(ip) < -40 {
let duration_secs = Duration::from_secs(self.config.ban.ban_duration_secs);
self.blacklist.add(ip, duration_secs, "low_reputation");
if !self.config.proxy_protocol.ingress_enabled {
self.xdp_ban(ip, duration_secs.as_secs());
}
self.push_event("block", ip, "low_reputation").await;
tracing::info!("low reputation ban {ip}: rate-limit abuse");
}
}
/// Mirrors a ban into the XDP map, if the kernel filter is attached.
fn xdp_ban(&self, ip: IpAddr, duration_secs: u64) {
let Some(xdp) = &self.xdp else {
return;
};
let prefix_len = if ip.is_ipv4() { 32 } else { 128 };
match xdp
.lock()
.expect("xdp lock poisoned")
.ban_cidr(ip, prefix_len, duration_secs)
{
Ok(()) => tracing::debug!("xdp ban {ip} for {duration_secs}s"),
Err(e) => tracing::warn!("xdp ban failed for {ip}: {e}"),
}
}
/// Forwards an event to ClickHouse when the writer is configured.
async fn push_event(&self, event_type: &str, ip: IpAddr, data_string: &str) {
let Some(writer) = &self.clickhouse else {
return;
};
let event = ClickHouseEvent {
timestamp: chrono::Utc::now(),
event_type: event_type.to_string(),
ip: ip.to_string(),
data_float: 0.0,
data_int: 0,
data_string: data_string.to_string(),
};
if let Err(e) = writer.lock().await.push(event).await {
tracing::debug!("clickhouse push error: {e}");
}
}
}

View file

@ -31,6 +31,10 @@ impl Gateway {
adjuster.record_connection();
adjuster.current_difficulty()
};
// The configured difficulty is a hard floor even if the adjuster was
// constructed without it; recording the connection above still adapts
// the value for later peers.
let diff = diff.max(pow_config.difficulty);
let result = enforce_pow(client, peer_ip, diff).await?;
if !result {
metrics::POW_CHALLENGES_TOTAL.with_label_values(&["failed"]).inc();

View file

@ -0,0 +1,208 @@
//! Filter order and upstream establishment for one accepted connection.
use super::Gateway;
use super::RELAY_IDLE_TIMEOUT;
use crate::metrics;
use std::sync::OnceLock;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Duration;
use tokio::io::AsyncWriteExt;
use tokio::net::TcpStream;
use tokio::sync::Semaphore;
/// Multiplier applied to the per-IP limit when sizing the global connection
/// budget: `4 x workers x max_connections_per_ip` in-flight connections.
const GLOBAL_BUDGET_FACTOR: usize = 4;
/// Ceiling of the global budget, so a large worker count cannot exhaust the
/// file-descriptor table of the host.
const GLOBAL_BUDGET_MAX: usize = 100_000;
/// One in every `BUDGET_LOG_EVERY` budget rejections is logged.
const BUDGET_LOG_EVERY: u64 = 128;
/// Process-wide in-flight connection budget.
///
/// Initialised on first use from the worker count and the per-IP limit of the
/// gateway configuration. A process-wide value is intentional: the budget caps
/// descriptors and tasks for the whole edge node, and there is exactly one
/// gateway per process in production.
static GLOBAL_BUDGET: OnceLock<Semaphore> = OnceLock::new();
/// Total connections rejected by the global budget, used for log rate limiting.
static BUDGET_REJECTIONS: AtomicU64 = AtomicU64::new(0);
impl Gateway {
/// Returns the process-wide in-flight connection budget.
fn global_budget(&self) -> &'static Semaphore {
GLOBAL_BUDGET.get_or_init(|| {
let workers = self.config.workers.count.max(1);
let per_ip = self.config.limits.max_connections_per_ip.max(1) as usize;
let capacity = workers
.saturating_mul(per_ip)
.saturating_mul(GLOBAL_BUDGET_FACTOR)
.clamp(1, GLOBAL_BUDGET_MAX);
Semaphore::new(capacity)
})
}
/// Runs the filter stages, PoW gate, protocol handshake and relay.
///
/// # Errors
/// Only upstream I/O failures are surfaced; every filter drop is `Ok(())`.
pub(super) async fn handle_client(
&self,
mut client: TcpStream,
peer_addr: std::net::SocketAddr,
client_addr: std::net::SocketAddr,
) -> anyhow::Result<()> {
let peer_ip = client_addr.ip();
let trusted = self.whitelist.contains(&peer_ip);
// Trusted peers bypass every ingress filter and are never auto-banned:
// blacklist entries and XDP/Redis bans propagate across all nodes, so a
// whitelisted address must never enter that path.
if !trusted {
if self.blacklist.is_blocked(peer_ip) {
metrics::CONNECTIONS_TOTAL.with_label_values(&["blocked"]).inc();
return Ok(());
}
// Feed the subnet detector only after the ban decision: a source we
// already block must not allocate tracker memory.
if let Some(tracker) = &self.subnet_tracker {
tracker.record_connection(peer_ip);
}
// The cheap token check runs before the expensive PoW exchange: a
// client that never solves the challenge must still exhaust its own
// rate budget instead of holding a task for the PoW timeout unpaid.
if !self.rate_limiter.check(peer_ip) {
self.block_rate_limit(peer_ip).await;
return Ok(());
}
}
// Global budget: `try_acquire` keeps an overloaded node responsive (the
// accept loop never queues unbounded per-connection tasks) and the
// connection is closed immediately on refusal.
let _global_permit = if trusted {
None
} else {
match self.global_budget().try_acquire() {
Ok(permit) => Some(permit),
Err(_) => {
metrics::CONNECTIONS_TOTAL.with_label_values(&["overloaded"]).inc();
let seen = BUDGET_REJECTIONS.fetch_add(1, Ordering::Relaxed);
if seen % BUDGET_LOG_EVERY == 0 {
tracing::warn!(
"global connection budget exhausted after {seen} rejections, dropping {peer_ip}"
);
}
return Ok(());
},
}
};
let Some(_connection_permit) = self.connection_limiter.try_acquire(peer_ip) else {
if trusted {
tracing::debug!("per-IP connection limit reached for trusted {peer_ip}");
} else {
self.block_rate_limit(peer_ip).await;
}
return Ok(());
};
if !self.pass_pow_gate(&mut client, peer_ip).await? {
return Ok(());
}
let handshake_budget = Duration::from_secs(self.config.limits.handshake_timeout_secs.max(1));
let handler = self.registry.primary()?;
let handshake = match tokio::time::timeout(handshake_budget, handler.handshake(&mut client)).await {
Ok(Ok(handshake)) => handshake,
Ok(Err(e)) => {
tracing::debug!("handshake rejected from {peer_addr}: {e}");
metrics::CONNECTIONS_TOTAL.with_label_values(&["blocked"]).inc();
return Ok(());
},
Err(_) => {
tracing::debug!("handshake timed out after {handshake_budget:?} from {peer_addr}");
metrics::CONNECTIONS_TOTAL.with_label_values(&["blocked"]).inc();
return Ok(());
},
};
metrics::CONNECTIONS_TOTAL.with_label_values(&["allowed"]).inc();
self.allowed_1s.fetch_add(1, Ordering::Relaxed);
self.reputation.record_good(peer_ip);
let mut backend = self
.connect_upstream(&handshake.upstream.addr, handshake_budget)
.await?;
self.send_preamble(
&mut client,
&mut backend,
client_addr,
&handshake.initial_backend_data,
handshake_budget,
)
.await?;
self.relay_with_idle(client, backend, RELAY_IDLE_TIMEOUT).await
}
/// Opens the upstream connection under the handshake timeout.
///
/// # Errors
/// Fails when the connect is refused or exceeds `budget`.
async fn connect_upstream(&self, negotiated: &str, budget: Duration) -> anyhow::Result<TcpStream> {
let upstream = self.next_upstream(negotiated);
match tokio::time::timeout(budget, TcpStream::connect(upstream)).await {
Ok(Ok(stream)) => Ok(stream),
Ok(Err(e)) => Err(anyhow::anyhow!("upstream connect to {upstream} failed: {e}")),
Err(_) => Err(anyhow::anyhow!(
"upstream connect to {upstream} timed out after {budget:?}"
)),
}
}
/// Writes the PROXY protocol header and the protocol's replay bytes to the
/// backend under the handshake timeout.
///
/// # Errors
/// Fails on address resolution and on a write that exceeds `budget`.
#[allow(unused_variables)]
async fn send_preamble(
&self,
client: &mut TcpStream,
backend: &mut TcpStream,
client_addr: std::net::SocketAddr,
initial_data: &[u8],
budget: Duration,
) -> anyhow::Result<()> {
let mut preamble = Vec::new();
#[cfg(feature = "protocol-minecraft")]
if self.config.protocol.minecraft.enabled && self.config.protocol.minecraft.send_proxy_protocol {
let destination = client.local_addr()?;
preamble.extend_from_slice(&super::super::proxy::protocol_v2(client_addr, destination)?);
}
preamble.extend_from_slice(initial_data);
if preamble.is_empty() {
return Ok(());
}
match tokio::time::timeout(budget, backend.write_all(&preamble)).await {
Ok(Ok(())) => Ok(()),
Ok(Err(e)) => Err(e.into()),
Err(_) => Err(anyhow::anyhow!("upstream preamble write timed out after {budget:?}")),
}
}
fn next_upstream<'a>(&'a self, negotiated: &'a str) -> &'a str {
let upstreams = &self.config.backend.upstreams;
if upstreams.len() > 1 {
let idx = self.upstream_cursor.fetch_add(1, Ordering::Relaxed) % upstreams.len();
return upstreams[idx].as_str();
}
if let Some(first) = upstreams.first() {
return first.as_str();
}
negotiated
}
}

View file

@ -0,0 +1,75 @@
//! Idle-aware bidirectional relay between the accepted client and the backend.
use super::Gateway;
use std::io;
use std::time::Duration;
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
use tokio::net::TcpStream;
const RELAY_BUFFER_BYTES: usize = 16 * 1024;
/// Idle budget of the client/backend relay.
///
/// Minecraft keeps one connection alive for a whole session and a legitimately
/// silent gap (menus, chunk loading, AFK timeouts) can last minutes, so 300s is
/// the safe default: it never cuts a real session and still reaps dead peers.
/// The value is a constant because `[limits]` is owned by a parallel change; it
/// is plumbed as a parameter through [`Gateway::relay_with_idle`] so a config
/// field can be wired in without touching the relay itself.
pub const RELAY_IDLE_TIMEOUT: Duration = Duration::from_secs(300);
impl Gateway {
/// Relays bytes in both directions until either side closes, an I/O error
/// occurs or a direction stays silent for `idle`.
///
/// The idle budget restarts on every successful read/write, so a long-lived
/// session with steady traffic is never cut, while a dead or half-open peer
/// is reaped instead of pinning two descriptors forever. Both directions run
/// under one `select!`: when one pump finishes the other is dropped and both
/// sockets close, preserving the cancellation semantics of
/// `copy_bidirectional`.
///
/// `idle` is a parameter rather than a constant so tests can exercise the
/// timeout with a short budget and per-upstream tuning can override it.
///
/// # Errors
/// Fails on the first I/O error and on an idle timeout; a clean half-close is
/// `Ok`.
pub async fn relay_with_idle(&self, client: TcpStream, backend: TcpStream, idle: Duration) -> anyhow::Result<()> {
let (client_read, mut client_write) = client.into_split();
let (backend_read, mut backend_write) = backend.into_split();
tokio::select! {
result = pump(client_read, &mut backend_write, idle) => result?,
result = pump(backend_read, &mut client_write, idle) => result?,
}
Ok(())
}
}
/// Copies one direction, bounded by `idle` between chunks.
///
/// # Errors
/// Fails on a read/write error and when a direction makes no progress within
/// `idle`.
async fn pump<R, W>(mut source: R, mut sink: W, idle: Duration) -> io::Result<()>
where
R: AsyncRead + Unpin,
W: AsyncWrite + Unpin,
{
let mut buffer = vec![0u8; RELAY_BUFFER_BYTES];
loop {
let read = tokio::time::timeout(idle, source.read(&mut buffer))
.await
.map_err(|_| idle_error())??;
if read == 0 {
return Ok(());
}
tokio::time::timeout(idle, sink.write_all(&buffer[..read]))
.await
.map_err(|_| idle_error())??;
}
}
fn idle_error() -> io::Error {
io::Error::new(io::ErrorKind::TimedOut, "relay idle timeout")
}

View file

@ -1,10 +1,38 @@
//! Encoding of upstream PROXY protocol v2 headers.
//! PROXY protocol v2 support: header encoding for the backend and strict
//! parsing of headers injected by trusted ingress peers.
use crate::config::ProxyProtocolConfig;
use std::net::SocketAddr;
use std::time::Duration;
use tokio::net::TcpStream;
mod inbound;
pub(super) use inbound::read_v2_header;
/// Resolves the effective client address from an inbound PROXY v2 header.
///
/// Only peers inside `config.trusted_cidrs` may inject a header; anyone else
/// is rejected instead of being believed. The whole header must arrive within
/// `config.ingress_timeout_ms` and fit in `config.max_header_bytes`.
///
/// # Errors
/// Fails when the peer is not trusted, on timeout and on any header parse error.
pub(super) async fn resolve_client_addr(
config: &ProxyProtocolConfig,
client: &mut TcpStream,
peer_addr: SocketAddr,
) -> anyhow::Result<SocketAddr> {
if !config.trusted_cidrs.iter().any(|cidr| cidr.contains(peer_addr.ip())) {
anyhow::bail!("untrusted peer attempted inbound PROXY protocol");
}
read_v2_header(
client,
Duration::from_millis(config.ingress_timeout_ms),
config.max_header_bytes,
)
.await
}
/// Encodes a PROXY protocol v2 TCP header for the backend.
///
/// The backend must accept connections only from trusted Rampart nodes, since

View file

@ -1,4 +1,15 @@
//! In-memory ban tables.
//!
//! Two tables are kept: bans detected locally and bans mirrored from Redis.
//! Expired entries are removed lazily on lookup and periodically by
//! [`Blacklist::clear_expired`]. Deadlines are monotonic and clamped, so a
//! hostile expiry can never overflow or panic.
mod prefix;
use self::prefix::PrefixBans;
use dashmap::DashMap;
use std::collections::HashSet;
use std::net::IpAddr;
use std::sync::Arc;
use std::time::{Duration, Instant};
@ -6,9 +17,14 @@ use tokio::sync::broadcast;
const LOCAL_BAN_EVENT_CAPACITY: usize = 4096;
/// Upper bound for a single ban; longer requests are clamped.
pub const MAX_BAN_DURATION_SECS: u64 = 31_536_000;
/// A locally detected ban, forwarded to the Redis publisher.
#[derive(Clone, Debug)]
pub struct LocalBanEvent {
pub ip: IpAddr,
/// Wall-clock expiry as Unix seconds.
pub expires_at_unix: i64,
pub reason: String,
}
@ -18,9 +34,11 @@ struct BanEntry {
_reason: String,
}
/// Combined view of local and Redis-mirrored bans.
pub struct Blacklist {
entries: Arc<DashMap<IpAddr, BanEntry>>,
redis_entries: Arc<DashMap<IpAddr, BanEntry>>,
prefixes: PrefixBans,
local_ban_events: broadcast::Sender<LocalBanEvent>,
}
@ -31,29 +49,53 @@ impl Default for Blacklist {
}
impl Blacklist {
/// Creates an empty blacklist.
pub fn new() -> Self {
let (local_ban_events, _) = broadcast::channel(LOCAL_BAN_EVENT_CAPACITY);
Self {
entries: Arc::new(DashMap::new()),
redis_entries: Arc::new(DashMap::new()),
prefixes: PrefixBans::new(),
local_ban_events,
}
}
/// Returns whether `ip` is banned: exact match or inside a CIDR ban.
pub fn is_blocked(&self, ip: IpAddr) -> bool {
Self::is_blocked_in(&self.entries, ip) || Self::is_blocked_in(&self.redis_entries, ip)
let ip = normalize(ip);
Self::is_blocked_in(&self.entries, ip)
|| Self::is_blocked_in(&self.redis_entries, ip)
|| self.prefixes.is_blocked(ip, Instant::now())
}
/// Adds a CIDR ban covering `ip/prefix_len` and broadcasts its network
/// address, so other nodes mirror at least that host.
///
/// Invalid prefix lengths (`0`, `> 32` for IPv4, `> 128` for IPv6) are
/// ignored: a `/0` ban would blackhole the whole address family.
pub fn add_cidr(&self, ip: IpAddr, prefix_len: u8, duration: Duration, reason: &str) {
let duration = clamp_duration(duration);
let Some(start) = self.prefixes.add(ip, prefix_len, deadline(duration)) else {
tracing::warn!(%ip, prefix_len, "ignoring CIDR ban with an invalid prefix length");
return;
};
let expires_at_unix = now_unix().saturating_add(duration.as_secs()).min(i64::MAX as u64) as i64;
let _ = self.local_ban_events.send(LocalBanEvent {
ip: start,
expires_at_unix,
reason: reason.to_string(),
});
}
/// Adds a local ban and broadcasts it for the Redis publisher.
pub fn add(&self, ip: IpAddr, duration: Duration, reason: &str) {
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_secs();
let expires_at_unix = now.saturating_add(duration.as_secs()).min(i64::MAX as u64) as i64;
let ip = normalize(ip);
let duration = clamp_duration(duration);
let expires_at_unix = now_unix().saturating_add(duration.as_secs()).min(i64::MAX as u64) as i64;
self.entries.insert(
ip,
BanEntry {
expires: Instant::now() + duration,
expires: deadline(duration),
_reason: reason.to_string(),
},
);
@ -64,120 +106,130 @@ impl Blacklist {
});
}
/// Subscribes to locally detected bans.
pub fn subscribe_local_bans(&self) -> broadcast::Receiver<LocalBanEvent> {
self.local_ban_events.subscribe()
}
/// Returns all active local bans.
pub fn local_snapshot(&self) -> Vec<LocalBanEvent> {
let now = Instant::now();
let now_unix = now_unix();
self.entries
.iter()
.filter(|entry| entry.expires > Instant::now())
.filter(|entry| entry.expires > now)
.map(|entry| LocalBanEvent {
ip: *entry.key(),
expires_at_unix: std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
.saturating_add(entry.expires.saturating_duration_since(Instant::now()).as_secs())
expires_at_unix: now_unix
.saturating_add(entry.expires.saturating_duration_since(now).as_secs())
.min(i64::MAX as u64) as i64,
reason: entry._reason.clone(),
})
.collect()
}
/// Removes a local ban.
pub fn remove(&self, ip: IpAddr) {
self.entries.remove(&ip);
self.entries.remove(&normalize(ip));
}
/// Adds a Redis-mirrored ban.
pub fn add_redis(&self, ip: IpAddr, duration: Duration, reason: &str) {
let ip = normalize(ip);
let duration = clamp_duration(duration);
self.redis_entries.insert(
ip,
BanEntry {
expires: Instant::now() + duration,
expires: deadline(duration),
_reason: reason.to_string(),
},
);
}
/// Removes a Redis-mirrored ban.
pub fn remove_redis(&self, ip: IpAddr) {
self.redis_entries.remove(&ip);
self.redis_entries.remove(&normalize(ip));
}
/// Replaces the Redis-managed ban set, preserving locally detected bans.
///
/// `redis_entries` is rebuilt from `entries` wholesale, so a ban that
/// disappeared upstream stops applying after this call.
/// New entries are inserted before stale ones are removed, so a reconnect
/// never exposes a window without bans. A ban that disappeared upstream
/// stops applying after this call.
pub fn replace_redis<I>(&self, entries: I)
where
I: IntoIterator<Item = (IpAddr, Duration, String)>,
{
self.redis_entries.clear();
let mut fresh = HashSet::new();
for (ip, duration, reason) in entries {
fresh.insert(ip);
self.add_redis(ip, duration, &reason);
}
self.redis_entries.retain(|ip, _| fresh.contains(ip));
}
/// Total number of tracked bans.
pub fn len(&self) -> usize {
self.entries.len() + self.redis_entries.len()
self.entries.len() + self.redis_entries.len() + self.prefixes.len()
}
/// Returns whether all ban tables are empty.
pub fn is_empty(&self) -> bool {
self.entries.is_empty() && self.redis_entries.is_empty()
self.len() == 0
}
/// Drops expired entries from all tables.
pub fn clear_expired(&self) {
self.entries.retain(|_, entry| entry.expires > Instant::now());
self.redis_entries.retain(|_, entry| entry.expires > Instant::now());
let now = Instant::now();
self.entries.retain(|_, entry| entry.expires > now);
self.redis_entries.retain(|_, entry| entry.expires > now);
self.prefixes.clear_expired(now);
}
/// Checks `entries`, lazily removing `ip` only if it is still expired.
///
/// The conditional removal avoids a TOCTOU race in which a concurrent
/// re-ban would be deleted right after the expired entry was observed.
fn is_blocked_in(entries: &DashMap<IpAddr, BanEntry>, ip: IpAddr) -> bool {
if let Some(entry) = entries.get(&ip) {
if entry.expires > Instant::now() {
return true;
}
drop(entry);
entries.remove(&ip);
let now = Instant::now();
if let Some(entry) = entries.get(&ip)
&& entry.expires > now
{
return true;
}
entries.remove_if(&ip, |_, entry| entry.expires <= now);
false
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::net::{IpAddr, Ipv4Addr};
fn ip(octets: [u8; 4]) -> IpAddr {
IpAddr::V4(Ipv4Addr::from(octets))
}
#[test]
fn test_blacklist_block() {
let bl = Blacklist::new();
bl.add(ip([1, 2, 3, 4]), Duration::from_secs(60), "test");
assert!(bl.is_blocked(ip([1, 2, 3, 4])));
}
#[test]
fn test_blacklist_not_blocked() {
let bl = Blacklist::new();
bl.add(ip([1, 2, 3, 4]), Duration::from_secs(60), "test");
assert!(!bl.is_blocked(ip([5, 6, 7, 8])));
}
#[test]
fn test_blacklist_expired() {
let bl = Blacklist::new();
bl.add(ip([1, 2, 3, 4]), Duration::from_millis(1), "test");
std::thread::sleep(Duration::from_millis(2));
assert!(!bl.is_blocked(ip([1, 2, 3, 4])));
}
#[test]
fn test_blacklist_remove() {
let bl = Blacklist::new();
bl.add(ip([1, 2, 3, 4]), Duration::from_secs(60), "test");
bl.remove(ip([1, 2, 3, 4]));
assert!(!bl.is_blocked(ip([1, 2, 3, 4])));
/// Collapses IPv4-mapped IPv6 addresses to IPv4, so one client never occupies
/// two different keys in the ban tables.
fn normalize(ip: IpAddr) -> IpAddr {
match ip {
IpAddr::V6(v6) => v6.to_ipv4_mapped().map_or(ip, IpAddr::V4),
IpAddr::V4(_) => ip,
}
}
/// Clamps a requested duration to [`MAX_BAN_DURATION_SECS`].
fn clamp_duration(duration: Duration) -> Duration {
duration.min(Duration::from_secs(MAX_BAN_DURATION_SECS))
}
/// Returns the monotonic deadline for an already clamped duration.
///
/// Overflow cannot occur for clamped values, but is handled defensively so a
/// malformed duration can never panic the data plane.
fn deadline(duration: Duration) -> Instant {
Instant::now()
.checked_add(duration)
.unwrap_or_else(|| Instant::now() + Duration::from_secs(MAX_BAN_DURATION_SECS))
}
/// Current wall-clock time as Unix seconds.
fn now_unix() -> u64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}

View file

@ -0,0 +1,119 @@
//! CIDR ban table.
//!
//! Exact-address bans live in two `DashMap`s in [`super`]; substrate-wide bans
//! (`/24`, `/64`, ...) are stored here as inclusive address ranges. The table is
//! a plain `RwLock<Vec<_>>`: entries are few (the subnet monitor bans at most a
//! handful per tick), so a linear scan is cheaper than a trie, and the hard cap
//! bounds the worst case under abuse.
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
use std::sync::RwLock;
use std::time::Instant;
/// Upper bound on distinct CIDR bans kept in memory.
const MAX_PREFIX_BANS: usize = 256;
/// One CIDR ban, stored as an inclusive address range.
struct PrefixBan {
start: IpAddr,
end: IpAddr,
expires: Instant,
}
/// Thread-safe set of active CIDR bans.
pub(super) struct PrefixBans {
entries: RwLock<Vec<PrefixBan>>,
}
impl PrefixBans {
/// Creates an empty table.
pub(super) fn new() -> Self {
Self {
entries: RwLock::new(Vec::new()),
}
}
/// Inserts `ip/prefix_len` until `expires`.
///
/// Returns the network address when the ban was inserted, or `None` for an
/// invalid prefix length (`0`, `> 32` for IPv4, `> 128` for IPv6): a `/0`
/// ban would blackhole an entire address family.
pub(super) fn add(&self, ip: IpAddr, prefix_len: u8, expires: Instant) -> Option<IpAddr> {
let (start, end) = cidr_range(ip, prefix_len)?;
let now = Instant::now();
let mut bans = self.entries.write().unwrap_or_else(|poison| poison.into_inner());
bans.retain(|ban| ban.expires > now);
if bans.len() >= MAX_PREFIX_BANS {
bans.remove(0);
}
bans.push(PrefixBan { start, end, expires });
Some(start)
}
/// Returns whether `ip` falls into an active CIDR ban.
pub(super) fn is_blocked(&self, ip: IpAddr, now: Instant) -> bool {
let bans = self.entries.read().unwrap_or_else(|poison| poison.into_inner());
bans.iter().any(|ban| ban.expires > now && range_contains(ban, ip))
}
/// Drops expired CIDR bans.
pub(super) fn clear_expired(&self, now: Instant) {
self.entries
.write()
.unwrap_or_else(|poison| poison.into_inner())
.retain(|ban| ban.expires > now);
}
/// Number of tracked CIDR bans (including not-yet-reaped expired ones).
pub(super) fn len(&self) -> usize {
self.entries.read().unwrap_or_else(|poison| poison.into_inner()).len()
}
}
/// Converts `ip/prefix_len` into an inclusive address range.
///
/// IPv4-mapped IPv6 addresses are normalised to IPv4 first, matching the prefix
/// keys used by the subnet detector and the XDP prefix stats.
fn cidr_range(ip: IpAddr, prefix_len: u8) -> Option<(IpAddr, IpAddr)> {
let ip = match ip {
IpAddr::V6(v6) => v6.to_ipv4_mapped().map_or(ip, IpAddr::V4),
IpAddr::V4(_) => ip,
};
match ip {
IpAddr::V4(v4) => {
if prefix_len == 0 || prefix_len > 32 {
return None;
}
let mask = u32::MAX << (32 - prefix_len);
let base = u32::from(v4) & mask;
Some((
IpAddr::V4(Ipv4Addr::from(base)),
IpAddr::V4(Ipv4Addr::from(base | !mask)),
))
},
IpAddr::V6(v6) => {
if prefix_len == 0 || prefix_len > 128 {
return None;
}
let mask = u128::MAX << (128 - prefix_len);
let base = u128::from(v6) & mask;
Some((
IpAddr::V6(Ipv6Addr::from(base)),
IpAddr::V6(Ipv6Addr::from(base | !mask)),
))
},
}
}
/// Returns whether `ip` is inside `ban`'s inclusive range.
fn range_contains(ban: &PrefixBan, ip: IpAddr) -> bool {
match (ban.start, ban.end, ip) {
(IpAddr::V4(start), IpAddr::V4(end), IpAddr::V4(addr)) => {
u32::from(start) <= u32::from(addr) && u32::from(addr) <= u32::from(end)
},
(IpAddr::V6(start), IpAddr::V6(end), IpAddr::V6(addr)) => {
u128::from(start) <= u128::from(addr) && u128::from(addr) <= u128::from(end)
},
_ => false,
}
}

View file

@ -1,155 +1,175 @@
use dashmap::DashMap;
//! Per-source token-bucket rate limiter for the accept path.
//!
//! # Capacity policy
//!
//! The bucket table is a hard-bounded sharded hash map: at most `max_buckets`
//! live buckets (one million for [`RateLimiter::new`]). Admission uses an
//! atomic reservation, so concurrent accept loops cannot overshoot the bound
//! the way a plain `len()` check would.
//!
//! # Saturation policy
//!
//! When the table is full, [`RateLimiter::check`] fails closed for a *new*
//! source: the connection is rejected until [`RateLimiter::sweep`] frees a
//! slot. Existing sources keep their token state and are never refused because
//! of the cap. Failing closed is deliberate for a protection device - the
//! alternative (unbounded buckets, or silently allowing everything once the
//! table is full) turns a memory exhaustion vector into a bypass.
//!
//! # Eviction policy
//!
//! [`RateLimiter::sweep`] visits eight of the sixty-four shards and drops
//! buckets idle for longer than `eviction_idle` (ten minutes by default). Work
//! per call is bounded, so the accept path never stalls on a full-table
//! `retain`, and one full cycle takes eight one-minute ticks.
use std::collections::HashMap;
use std::collections::hash_map::RandomState;
use std::hash::BuildHasher;
use std::net::IpAddr;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Mutex, MutexGuard};
use std::time::{Duration, Instant};
/// Default hard bound of the bucket table (one million sources).
const MAX_BUCKETS: usize = 1_000_000;
/// Default idle age after which a bucket may be reclaimed.
const EVICTION_IDLE: Duration = Duration::from_secs(600);
const SWEEP_INTERVAL: Duration = Duration::from_secs(60);
/// Number of independently locked map shards.
const SHARDS: usize = 64;
/// Shards examined by one [`RateLimiter::sweep`] call.
const SWEEP_SHARDS_PER_TICK: usize = 8;
/// Token-bucket state of one source address.
struct Bucket {
tokens: f64,
last_refill: Instant,
last_access: Instant,
}
/// Per-source token buckets with a hard capacity bound.
pub struct RateLimiter {
buckets: Arc<DashMap<IpAddr, Bucket>>,
shards: Box<[Mutex<HashMap<IpAddr, Bucket>>]>,
hasher: RandomState,
live: AtomicUsize,
max_buckets: usize,
max_tokens: f64,
refill_rate: f64,
_refill_interval: Duration,
epoch: Instant,
last_sweep_elapsed: AtomicU64,
eviction_idle: Duration,
sweep_cursor: AtomicUsize,
}
impl RateLimiter {
/// Builds a limiter with the production capacity policy.
#[must_use]
pub fn new(rate_per_sec: f64, burst: f64) -> Self {
Self::with_eviction(rate_per_sec, burst, EVICTION_IDLE)
Self::with_capacity(rate_per_sec, burst, MAX_BUCKETS, EVICTION_IDLE)
}
fn with_eviction(rate_per_sec: f64, burst: f64, eviction_idle: Duration) -> Self {
/// Builds a limiter with an explicit capacity policy.
///
/// `max_buckets` is the hard bound of the table and `eviction_idle` the
/// idle age after which [`Self::sweep`] may reclaim a bucket. [`Self::new`]
/// supplies the production defaults; this constructor exists for tests and
/// for deployments with a tighter memory budget.
#[must_use]
pub fn with_capacity(rate_per_sec: f64, burst: f64, max_buckets: usize, eviction_idle: Duration) -> Self {
Self {
buckets: Arc::new(DashMap::new()),
shards: (0..SHARDS).map(|_| Mutex::new(HashMap::new())).collect(),
hasher: RandomState::new(),
live: AtomicUsize::new(0),
max_buckets,
max_tokens: burst,
refill_rate: rate_per_sec,
_refill_interval: Duration::from_secs(1),
epoch: Instant::now(),
last_sweep_elapsed: AtomicU64::new(0),
eviction_idle,
sweep_cursor: AtomicUsize::new(0),
}
}
/// Consumes one token for `ip`, returning `false` when the source is over
/// its rate or the table is saturated (see the module documentation).
pub fn check(&self, ip: IpAddr) -> bool {
let now = Instant::now();
let mut entry = self.buckets.entry(ip).or_insert_with(|| Bucket {
let index = self.shard_index(ip);
let mut shard = self.lock(index);
if let Some(bucket) = shard.get_mut(&ip) {
return refill_and_take(bucket, now, self.max_tokens, self.refill_rate);
}
if !self.reserve() {
return false;
}
let mut bucket = Bucket {
tokens: self.max_tokens,
last_refill: now,
last_access: now,
});
let elapsed = now.duration_since(entry.last_refill);
let refill = elapsed.as_secs_f64() * self.refill_rate;
entry.tokens = (entry.tokens + refill).min(self.max_tokens);
entry.last_refill = now;
entry.last_access = now;
if entry.tokens >= 1.0 {
entry.tokens -= 1.0;
true
} else {
false
}
};
let allowed = refill_and_take(&mut bucket, now, self.max_tokens, self.refill_rate);
shard.insert(ip, bucket);
allowed
}
/// Number of live buckets.
#[must_use]
pub fn len(&self) -> usize {
self.buckets.len()
(0..self.shards.len()).map(|index| self.lock(index).len()).sum()
}
/// Whether the table is empty.
#[must_use]
pub fn is_empty(&self) -> bool {
self.buckets.is_empty()
self.len() == 0
}
/// Evicts idle buckets.
///
/// Runs either when the table grows past `MAX_BUCKETS` (cap enforcement)
/// or once per `SWEEP_INTERVAL`. Buckets touched within `eviction_idle`
/// survive, so active sources keep their token state.
pub fn sweep(&self) {
/// Evicts idle buckets from the next shard slice and returns how many were
/// removed. See the eviction policy in the module documentation.
pub fn sweep(&self) -> usize {
let now = Instant::now();
let elapsed_secs = now.duration_since(self.epoch).as_secs();
let last = self.last_sweep_elapsed.load(Ordering::Relaxed);
let due = last == 0 || elapsed_secs.saturating_sub(last) >= SWEEP_INTERVAL.as_secs();
let over_cap = self.buckets.len() > MAX_BUCKETS;
if !over_cap && !due {
return;
let shard_count = self.shards.len();
let start = self.sweep_cursor.fetch_add(SWEEP_SHARDS_PER_TICK, Ordering::Relaxed);
let mut removed = 0;
for offset in 0..SWEEP_SHARDS_PER_TICK.min(shard_count) {
let index = (start + offset) % shard_count;
let mut shard = self.lock(index);
let before = shard.len();
shard.retain(|_, bucket| now.duration_since(bucket.last_access) < self.eviction_idle);
removed += before - shard.len();
}
self.buckets
.retain(|_, b| now.duration_since(b.last_access) < self.eviction_idle);
self.last_sweep_elapsed.store(elapsed_secs, Ordering::Relaxed);
if removed > 0 {
self.live.fetch_sub(removed, Ordering::AcqRel);
}
removed
}
/// Reserves one slot in the table, enforcing the hard bound.
fn reserve(&self) -> bool {
let live = self.live.fetch_add(1, Ordering::AcqRel);
if live >= self.max_buckets {
self.live.fetch_sub(1, Ordering::AcqRel);
return false;
}
true
}
fn shard_index(&self, ip: IpAddr) -> usize {
(self.hasher.hash_one(ip) as usize) % self.shards.len()
}
fn lock(&self, index: usize) -> MutexGuard<'_, HashMap<IpAddr, Bucket>> {
// A poisoned shard still holds consistent token buckets; keep serving
// instead of taking the whole limiter down.
self.shards[index].lock().unwrap_or_else(|poison| poison.into_inner())
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::net::{IpAddr, Ipv4Addr};
fn test_ip(octet: u8) -> IpAddr {
IpAddr::V4(Ipv4Addr::new(10, 0, 0, octet))
}
#[test]
fn test_rate_limit_under() {
let limiter = RateLimiter::new(10.0, 10.0);
assert!(limiter.check(test_ip(1)));
}
#[test]
fn test_rate_limit_over() {
let limiter = RateLimiter::new(1.0, 1.0);
assert!(limiter.check(test_ip(1)));
assert!(!limiter.check(test_ip(1)));
}
#[test]
fn test_rate_limit_burst() {
let limiter = RateLimiter::new(1.0, 5.0);
for _ in 0..5 {
assert!(limiter.check(test_ip(2)));
}
assert!(!limiter.check(test_ip(2)));
}
#[test]
fn test_rate_limit_refill() {
let limiter = RateLimiter::new(100.0, 1.0);
assert!(limiter.check(test_ip(3)));
assert!(!limiter.check(test_ip(3)));
std::thread::sleep(Duration::from_millis(20));
assert!(limiter.check(test_ip(3)));
}
#[test]
fn test_sweep_removes_idle_keeps_active() {
let limiter = RateLimiter::with_eviction(1.0, 10.0, Duration::from_millis(20));
limiter.check(test_ip(1));
limiter.check(test_ip(2));
std::thread::sleep(Duration::from_millis(50));
limiter.check(test_ip(2));
limiter.sweep();
assert_eq!(limiter.len(), 1);
assert!(!limiter.buckets.contains_key(&test_ip(1)));
assert!(limiter.buckets.contains_key(&test_ip(2)));
}
#[test]
fn test_sweep_does_not_remove_active() {
let limiter = RateLimiter::with_eviction(1.0, 10.0, Duration::from_millis(50));
limiter.check(test_ip(1));
limiter.sweep();
assert_eq!(limiter.len(), 1);
assert!(limiter.buckets.contains_key(&test_ip(1)));
fn refill_and_take(bucket: &mut Bucket, now: Instant, max_tokens: f64, refill_rate: f64) -> bool {
let elapsed = now.duration_since(bucket.last_refill);
bucket.tokens = (bucket.tokens + elapsed.as_secs_f64() * refill_rate).min(max_tokens);
bucket.last_refill = now;
bucket.last_access = now;
if bucket.tokens >= 1.0 {
bucket.tokens -= 1.0;
true
} else {
false
}
}

View file

@ -1,3 +1,5 @@
//! `POST /api/v1/auth/login`: password check with a per-IP attempt limiter.
use crate::manager::AppState;
use axum::{
Json,
@ -11,7 +13,6 @@ use std::{
sync::Arc,
time::{Duration, Instant},
};
use subtle::ConstantTimeEq;
const LOGIN_WINDOW: Duration = Duration::from_secs(60);
const LOGIN_MAX_ATTEMPTS: u32 = 5;
@ -33,7 +34,7 @@ pub async fn login(
));
}
if !verify_password(&req.password, &state.api_password) {
if !crate::manager::security::verify_password(&req.password, &state.api_password) {
return Err((
StatusCode::UNAUTHORIZED,
Json(serde_json::json!({"error": "invalid password"})),
@ -49,16 +50,11 @@ pub async fn login(
}
}
fn verify_password(provided: &str, expected: &str) -> bool {
let provided = provided.as_bytes();
let expected = expected.as_bytes();
let len_match = (provided.len() as u64).ct_eq(&(expected.len() as u64));
let min_len = provided.len().min(expected.len());
let bytes_match = provided[..min_len].ct_eq(&expected[..min_len]);
bool::from(len_match & bytes_match)
}
fn allow_login_attempt(limiter: &DashMap<IpAddr, (Instant, u32)>, ip: IpAddr) -> bool {
/// Records a login attempt for `ip` and reports whether it is within the window.
///
/// The limiter allows `LOGIN_MAX_ATTEMPTS` attempts per `LOGIN_WINDOW` and
/// resets the counter when the window elapses.
pub fn allow_login_attempt(limiter: &DashMap<IpAddr, (Instant, u32)>, ip: IpAddr) -> bool {
let now = Instant::now();
let mut slot = limiter.entry(ip).or_insert((now, 0));
let (last_reset, attempts) = &mut *slot;
@ -72,33 +68,3 @@ fn allow_login_attempt(limiter: &DashMap<IpAddr, (Instant, u32)>, ip: IpAddr) ->
}
true
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn verify_password_matches() {
assert!(verify_password("s3cret-pass", "s3cret-pass"));
}
#[test]
fn verify_password_rejects_wrong() {
assert!(!verify_password("wrong-pass", "s3cret-pass"));
}
#[test]
fn verify_password_rejects_different_length() {
assert!(!verify_password("short", "a-longer-password"));
}
#[test]
fn allow_login_attempt_respects_limit() {
let limiter = DashMap::new();
let ip = IpAddr::from([127, 0, 0, 1]);
for _ in 0..LOGIN_MAX_ATTEMPTS {
assert!(allow_login_attempt(&limiter, ip));
}
assert!(!allow_login_attempt(&limiter, ip));
}
}

View file

@ -3,6 +3,7 @@
//! Owns [`AppState`], the handle shared by every handler through [`SharedState`].
pub mod api;
pub mod auth;
pub mod security;
pub mod sync;
use dashmap::DashMap;

135
src/manager/security.rs Normal file
View file

@ -0,0 +1,135 @@
//! Startup security policy of the management plane.
//!
//! Weak credentials turn the control plane API into an authentication bypass:
//! an empty `API_PASSWORD` matches an empty login, and a guessable
//! `JWT_SECRET` lets anyone mint admin tokens. The manager validates both
//! before it binds a socket, and derives its CORS policy from an explicit
//! `CORS_ORIGIN` value instead of silently allowing every origin.
use anyhow::{Result, bail};
use subtle::ConstantTimeEq;
use tower_http::cors::{AllowOrigin, Any, CorsLayer};
/// Minimum length of `API_PASSWORD` in characters.
const MIN_API_PASSWORD_CHARS: usize = 16;
/// Minimum number of non-whitespace bytes in `JWT_SECRET`.
const MIN_JWT_SECRET_BYTES: usize = 32;
/// Placeholder password shipped in documentation and examples.
const DEFAULT_API_PASSWORD: &str = "changeme";
/// Validates `API_PASSWORD` before the manager starts serving.
///
/// # Errors
/// Rejects an empty or whitespace-only password, a password shorter than
/// `MIN_API_PASSWORD_CHARS` characters, and the `changeme` placeholder.
pub fn validate_api_password(password: &str) -> Result<()> {
let trimmed = password.trim();
if trimmed.is_empty() {
bail!("API_PASSWORD must not be empty: an empty password would accept an empty login");
}
if trimmed.chars().count() < MIN_API_PASSWORD_CHARS {
bail!("API_PASSWORD must be at least {MIN_API_PASSWORD_CHARS} characters long");
}
if trimmed.eq_ignore_ascii_case(DEFAULT_API_PASSWORD) {
bail!("API_PASSWORD must not be the default placeholder '{DEFAULT_API_PASSWORD}'");
}
Ok(())
}
/// Validates `JWT_SECRET` before the manager starts serving.
///
/// # Errors
/// Rejects a secret with fewer than `MIN_JWT_SECRET_BYTES` non-whitespace
/// bytes, which includes empty and all-whitespace secrets.
pub fn validate_jwt_secret(secret: &str) -> Result<()> {
let significant = secret.bytes().filter(|byte| !byte.is_ascii_whitespace()).count();
if significant < MIN_JWT_SECRET_BYTES {
bail!("JWT_SECRET must contain at least {MIN_JWT_SECRET_BYTES} non-whitespace bytes, got {significant}");
}
Ok(())
}
/// Compares a login password with the configured one in constant time.
///
/// An empty stored password never verifies: that is the authentication bypass
/// the startup policy ([`validate_api_password`]) rules out, and the handler
/// keeps the same guarantee for any state built by other code paths.
#[must_use]
pub fn verify_password(provided: &str, expected: &str) -> bool {
if expected.is_empty() {
return false;
}
let provided = provided.as_bytes();
let expected = expected.as_bytes();
let len_match = (provided.len() as u64).ct_eq(&(expected.len() as u64));
let min_len = provided.len().min(expected.len());
let bytes_match = provided[..min_len].ct_eq(&expected[..min_len]);
bool::from(len_match & bytes_match)
}
/// Cross-origin resource sharing policy derived from `CORS_ORIGIN`.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum CorsPolicy {
/// No cross-origin layer at all: browsers on other origins are denied.
Deny,
/// Every origin is allowed; requires an explicit `CORS_ORIGIN=*`.
Any,
/// Only the listed origins are allowed.
AllowList(Vec<String>),
}
impl CorsPolicy {
/// Parses the raw `CORS_ORIGIN` value.
///
/// An unset or empty variable denies cross-origin access, `*` must be set
/// explicitly to allow any origin, and anything else is a comma-separated
/// list of origins.
///
/// # Errors
/// Fails on an empty entry, on a wildcard mixed with concrete origins, and
/// on an entry that is not a valid HTTP header value.
pub fn parse(raw: Option<&str>) -> Result<Self> {
let Some(raw) = raw else {
return Ok(Self::Deny);
};
let raw = raw.trim();
if raw.is_empty() {
return Ok(Self::Deny);
}
if raw == "*" {
return Ok(Self::Any);
}
let mut origins = Vec::new();
for entry in raw.split(',') {
let origin = entry.trim();
if origin.is_empty() {
bail!("CORS_ORIGIN contains an empty origin entry");
}
if origin == "*" {
bail!("CORS_ORIGIN wildcard '*' must be used alone, not inside an origin list");
}
axum::http::HeaderValue::from_str(origin)
.map_err(|_| anyhow::anyhow!("CORS_ORIGIN entry '{origin}' is not a valid HTTP header value"))?;
origins.push(origin.to_owned());
}
Ok(Self::AllowList(origins))
}
/// Builds the axum CORS layer for this policy.
///
/// Returns `None` for [`CorsPolicy::Deny`] so the router is served without
/// any cross-origin layer.
#[must_use]
pub fn cors_layer(&self) -> Option<CorsLayer> {
match self {
Self::Deny => None,
Self::Any => Some(CorsLayer::new().allow_origin(Any)),
Self::AllowList(origins) => {
let allowed = origins
.iter()
.filter_map(|origin| axum::http::HeaderValue::from_str(origin).ok());
Some(CorsLayer::new().allow_origin(AllowOrigin::list(allowed)))
},
}
}
}

View file

@ -1,13 +1,42 @@
//! ClickHouse event sink.
//!
//! The data plane must never block on the sink: [`ClickHouseWriter::push`] only
//! touches memory, while the background task drains a batch under the writer
//! lock and performs network I/O outside of it. Every request is bounded by a
//! request timeout and every flush by a global timeout, so an unreachable or
//! hung ClickHouse degrades into dropped events instead of a stalled daemon.
use chrono::{DateTime, Utc};
use prometheus::IntCounter;
use serde::Serialize;
use std::sync::Arc;
use std::sync::{Arc, LazyLock};
use std::time::Duration;
use tokio::sync::Mutex;
use tokio::sync::watch;
const BATCH_SIZE: usize = 1000;
const FLUSH_INTERVAL: Duration = Duration::from_secs(1);
/// Maximum number of events sent in one INSERT.
pub const BATCH_SIZE: usize = 1000;
/// Hard cap on buffered events; further pushes are dropped and counted.
pub const MAX_BUFFER_EVENTS: usize = 10_000;
const FLUSH_INTERVAL: Duration = Duration::from_secs(1);
const FLUSH_TIMEOUT: Duration = Duration::from_secs(5);
const REQUEST_TIMEOUT: Duration = Duration::from_secs(2);
const CONNECT_TIMEOUT: Duration = Duration::from_secs(1);
const ERROR_BODY_LIMIT: usize = 512;
/// Batches drained per tick; keeps sustained event rates from backing up.
const MAX_BATCHES_PER_FLUSH: usize = 4;
/// Events discarded because the buffer was full or a retry could not fit.
pub static CLICKHOUSE_DROPPED: LazyLock<IntCounter> = LazyLock::new(|| {
prometheus::register_int_counter!(
"rampart_clickhouse_dropped_total",
"ClickHouse events dropped (buffer overflow or failed flush)"
)
.expect("CLICKHOUSE_DROPPED")
});
/// One row of the `rampart_events` table.
#[derive(Debug, Clone, Serialize)]
pub struct ClickHouseEvent {
pub timestamp: DateTime<Utc>,
@ -18,72 +47,186 @@ pub struct ClickHouseEvent {
pub data_string: String,
}
/// Buffered ClickHouse writer shared by the data plane and the flush task.
pub struct ClickHouseWriter {
url: String,
client: reqwest::Client,
buffer: Vec<ClickHouseEvent>,
dropped: u64,
}
impl ClickHouseWriter {
/// Creates a writer for `url` with bounded request and connect timeouts.
pub fn new(url: &str) -> Self {
let client = reqwest::Client::builder()
.timeout(REQUEST_TIMEOUT)
.connect_timeout(CONNECT_TIMEOUT)
.build()
.expect("clickhouse http client");
Self {
url: url.to_string(),
client: reqwest::Client::new(),
client,
buffer: Vec::with_capacity(BATCH_SIZE),
dropped: 0,
}
}
/// Queues `event` for the next flush.
///
/// This never performs I/O and never fails; when the buffer is full the
/// event is dropped and counted instead of blocking the data plane.
///
/// # Errors
/// Currently infallible; the result is kept for API stability.
pub async fn push(&mut self, event: ClickHouseEvent) -> anyhow::Result<()> {
self.buffer.push(event);
if self.buffer.len() >= BATCH_SIZE {
self.flush().await?;
}
Ok(())
}
pub async fn flush(&mut self) -> anyhow::Result<()> {
if self.buffer.is_empty() {
if self.buffer.len() >= MAX_BUFFER_EVENTS {
self.record_drop(1);
return Ok(());
}
let events = std::mem::take(&mut self.buffer);
let json = serde_json::to_string(&events)?;
let response = self
.client
.post(&self.url)
.query(&[("query", "INSERT INTO rampart_events FORMAT JSONEachRow")])
.header("Content-Type", "application/json")
.body(json)
.send()
.await?;
let status = response.status();
if !status.is_success() {
let text = response.text().await?;
anyhow::bail!("clickhouse insert failed ({}): {}", status, text);
}
tracing::debug!("flushed {} events to clickhouse", events.len());
self.buffer.push(event);
Ok(())
}
/// Sends every buffered event, re-queueing the batch on failure.
///
/// # Errors
/// Returns an error when the insert fails or exceeds `FLUSH_TIMEOUT`; the
/// batch stays buffered for the next attempt.
pub async fn flush(&mut self) -> anyhow::Result<()> {
let batch = std::mem::take(&mut self.buffer);
if batch.is_empty() {
return Ok(());
}
match tokio::time::timeout(FLUSH_TIMEOUT, send_batch(&self.client, &self.url, &batch)).await {
Ok(Ok(())) => {
tracing::debug!(events = batch.len(), "flushed events to clickhouse");
Ok(())
},
Ok(Err(error)) => {
self.requeue_batch(batch);
Err(error)
},
Err(_) => {
self.requeue_batch(batch);
anyhow::bail!("clickhouse flush timed out after {FLUSH_TIMEOUT:?}")
},
}
}
/// Number of events waiting in the buffer.
pub fn pending(&self) -> usize {
self.buffer.len()
}
/// Number of events this writer discarded.
pub fn dropped(&self) -> u64 {
self.dropped
}
fn take_batch(&mut self) -> Vec<ClickHouseEvent> {
let take = self.buffer.len().min(BATCH_SIZE);
self.buffer.drain(..take).collect()
}
/// Puts a failed batch back in front of the buffer, keeping it bounded.
fn requeue_batch(&mut self, mut batch: Vec<ClickHouseEvent>) {
batch.append(&mut self.buffer);
if batch.len() > MAX_BUFFER_EVENTS {
let overflow = batch.len() - MAX_BUFFER_EVENTS;
batch.truncate(MAX_BUFFER_EVENTS);
self.record_drop(overflow);
}
self.buffer = batch;
}
fn record_drop(&mut self, count: usize) {
let count = u64::try_from(count).unwrap_or(u64::MAX);
let first_drop = self.dropped == 0;
self.dropped = self.dropped.saturating_add(count);
CLICKHOUSE_DROPPED.inc_by(count);
// Log once per writer: further drops are visible in the counter only,
// so a sustained outage cannot flood the log.
if first_drop {
tracing::warn!(count, "clickhouse buffer full; dropping events");
}
}
}
/// Runs the periodic flush loop until shutdown.
///
/// A closed shutdown channel is treated as a shutdown request: the final
/// flush runs and the task exits instead of busy-looping on `changed()`.
pub fn start_flush_task(writer: Arc<Mutex<ClickHouseWriter>>, mut shutdown: watch::Receiver<bool>) {
tokio::spawn(async move {
loop {
tokio::select! {
_ = tokio::time::sleep(FLUSH_INTERVAL) => {
if let Err(e) = writer.lock().await.flush().await {
tracing::error!("clickhouse flush error: {e}");
}
}
_ = shutdown.changed() => {
if *shutdown.borrow() {
tracing::info!("flushing clickhouse on shutdown");
if let Err(e) = writer.lock().await.flush().await {
tracing::error!("clickhouse final flush error: {e}");
for _ in 0..MAX_BATCHES_PER_FLUSH {
if !flush_batch(&writer).await {
break;
}
}
},
changed = shutdown.changed() => {
if changed.is_err() || *shutdown.borrow_and_update() {
tracing::info!("flushing clickhouse on shutdown");
flush_batch(&writer).await;
return;
}
}
},
}
}
});
}
/// Drains one batch under the lock and sends it without holding the lock.
///
/// Returns `false` when there was nothing to send or the send failed.
async fn flush_batch(writer: &Arc<Mutex<ClickHouseWriter>>) -> bool {
let (client, url, batch) = {
let mut writer = writer.lock().await;
let batch = writer.take_batch();
(writer.client.clone(), writer.url.clone(), batch)
};
if batch.is_empty() {
return false;
}
match tokio::time::timeout(FLUSH_TIMEOUT, send_batch(&client, &url, &batch)).await {
Ok(Ok(())) => {
tracing::debug!(events = batch.len(), "flushed events to clickhouse");
true
},
Ok(Err(error)) => {
tracing::warn!(%error, events = batch.len(), "clickhouse flush failed; re-queueing");
requeue_batch(writer, batch).await;
false
},
Err(_) => {
tracing::warn!(events = batch.len(), "clickhouse flush timed out; re-queueing");
requeue_batch(writer, batch).await;
false
},
}
}
async fn requeue_batch(writer: &Arc<Mutex<ClickHouseWriter>>, batch: Vec<ClickHouseEvent>) {
writer.lock().await.requeue_batch(batch);
}
async fn send_batch(client: &reqwest::Client, url: &str, batch: &[ClickHouseEvent]) -> anyhow::Result<()> {
let json = serde_json::to_string(batch)?;
let response = client
.post(url)
.query(&[("query", "INSERT INTO rampart_events FORMAT JSONEachRow")])
.header("Content-Type", "application/json")
.body(json)
.send()
.await?;
let status = response.status();
if !status.is_success() {
let mut text = response.text().await.unwrap_or_default();
text.truncate(ERROR_BODY_LIMIT);
anyhow::bail!("clickhouse insert failed ({status}): {text}");
}
Ok(())
}

View file

@ -1,41 +1,59 @@
//! Redis-backed [`StateStore`] and blacklist synchronisation.
//!
//! Commands share one reconnecting [`ConnectionManager`] and are bounded by
//! `connection::REDIS_TIMEOUT`; pub/sub connections are acquired and
//! subscribed under the same timeout.
use crate::store::StateStore;
use redis::AsyncCommands;
use redis::aio::ConnectionManager;
mod backoff;
mod connection;
mod sync;
pub use backoff::{BACKOFF_MAX, BACKOFF_MIN, next_backoff};
pub use sync::{start_blacklist_sync, start_local_ban_publisher};
/// Redis-backed state store.
pub struct RedisStore {
client: redis::Client,
manager: ConnectionManager,
}
impl RedisStore {
pub fn new(url: &str) -> anyhow::Result<Self> {
/// Connects to `url` and establishes a reconnecting manager.
///
/// # Errors
/// Returns an error for an invalid URL, an unreachable server or an attempt
/// that exceeds `connection::REDIS_TIMEOUT`.
pub async fn new(url: &str) -> anyhow::Result<Self> {
let client = redis::Client::open(url)?;
Ok(Self { client })
let manager = connection::connection_manager(&client).await?;
Ok(Self { manager })
}
}
impl StateStore for RedisStore {
async fn get(&self, key: &str) -> anyhow::Result<Option<String>> {
let mut conn = self.client.get_multiplexed_async_connection().await?;
Ok(conn.get(key).await?)
let mut conn = self.manager.clone();
connection::command(conn.get(key)).await
}
async fn set(&self, key: &str, value: &str) -> anyhow::Result<()> {
let mut conn = self.client.get_multiplexed_async_connection().await?;
let _: () = conn.set(key, value).await?;
let mut conn = self.manager.clone();
let _: () = connection::command(conn.set(key, value)).await?;
Ok(())
}
async fn del(&self, key: &str) -> anyhow::Result<()> {
let mut conn = self.client.get_multiplexed_async_connection().await?;
let _: () = conn.del(key).await?;
let mut conn = self.manager.clone();
let _: () = connection::command(conn.del(key)).await?;
Ok(())
}
async fn publish(&self, channel: &str, message: &str) -> anyhow::Result<()> {
let mut conn = self.client.get_multiplexed_async_connection().await?;
let _: () = conn.publish(channel, message).await?;
let mut conn = self.manager.clone();
let _: i64 = connection::command(conn.publish(channel, message)).await?;
Ok(())
}
}

View file

@ -0,0 +1,14 @@
//! Exponential reconnect backoff shared by the Redis sync tasks.
use std::time::Duration;
/// Initial reconnect delay.
pub const BACKOFF_MIN: Duration = Duration::from_secs(1);
/// Upper bound for the reconnect delay.
pub const BACKOFF_MAX: Duration = Duration::from_secs(30);
/// Doubles `previous`, capped at `max`; never overflows.
#[must_use]
pub fn next_backoff(previous: Duration, max: Duration) -> Duration {
previous.saturating_mul(2).min(max)
}

View file

@ -0,0 +1,36 @@
//! Timeout-bounded Redis connection helpers.
//!
//! Every connect and command in the store layer goes through this module so a
//! stalled server can never block a data-plane or shutdown path.
use redis::aio::ConnectionManager;
use std::future::Future;
use std::time::Duration;
/// Maximum time spent on one Redis connect or command.
pub const REDIS_TIMEOUT: Duration = Duration::from_secs(2);
/// Opens a reconnecting connection manager, bounded by `REDIS_TIMEOUT`.
///
/// # Errors
/// Returns an error for an unreachable server or a timed-out attempt.
pub async fn connection_manager(client: &redis::Client) -> anyhow::Result<ConnectionManager> {
match tokio::time::timeout(REDIS_TIMEOUT, client.get_connection_manager()).await {
Ok(result) => Ok(result?),
Err(_) => anyhow::bail!("redis connection timed out after {REDIS_TIMEOUT:?}"),
}
}
/// Awaits a Redis command future, bounded by `REDIS_TIMEOUT`.
///
/// # Errors
/// Returns an error when the command fails or exceeds the timeout.
pub async fn command<F, T>(future: F) -> anyhow::Result<T>
where
F: Future<Output = redis::RedisResult<T>>,
{
match tokio::time::timeout(REDIS_TIMEOUT, future).await {
Ok(result) => Ok(result?),
Err(_) => anyhow::bail!("redis command timed out after {REDIS_TIMEOUT:?}"),
}
}

View file

@ -1,9 +1,18 @@
//! Blacklist synchronisation over a Redis pub/sub channel.
//!
//! The subscriber reconciles the durable snapshot on every (re)connect, then
//! applies events. All connects, subscribes and commands are timeout-bounded;
//! a failed attempt backs off exponentially and a panicking handler is logged
//! instead of killing the task.
mod event;
mod publisher;
mod snapshot;
pub use publisher::start_local_ban_publisher;
use super::backoff::{BACKOFF_MAX, BACKOFF_MIN, next_backoff};
use super::connection::{REDIS_TIMEOUT, connection_manager};
use crate::filter::blacklist::Blacklist;
use futures::StreamExt;
use std::sync::Arc;
@ -11,19 +20,17 @@ use std::time::Duration;
use tokio::sync::watch;
const BLACKLIST_CHANNEL: &str = "rampart:blacklist:events";
const BACKOFF_MIN: Duration = Duration::from_secs(1);
const BACKOFF_MAX: Duration = Duration::from_secs(30);
fn next_backoff(prev: Duration, max: Duration) -> Duration {
prev.saturating_mul(2).min(max)
}
/// Why a subscription attempt ended.
enum Flow {
/// Shutdown was requested.
Stop,
/// The attempt failed; reconnect after a backoff delay.
Retry,
}
async fn wait_for_shutdown(shutdown: &mut watch::Receiver<bool>) {
/// Resolves once shutdown is requested or the channel is closed.
pub(super) async fn wait_for_shutdown(shutdown: &mut watch::Receiver<bool>) {
loop {
if shutdown.changed().await.is_err() || *shutdown.borrow_and_update() {
return;
@ -31,6 +38,7 @@ async fn wait_for_shutdown(shutdown: &mut watch::Receiver<bool>) {
}
}
/// Runs the blacklist subscriber until shutdown.
pub async fn start_blacklist_sync(
client: &redis::Client,
blacklist: Arc<Blacklist>,
@ -63,30 +71,44 @@ async fn subscribe_once(
backoff: &mut Duration,
) -> Flow {
let mut pubsub = tokio::select! {
connected = client.get_async_pubsub() => match connected {
Ok(pubsub) => pubsub,
Err(e) => {
connected = tokio::time::timeout(REDIS_TIMEOUT, client.get_async_pubsub()) => match connected {
Ok(Ok(pubsub)) => pubsub,
Ok(Err(e)) => {
tracing::error!("failed to connect to Redis for blacklist sync: {e}");
return Flow::Retry;
},
Err(_) => {
tracing::error!("timed out connecting to Redis for blacklist sync");
return Flow::Retry;
},
},
() = wait_for_shutdown(shutdown) => return Flow::Stop,
};
tokio::select! {
subscribed = pubsub.subscribe(BLACKLIST_CHANNEL) => match subscribed {
Ok(()) => tracing::info!("subscribed to {BLACKLIST_CHANNEL}"),
Err(e) => {
subscribed = tokio::time::timeout(REDIS_TIMEOUT, pubsub.subscribe(BLACKLIST_CHANNEL)) => match subscribed {
Ok(Ok(())) => {
tracing::info!("subscribed to {BLACKLIST_CHANNEL}");
*backoff = BACKOFF_MIN;
},
Ok(Err(e)) => {
tracing::error!("failed to subscribe to blacklist events: {e}");
return Flow::Retry;
},
Err(_) => {
tracing::error!("timed out subscribing to blacklist events");
return Flow::Retry;
},
},
() = wait_for_shutdown(shutdown) => return Flow::Stop,
}
tokio::select! {
loaded = snapshot::reconcile(client, blacklist) => match loaded {
Ok(()) => tracing::info!("blacklist snapshot reconciled from Redis"),
loaded = reconcile(blacklist, client) => match loaded {
Ok(()) => {
tracing::info!("blacklist snapshot reconciled from Redis");
*backoff = BACKOFF_MIN;
},
Err(e) => {
tracing::error!("failed to reconcile blacklist snapshot: {e}");
return Flow::Retry;
@ -99,13 +121,7 @@ async fn subscribe_once(
loop {
tokio::select! {
maybe_msg = stream.next() => match maybe_msg {
Some(msg) => {
if let Err(e) = event::handle(&msg, blacklist) {
tracing::error!("blacklist event error: {e}");
} else {
*backoff = BACKOFF_MIN;
}
},
Some(msg) => handle_message(&msg, blacklist, backoff),
None => {
tracing::error!("blacklist pubsub stream ended; resubscribing");
return Flow::Retry;
@ -116,22 +132,18 @@ async fn subscribe_once(
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn backoff_doubles_until_cap() {
let mut b = BACKOFF_MIN;
for expected_secs in [2, 4, 8, 16, 30, 30] {
b = next_backoff(b, BACKOFF_MAX);
assert_eq!(b, Duration::from_secs(expected_secs));
}
}
#[test]
fn backoff_never_exceeds_max_even_with_huge_prev() {
let b = next_backoff(Duration::from_secs(u64::MAX / 2), BACKOFF_MAX);
assert!(b <= BACKOFF_MAX);
/// Handles one event without letting a rejected or panicking payload kill the
/// subscriber task.
fn handle_message(msg: &redis::Msg, blacklist: &Blacklist, backoff: &mut Duration) {
let handled = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| event::handle(msg, blacklist)));
match handled {
Ok(Ok(())) => *backoff = BACKOFF_MIN,
Ok(Err(error)) => tracing::warn!(%error, "rejected blacklist event"),
Err(_) => tracing::error!("blacklist event handler panicked; continuing"),
}
}
async fn reconcile(blacklist: &Blacklist, client: &redis::Client) -> anyhow::Result<()> {
let manager = connection_manager(client).await?;
snapshot::reconcile(&manager, blacklist).await
}

View file

@ -1,10 +1,14 @@
use crate::filter::blacklist::Blacklist;
//! Decoding and validation of blacklist events received from Redis.
//!
//! A remote or stale manager must not be able to ban an address for an
//! arbitrary duration, so every ban is range-checked against
//! [`MAX_BAN_DURATION_SECS`] before it reaches the in-memory table.
use crate::filter::blacklist::{Blacklist, MAX_BAN_DURATION_SECS};
use redis::Msg;
use serde::Deserialize;
use std::time::Duration;
const MAX_BAN_DURATION_SECS: u64 = 31_536_000;
#[derive(Deserialize)]
struct BlacklistEvent {
ip: String,
@ -27,31 +31,54 @@ pub(super) fn handle(msg: &Msg, blacklist: &Blacklist) -> anyhow::Result<()> {
match event.action.as_str() {
"ban" => {
let duration = match event.expires_at_unix {
Some(expires_at) => {
if chrono::DateTime::from_timestamp(expires_at, 0).is_none() {
anyhow::bail!("invalid blacklist event expiry timestamp");
}
let remaining = expires_at.saturating_sub(chrono::Utc::now().timestamp());
if remaining <= 0 {
blacklist.remove_redis(ip);
return Ok(());
}
let remaining = u64::try_from(remaining)?;
if remaining > MAX_BAN_DURATION_SECS {
anyhow::bail!("blacklist event expiry exceeds maximum duration");
}
Duration::from_secs(remaining)
},
None => Duration::from_secs(event.duration_secs),
Some(expires_at) => absolute_duration(expires_at)?,
None => checked_duration(event.duration_secs)?,
};
blacklist.add_redis(ip, duration, "manager_event");
tracing::info!("blacklist add via Redis: {}", event.ip);
match duration {
Some(duration) => {
blacklist.add_redis(ip, duration, "manager_event");
tracing::info!(ip = %event.ip, "blacklist add via Redis");
},
None => {
blacklist.remove_redis(ip);
tracing::info!(ip = %event.ip, "blacklist expiry already passed");
},
}
},
"unban" => {
blacklist.remove_redis(ip);
tracing::info!("blacklist remove via Redis: {}", event.ip);
tracing::info!(ip = %event.ip, "blacklist remove via Redis");
},
action => anyhow::bail!("unknown action: {action}"),
action => anyhow::bail!("unknown blacklist action: {action}"),
}
Ok(())
}
/// Converts an absolute expiry into the remaining ban duration.
///
/// Returns `Ok(None)` when the expiry is already in the past.
fn absolute_duration(expires_at: i64) -> anyhow::Result<Option<Duration>> {
if chrono::DateTime::from_timestamp(expires_at, 0).is_none() {
anyhow::bail!("invalid blacklist event expiry timestamp");
}
let remaining = expires_at.saturating_sub(chrono::Utc::now().timestamp());
if remaining <= 0 {
return Ok(None);
}
let remaining = u64::try_from(remaining)?;
if remaining > MAX_BAN_DURATION_SECS {
anyhow::bail!("blacklist event expiry exceeds {MAX_BAN_DURATION_SECS}s");
}
Ok(Some(Duration::from_secs(remaining)))
}
/// Validates a relative ban duration.
fn checked_duration(secs: u64) -> anyhow::Result<Option<Duration>> {
if secs > MAX_BAN_DURATION_SECS {
anyhow::bail!("blacklist event duration {secs}s exceeds {MAX_BAN_DURATION_SECS}s");
}
if secs == 0 {
return Ok(None);
}
Ok(Some(Duration::from_secs(secs)))
}

View file

@ -1,6 +1,16 @@
//! Publishes locally detected bans to Redis.
//!
//! Each ban is stored atomically (set member, metadata hash, event publish) via
//! a Lua script on a shared reconnecting connection. Failed publishes retry
//! with exponential backoff and abandon the event when shutdown is requested.
use super::wait_for_shutdown;
use crate::filter::blacklist::{Blacklist, LocalBanEvent};
use crate::store::redis::backoff::{BACKOFF_MAX, BACKOFF_MIN, next_backoff};
use crate::store::redis::connection::{command, connection_manager};
use redis::aio::ConnectionManager;
use serde::Serialize;
use std::time::Duration;
use std::sync::Arc;
use tokio::sync::{broadcast, watch};
const BLACKLIST_KEY: &str = "rampart:blacklist";
@ -24,14 +34,16 @@ struct BanEvent<'a> {
reason: &'a str,
}
/// Publishes local bans and the current local snapshot until shutdown.
pub async fn start_local_ban_publisher(
client: redis::Client,
blacklist: std::sync::Arc<Blacklist>,
blacklist: Arc<Blacklist>,
mut events: broadcast::Receiver<LocalBanEvent>,
mut shutdown: watch::Receiver<bool>,
) {
let mut manager: Option<ConnectionManager> = None;
for event in blacklist.local_snapshot() {
if !publish_with_retry(&client, event, &mut shutdown).await {
if !publish_with_retry(&client, &mut manager, event, &mut shutdown).await {
return;
}
}
@ -40,14 +52,14 @@ pub async fn start_local_ban_publisher(
tokio::select! {
received = events.recv() => match received {
Ok(event) => {
if !publish_with_retry(&client, event, &mut shutdown).await {
if !publish_with_retry(&client, &mut manager, event, &mut shutdown).await {
return;
}
},
Err(broadcast::error::RecvError::Lagged(skipped)) => {
tracing::warn!(skipped, "local ban publisher lagged; reconciling active local bans");
for event in blacklist.local_snapshot() {
if !publish_with_retry(&client, event, &mut shutdown).await {
if !publish_with_retry(&client, &mut manager, event, &mut shutdown).await {
return;
}
}
@ -63,19 +75,29 @@ pub async fn start_local_ban_publisher(
}
}
/// Publishes one event, retrying until it succeeds or shutdown is requested.
async fn publish_with_retry(
client: &redis::Client,
manager: &mut Option<ConnectionManager>,
event: LocalBanEvent,
shutdown: &mut watch::Receiver<bool>,
) -> bool {
let mut delay = Duration::from_secs(1);
let mut delay = BACKOFF_MIN;
loop {
match publish_ban(client, &event).await {
let result = tokio::select! {
result = publish_ban(client, manager, &event) => result,
() = wait_for_shutdown(shutdown) => return false,
};
match result {
Ok(()) => return true,
Err(error) => tracing::warn!(%error, ip = %event.ip, "failed to persist local ban; retrying"),
Err(error) => {
// Force a reconnect on the next attempt.
*manager = None;
tracing::warn!(%error, ip = %event.ip, "failed to persist local ban; retrying");
},
}
tokio::select! {
_ = tokio::time::sleep(delay) => delay = (delay * 2).min(Duration::from_secs(30)),
_ = tokio::time::sleep(delay) => delay = next_backoff(delay, BACKOFF_MAX),
changed = shutdown.changed() => {
if changed.is_err() || *shutdown.borrow_and_update() {
return false;
@ -85,7 +107,11 @@ async fn publish_with_retry(
}
}
async fn publish_ban(client: &redis::Client, event: &LocalBanEvent) -> anyhow::Result<()> {
async fn publish_ban(
client: &redis::Client,
manager: &mut Option<ConnectionManager>,
event: &LocalBanEvent,
) -> anyhow::Result<()> {
let now = chrono::Utc::now().timestamp();
let remaining = event.expires_at_unix.saturating_sub(now);
if remaining <= 0 {
@ -103,17 +129,26 @@ async fn publish_ban(client: &redis::Client, event: &LocalBanEvent) -> anyhow::R
expires_at_unix: event.expires_at_unix,
reason: &event.reason,
})?;
let mut conn = client.get_multiplexed_async_connection().await?;
let _: i64 = redis::cmd("EVAL")
.arg(STORE_BAN_SCRIPT)
.arg(3)
.arg(BLACKLIST_KEY)
.arg(BLACKLIST_METADATA_KEY)
.arg(BLACKLIST_CHANNEL)
.arg(event.ip.to_string())
.arg(metadata)
.arg(payload)
.query_async(&mut conn)
.await?;
let mut conn = match manager {
Some(conn) => conn.clone(),
None => {
let conn = connection_manager(client).await?;
*manager = Some(conn.clone());
conn
},
};
let _: i64 = command(
redis::cmd("EVAL")
.arg(STORE_BAN_SCRIPT)
.arg(3)
.arg(BLACKLIST_KEY)
.arg(BLACKLIST_METADATA_KEY)
.arg(BLACKLIST_CHANNEL)
.arg(event.ip.to_string())
.arg(metadata)
.arg(payload)
.query_async(&mut conn),
)
.await?;
Ok(())
}

View file

@ -1,5 +1,13 @@
use crate::filter::blacklist::Blacklist;
//! Durable blacklist snapshot reconciliation.
//!
//! The snapshot is rebuilt from `rampart:blacklist` (set) and
//! `rampart:blacklist:metadata` (hash) after every (re)subscribe and then
//! swapped into the mirrored in-memory table.
use crate::filter::blacklist::{Blacklist, MAX_BAN_DURATION_SECS};
use crate::store::redis::connection::command;
use redis::AsyncCommands;
use redis::aio::ConnectionManager;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::time::Duration;
@ -7,7 +15,6 @@ use std::time::Duration;
const BLACKLIST_KEY: &str = "rampart:blacklist";
const BLACKLIST_METADATA_KEY: &str = "rampart:blacklist:metadata";
const LEGACY_BAN_DURATION_SECS: i64 = 300;
const MAX_BAN_DURATION_SECS: i64 = 31_536_000;
#[derive(Deserialize, Serialize)]
struct BlacklistMetadata {
@ -16,11 +23,15 @@ struct BlacklistMetadata {
reason: String,
}
/// Restore current durable bans after subscribing to the event channel.
pub(super) async fn reconcile(client: &redis::Client, blacklist: &Blacklist) -> anyhow::Result<()> {
let mut conn = client.get_multiplexed_async_connection().await?;
let members: Vec<String> = redis::cmd("SMEMBERS").arg(BLACKLIST_KEY).query_async(&mut conn).await?;
let metadata: HashMap<String, String> = conn.hgetall(BLACKLIST_METADATA_KEY).await?;
/// Restores current durable bans after subscribing to the event channel.
///
/// # Errors
/// Returns an error when Redis is unreachable, a command times out or a
/// recorded timestamp is outside the representable range.
pub(super) async fn reconcile(manager: &ConnectionManager, blacklist: &Blacklist) -> anyhow::Result<()> {
let mut conn = manager.clone();
let members: Vec<String> = command(redis::cmd("SMEMBERS").arg(BLACKLIST_KEY).query_async(&mut conn)).await?;
let metadata: HashMap<String, String> = command(conn.hgetall(BLACKLIST_METADATA_KEY)).await?;
let now = chrono::Utc::now().timestamp();
let mut active = Vec::with_capacity(members.len());
@ -39,22 +50,28 @@ pub(super) async fn reconcile(client: &redis::Client, blacklist: &Blacklist) ->
None => continue,
},
};
if record.expires_at_unix <= now
|| chrono::DateTime::from_timestamp(record.created_at_unix, 0).is_none()
if chrono::DateTime::from_timestamp(record.created_at_unix, 0).is_none()
|| chrono::DateTime::from_timestamp(record.expires_at_unix, 0).is_none()
|| record.expires_at_unix.saturating_sub(now) > MAX_BAN_DURATION_SECS
{
continue;
}
let remaining = u64::try_from(record.expires_at_unix - now)?;
let remaining = record.expires_at_unix.saturating_sub(now);
if remaining <= 0 {
continue;
}
let remaining = u64::try_from(remaining)?;
if remaining > MAX_BAN_DURATION_SECS {
continue;
}
active.push((ip, Duration::from_secs(remaining), record.reason));
}
blacklist.replace_redis(active);
Ok(())
}
/// Backfills metadata for a ban written by an older manager.
async fn legacy_record(
conn: &mut redis::aio::MultiplexedConnection,
conn: &mut ConnectionManager,
target: &str,
now: i64,
) -> anyhow::Result<Option<BlacklistMetadata>> {
@ -63,15 +80,17 @@ async fn legacy_record(
expires_at_unix: now.saturating_add(LEGACY_BAN_DURATION_SECS),
reason: "legacy_unverified".to_string(),
};
let inserted: i64 = redis::cmd("HSETNX")
.arg(BLACKLIST_METADATA_KEY)
.arg(target)
.arg(serde_json::to_string(&legacy)?)
.query_async(conn)
.await?;
let inserted: i64 = command(
redis::cmd("HSETNX")
.arg(BLACKLIST_METADATA_KEY)
.arg(target)
.arg(serde_json::to_string(&legacy)?)
.query_async(conn),
)
.await?;
if inserted == 1 {
return Ok(Some(legacy));
}
let current: Option<String> = conn.hget(BLACKLIST_METADATA_KEY, target).await?;
let current: Option<String> = command(conn.hget(BLACKLIST_METADATA_KEY, target)).await?;
Ok(current.and_then(|value| serde_json::from_str(&value).ok()))
}

View file

@ -1,16 +1,93 @@
//! Attack state machine over the hour-of-week traffic profile.
//!
//! [`AttackDetector`] scores every one-second window against the baselines
//! *before* the window is folded into them. Scoring first is what keeps a
//! sustained flood visible: a detector that learns the attack as normal stops
//! escalating after a few ticks, and anything that depends on
//! [`AttackStatus::UnderAttack`] (for example offender bans) stops working for
//! the rest of the attack.
//!
//! Two baselines are combined:
//!
//! - [`TrafficProfiler`]: the hour-of-week shape, updated from clean windows.
//! - A slow seeded EWMA (1% per window): the global traffic level. It covers
//! the cold start and the first tick of every hour slot, where the
//! hour-of-week baseline is still zero or close to it.
//!
//! A window is scored against the higher of the two, and attack-level windows
//! are not recorded at all. A flood therefore never becomes the baseline it is
//! compared to, while ordinary drift (below [`DetectorThresholds::attack_score`])
//! keeps both baselines current.
//!
//! Leaving [`AttackStatus::UnderAttack`] requires
//! [`DetectorThresholds::recover_after`] consecutive windows below
//! [`DetectorThresholds::attack_score`]. The exit hysteresis keeps a single
//! calm window in the middle of an attack from restarting the escalation
//! count.
use crate::traffic::ewma::Ewma;
use crate::traffic::profiler::TrafficProfiler;
/// Smoothing factor of the slow global baseline: one percent per window, so a
/// legitimate traffic shift is learned in about a hundred seconds.
const SLOW_ALPHA: f64 = 0.01;
/// Upper bound of a reported score; `10` means "at least ten times the
/// baseline" and keeps the attack state machine from chasing huge ratios.
const MAX_ANOMALY_RATIO: f64 = 10.0;
/// Classification of the current traffic window.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum AttackStatus {
/// Traffic is within the learned baseline.
Normal,
/// A ratio above [`DetectorThresholds::suspicious_score`] was observed,
/// but the escalation count is not reached yet.
Suspicious,
/// Consecutive anomalous windows were observed; the edge escalates.
UnderAttack,
}
/// Tunables of the attack state machine.
///
/// The scores are ratios to the baseline (a value of `3.0` means "three times
/// the usual rate"). The defaults match the v0.3 policy; a caller that needs a
/// different sensitivity passes its own values to
/// [`AttackDetector::with_thresholds`].
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct DetectorThresholds {
/// Ratio that marks a window as suspicious.
pub suspicious_score: f64,
/// Ratio that counts a window as anomalous for escalation.
pub attack_score: f64,
/// Consecutive anomalous windows required to enter `UnderAttack`.
pub escalate_after: u32,
/// Consecutive sub-attack windows required to leave `UnderAttack`.
pub recover_after: u32,
}
impl Default for DetectorThresholds {
fn default() -> Self {
Self {
suspicious_score: 2.0,
attack_score: 3.0,
escalate_after: 3,
recover_after: 5,
}
}
}
/// Detects floods from the packets-per-second and connections-per-second
/// profiles.
pub struct AttackDetector {
pps_profiler: TrafficProfiler,
cps_profiler: TrafficProfiler,
pps_slow: Ewma,
cps_slow: Ewma,
primed: bool,
thresholds: DetectorThresholds,
consecutive_anomalies: u32,
recovery_windows: u32,
under_attack: bool,
}
impl Default for AttackDetector {
@ -20,95 +97,99 @@ impl Default for AttackDetector {
}
impl AttackDetector {
/// Builds a detector with the default [`DetectorThresholds`].
#[must_use]
pub fn new() -> Self {
Self::with_thresholds(DetectorThresholds::default())
}
/// Builds a detector with explicit thresholds.
#[must_use]
pub fn with_thresholds(thresholds: DetectorThresholds) -> Self {
Self {
pps_profiler: TrafficProfiler::new(),
cps_profiler: TrafficProfiler::new(),
pps_slow: Ewma::new(SLOW_ALPHA),
cps_slow: Ewma::new(SLOW_ALPHA),
primed: false,
thresholds,
consecutive_anomalies: 0,
recovery_windows: 0,
under_attack: false,
}
}
/// Analyses one metrics window (packets/s and new connections/s) against
/// the accumulated profiles.
/// Analyses one metrics window (packets/s and new connections/s) and
/// returns the attack classification.
///
/// A spike in either metric counts as an anomaly. Three consecutive
/// anomalous windows escalate to [`AttackStatus::UnderAttack`]; a single
/// one above the suspicious score yields [`AttackStatus::Suspicious`].
/// A window is anomalous when its ratio to the baseline reaches
/// [`DetectorThresholds::attack_score`].
/// [`DetectorThresholds::escalate_after`] consecutive anomalous windows
/// enter `UnderAttack`; the status is left only after
/// [`DetectorThresholds::recover_after`] consecutive windows below the
/// attack score. The first window only primes the baselines.
pub fn analyze(&mut self, pps: f64, cps: f64) -> AttackStatus {
self.pps_profiler.record(pps);
self.cps_profiler.record(cps);
let score = self
.pps_profiler
.anomaly_score(pps)
.max(self.cps_profiler.anomaly_score(cps));
if score > 3.0 {
self.consecutive_anomalies += 1;
} else {
self.consecutive_anomalies = 0;
if !self.primed {
self.primed = true;
self.pps_slow.seed(pps);
self.cps_slow.seed(cps);
self.pps_profiler.record(pps);
self.cps_profiler.record(cps);
return AttackStatus::Normal;
}
if self.consecutive_anomalies >= 3 {
// Score against the baselines first: recording first would fold the
// spike into the baseline it is scored against.
let pps_baseline = self.pps_profiler.baseline().max(self.pps_slow.value());
let cps_baseline = self.cps_profiler.baseline().max(self.cps_slow.value());
let score = anomaly_ratio(pps, pps_baseline).max(anomaly_ratio(cps, cps_baseline));
self.update_state(score);
// Attack-level windows never enter the baselines (see the module
// documentation); everything else keeps both of them current.
if score < self.thresholds.attack_score {
self.pps_profiler.record(pps);
self.cps_profiler.record(cps);
self.pps_slow.update(pps);
self.cps_slow.update(cps);
}
if self.under_attack {
AttackStatus::UnderAttack
} else if score > 2.0 {
} else if score >= self.thresholds.suspicious_score {
AttackStatus::Suspicious
} else {
AttackStatus::Normal
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_attack_status_normal() {
let mut d = AttackDetector::new();
for _ in 0..10 {
d.analyze(100.0, 10.0);
fn update_state(&mut self, score: f64) {
if score >= self.thresholds.attack_score {
self.consecutive_anomalies = self.consecutive_anomalies.saturating_add(1);
self.recovery_windows = 0;
} else {
self.consecutive_anomalies = 0;
self.recovery_windows = self.recovery_windows.saturating_add(1);
}
assert_eq!(d.analyze(100.0, 10.0), AttackStatus::Normal);
}
#[test]
fn test_attack_status_suspicious_pps_spike() {
let mut d = AttackDetector::new();
for _ in 0..10 {
d.analyze(1.0, 1.0);
if self.under_attack {
if self.recovery_windows >= self.thresholds.recover_after {
self.under_attack = false;
}
} else if self.consecutive_anomalies >= self.thresholds.escalate_after {
self.under_attack = true;
}
assert_eq!(d.analyze(5.0, 1.0), AttackStatus::Suspicious);
}
#[test]
fn test_attack_status_suspicious_cps_spike() {
let mut d = AttackDetector::new();
for _ in 0..10 {
d.analyze(1.0, 1.0);
}
assert_eq!(d.analyze(1.0, 5.0), AttackStatus::Suspicious);
}
#[test]
fn test_attack_status_under_attack() {
let mut d = AttackDetector::new();
for _ in 0..10 {
d.analyze(1.0, 1.0);
}
d.analyze(10_000.0, 100.0);
d.analyze(10_000.0, 100.0);
assert_eq!(d.analyze(10_000.0, 100.0), AttackStatus::UnderAttack);
}
#[test]
fn test_consecutive_resets_on_normal() {
let mut d = AttackDetector::new();
for _ in 0..10 {
d.analyze(1.0, 1.0);
}
d.analyze(100.0, 10.0);
d.analyze(100.0, 10.0);
d.analyze(1.0, 1.0);
assert_eq!(d.analyze(1.0, 1.0), AttackStatus::Normal);
}
}
/// Ratio of a window to its baseline, capped at [`MAX_ANOMALY_RATIO`].
///
/// Returns zero for an unset baseline and for non-finite windows, so a broken
/// counter cannot pin the detector in the attack state.
fn anomaly_ratio(sample: f64, baseline: f64) -> f64 {
if !sample.is_finite() || baseline <= 0.0 {
return 0.0;
}
(sample.max(0.0) / baseline).min(MAX_ANOMALY_RATIO)
}

View file

@ -1,5 +1,29 @@
//! Time-aware exponential moving average used by the traffic profilers.
//!
//! The average is expressed in samples per unit time and is fed once per
//! metrics window by [`crate::traffic::profiler::TrafficProfiler`].
use std::time::Instant;
/// Smoothing factor used for `alpha` values outside the valid `(0, 1]` range
/// and for `NaN`, so a malformed configuration cannot freeze the baseline.
const DEFAULT_ALPHA: f64 = 0.125;
/// Exponential moving average with a wall-clock aware step count.
///
/// # Formula
///
/// For a sample `x` that arrives `dt` seconds after the previous update,
/// `steps = max(dt, 1)` and `weight = (1 - alpha)^steps`. The average is then
///
/// ```text
/// value = value * weight + x * (1 - weight)
/// ```
///
/// A larger `alpha` gives more weight to the newest sample; `alpha = 1` makes
/// the average equal to the last finite sample. The step count keeps the
/// smoothing independent of the tick rate: one update after a two-second gap
/// is worth two one-second updates.
pub struct Ewma {
value: f64,
last: Instant,
@ -7,51 +31,59 @@ pub struct Ewma {
}
impl Ewma {
/// Builds a smoothed average that starts at zero.
///
/// `alpha` is clamped to `(0, 1]`; `NaN`, infinity and non-positive values
/// fall back to the default 0.125 instead of silently disabling the
/// smoothing or producing a non-finite baseline.
#[must_use]
pub fn new(alpha: f64) -> Self {
Self {
value: 0.0,
last: Instant::now(),
alpha,
alpha: sanitize_alpha(alpha),
}
}
/// Folds `sample` into the average, following the formula in the type
/// documentation.
///
/// Non-finite samples are ignored (the update timestamp is not advanced
/// either), so one broken counter reading cannot poison the baseline.
pub fn update(&mut self, sample: f64) {
if !sample.is_finite() {
return;
}
let now = Instant::now();
let elapsed = now.duration_since(self.last).as_secs_f64();
let steps = elapsed.max(1.0);
let steps = now.duration_since(self.last).as_secs_f64().max(1.0);
let weight = (1.0 - self.alpha).powf(steps);
self.value = self.value * weight + sample * (1.0 - weight);
self.last = now;
}
/// Replaces the average with `value` and restarts the step timer.
///
/// Used to prime a baseline from the first observed window instead of
/// climbing from zero, which would otherwise read as an anomaly.
pub fn seed(&mut self, value: f64) {
if !value.is_finite() {
return;
}
self.value = value;
self.last = Instant::now();
}
/// Current smoothed value.
#[must_use]
pub fn value(&self) -> f64 {
self.value
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_ewma_initial() {
let ewma = Ewma::new(0.125);
assert_eq!(ewma.value(), 0.0);
}
#[test]
fn test_ewma_update() {
let mut ewma = Ewma::new(1.0);
ewma.update(100.0);
assert!((ewma.value() - 100.0).abs() < 1.0);
}
#[test]
fn test_ewma_convergence() {
let mut ewma = Ewma::new(0.5);
for _ in 0..10 {
ewma.update(50.0);
}
assert!((ewma.value() - 50.0).abs() < 1.0);
fn sanitize_alpha(alpha: f64) -> f64 {
if alpha.is_finite() && alpha > 0.0 {
alpha.min(1.0)
} else {
DEFAULT_ALPHA
}
}

View file

@ -1,9 +1,79 @@
use dashmap::DashMap;
use std::net::IpAddr;
use std::sync::Arc;
//! Per-source reputation with time decay and idle-TTL eviction.
//!
//! A source starts neutral (`0`); [`IpReputation::record_bad`] subtracts ten
//! points and [`IpReputation::record_good`] adds one, clamped to `[-100, 100]`.
//! Scores decay toward neutral on read (half of the magnitude per thirty
//! minutes of inactivity), so one old success cannot keep a source trusted
//! forever and one old failure cannot keep it toxic.
//!
//! State lives in sixty-four locked shards. [`IpReputation::sweep`] visits
//! eight of them per call and drops entries idle for longer than ten minutes;
//! recording advances one step per minute, so the table is reclaimed without
//! an external scheduler and no full-map `retain`.
//!
//! [`IpReputation::offenders_below`] returns at most 64 of the worst sources,
//! so one escalation cannot flood the ban list or the log.
use std::collections::hash_map::RandomState;
use std::collections::{BinaryHeap, HashMap};
use std::hash::BuildHasher;
use std::net::IpAddr;
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
use std::sync::{Mutex, MutexGuard};
use std::time::{Duration, Instant};
/// Idle age after which a source is forgotten.
const IDLE_TTL: Duration = Duration::from_secs(600);
/// Time after which a dead source's score is halved.
const DECAY_HALF_LIFE: Duration = Duration::from_secs(1800);
/// Minimum interval between two automatic sweep steps.
const SWEEP_INTERVAL: Duration = Duration::from_secs(60);
/// Number of independently locked score shards.
const SHARDS: usize = 64;
/// Shards examined by one [`IpReputation::sweep`] call.
const SWEEP_SHARDS_PER_TICK: usize = 8;
/// Upper bound of a single offender batch.
const MAX_OFFENDERS_PER_CYCLE: usize = 64;
const GOOD_STEP: i32 = 1;
const BAD_STEP: i32 = -10;
const MAX_SCORE: i32 = 100;
const MIN_SCORE: i32 = -100;
/// Tunables of [`IpReputation`].
#[derive(Debug, Clone, Copy)]
pub struct ReputationPolicy {
/// Idle age after which a source is forgotten.
pub idle_ttl: Duration,
/// Time after which a dead source's score is halved.
pub decay_half_life: Duration,
/// Upper bound of a single [`IpReputation::offenders_below`] batch.
pub max_offenders: usize,
}
impl Default for ReputationPolicy {
fn default() -> Self {
Self {
idle_ttl: IDLE_TTL,
decay_half_life: DECAY_HALF_LIFE,
max_offenders: MAX_OFFENDERS_PER_CYCLE,
}
}
}
/// Reputation entry: the raw score and when it was last touched.
struct Entry {
score: i32,
last_update: Instant,
}
/// Reputation scores of the sources seen by the edge node.
pub struct IpReputation {
scores: Arc<DashMap<IpAddr, i32>>,
shards: Box<[Mutex<HashMap<IpAddr, Entry>>]>,
hasher: RandomState,
policy: ReputationPolicy,
epoch: Instant,
last_sweep: AtomicU64,
sweep_cursor: AtomicUsize,
}
impl Default for IpReputation {
@ -13,129 +83,166 @@ impl Default for IpReputation {
}
impl IpReputation {
/// Builds a reputation table with the production policy.
#[must_use]
pub fn new() -> Self {
Self::with_policy(ReputationPolicy::default())
}
/// Builds a reputation table with an explicit policy.
pub fn with_policy(policy: ReputationPolicy) -> Self {
Self {
scores: Arc::new(DashMap::new()),
shards: (0..SHARDS).map(|_| Mutex::new(HashMap::new())).collect(),
hasher: RandomState::new(),
policy,
epoch: Instant::now(),
last_sweep: AtomicU64::new(0),
sweep_cursor: AtomicUsize::new(0),
}
}
/// Records a successful connection.
pub fn record_good(&self, ip: IpAddr) {
let mut entry = self.scores.entry(ip).or_insert(0);
*entry = (*entry + 1).min(100);
self.sweep_if_due();
self.adjust(ip, GOOD_STEP);
}
/// Records a failed or abusive connection.
pub fn record_bad(&self, ip: IpAddr) {
let mut entry = self.scores.entry(ip).or_insert(0);
*entry = (*entry - 10).max(-100);
self.sweep_if_due();
self.adjust(ip, BAD_STEP);
}
/// Current decayed score of `ip`; `0` for unknown or idle sources.
pub fn score(&self, ip: IpAddr) -> i32 {
self.scores.get(&ip).map(|v| *v).unwrap_or(0)
let now = Instant::now();
let index = self.shard_index(ip);
let shard = self.lock(index);
let Some(entry) = shard.get(&ip) else { return 0 };
let idle = now.duration_since(entry.last_update);
if idle > self.policy.idle_ttl {
return 0;
}
decayed_score(entry.score, idle, self.policy.decay_half_life)
}
/// Returns the sources scored at or below `threshold` (the offenders).
/// Returns up to [`ReputationPolicy::max_offenders`] of the worst sources
/// scored at or below `threshold`.
///
/// The result is a snapshot copy, so it is safe to ban from it while other
/// threads keep recording events.
/// The snapshot is safe to ban from while other threads keep recording
/// events. When more offenders exist than the batch cap, exactly one
/// aggregated warning carries the total count.
pub fn offenders_below(&self, threshold: i32) -> Vec<IpAddr> {
self.scores
.iter()
.filter(|entry| *entry.value() <= threshold)
.map(|entry| *entry.key())
.collect()
let now = Instant::now();
let limit = self.policy.max_offenders;
let mut worst: BinaryHeap<(i32, IpAddr)> = BinaryHeap::new();
let mut found = 0usize;
for index in 0..self.shards.len() {
let shard = self.lock(index);
for (ip, entry) in shard.iter() {
let idle = now.duration_since(entry.last_update);
if idle > self.policy.idle_ttl {
continue;
}
let score = decayed_score(entry.score, idle, self.policy.decay_half_life);
if score > threshold {
continue;
}
found += 1;
worst.push((score, *ip));
if worst.len() > limit {
worst.pop();
}
}
}
let offenders: Vec<IpAddr> = worst.into_iter().map(|(_, ip)| ip).collect();
if found > offenders.len() {
tracing::warn!(
found,
returned = offenders.len(),
"reputation offender batch truncated for this cycle"
);
}
offenders
}
/// Whether the source is above the trusted score.
pub fn is_trusted(&self, ip: IpAddr) -> bool {
self.score(ip) > 50
}
/// Number of live entries.
pub fn len(&self) -> usize {
(0..self.shards.len()).map(|index| self.lock(index).len()).sum()
}
/// Whether the table is empty.
pub fn is_empty(&self) -> bool {
self.len() == 0
}
/// Evicts idle entries from the next shard slice; returns the count.
pub fn sweep(&self) -> usize {
let now = Instant::now();
let shard_count = self.shards.len();
let start = self.sweep_cursor.fetch_add(SWEEP_SHARDS_PER_TICK, Ordering::Relaxed);
let mut removed = 0;
for offset in 0..SWEEP_SHARDS_PER_TICK.min(shard_count) {
let index = (start + offset) % shard_count;
let mut shard = self.lock(index);
let before = shard.len();
shard.retain(|_, entry| now.duration_since(entry.last_update) <= self.policy.idle_ttl);
removed += before - shard.len();
}
removed
}
fn adjust(&self, ip: IpAddr, delta: i32) {
let now = Instant::now();
let index = self.shard_index(ip);
let mut shard = self.lock(index);
let entry = shard.entry(ip).or_insert(Entry {
score: 0,
last_update: now,
});
let idle = now.duration_since(entry.last_update);
let decayed = decayed_score(entry.score, idle, self.policy.decay_half_life);
entry.score = (decayed + delta).clamp(MIN_SCORE, MAX_SCORE);
entry.last_update = now;
}
/// Advances one sweep step when [`SWEEP_INTERVAL`] has elapsed, so eviction
/// does not depend on an external timer.
fn sweep_if_due(&self) {
let elapsed = self.epoch.elapsed().as_secs();
let last = self.last_sweep.load(Ordering::Relaxed);
if elapsed.saturating_sub(last) < SWEEP_INTERVAL.as_secs() {
return;
}
let advanced = self
.last_sweep
.compare_exchange(last, elapsed, Ordering::AcqRel, Ordering::Relaxed);
if advanced.is_ok() {
self.sweep();
}
}
fn shard_index(&self, ip: IpAddr) -> usize {
(self.hasher.hash_one(ip) as usize) % self.shards.len()
}
fn lock(&self, index: usize) -> MutexGuard<'_, HashMap<IpAddr, Entry>> {
// A poisoned shard still holds consistent scores; keep serving.
self.shards[index].lock().unwrap_or_else(|poison| poison.into_inner())
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
fn v4(octets: [u8; 4]) -> IpAddr {
IpAddr::V4(Ipv4Addr::from(octets))
}
#[test]
fn test_reputation_initial_score() {
let rep = IpReputation::new();
assert_eq!(rep.score(v4([192, 168, 1, 1])), 0);
}
#[test]
fn test_reputation_good() {
let rep = IpReputation::new();
let ip = v4([10, 0, 0, 1]);
rep.record_good(ip);
assert_eq!(rep.score(ip), 1);
}
#[test]
fn test_reputation_bad() {
let rep = IpReputation::new();
let ip = v4([10, 0, 0, 2]);
rep.record_bad(ip);
assert_eq!(rep.score(ip), -10);
}
#[test]
fn test_reputation_cap_positive() {
let rep = IpReputation::new();
let ip = v4([10, 0, 0, 3]);
for _ in 0..200 {
rep.record_good(ip);
}
assert_eq!(rep.score(ip), 100);
}
#[test]
fn test_reputation_cap_negative() {
let rep = IpReputation::new();
let ip = v4([10, 0, 0, 4]);
for _ in 0..20 {
rep.record_bad(ip);
}
assert_eq!(rep.score(ip), -100);
}
#[test]
fn test_is_trusted() {
let rep = IpReputation::new();
let ip = v4([10, 0, 0, 5]);
assert!(!rep.is_trusted(ip));
for _ in 0..51 {
rep.record_good(ip);
}
assert!(rep.is_trusted(ip));
}
#[test]
fn test_reputation_v6() {
let rep = IpReputation::new();
let ip = IpAddr::V6(Ipv6Addr::LOCALHOST);
rep.record_bad(ip);
assert_eq!(rep.score(ip), -10);
}
#[test]
fn test_offenders_below_threshold() {
let rep = IpReputation::new();
let bad = v4([10, 0, 0, 1]);
let good = v4([10, 0, 0, 2]);
for _ in 0..6 {
rep.record_bad(bad);
}
rep.record_good(good);
let offenders = rep.offenders_below(-50);
assert_eq!(offenders, vec![bad]);
}
#[test]
fn test_offenders_empty_when_none_below() {
let rep = IpReputation::new();
assert!(rep.offenders_below(-50).is_empty());
fn decayed_score(score: i32, idle: Duration, half_life: Duration) -> i32 {
if score == 0 || half_life.is_zero() {
return 0;
}
let factor = 0.5f64.powf(idle.as_secs_f64() / half_life.as_secs_f64());
// Rounding (instead of truncation) keeps an immediate second update from
// decaying a score of `1` back to zero on the microsecond scale.
(f64::from(score) * factor).round() as i32
}

View file

@ -17,17 +17,22 @@ pub struct PrefixKey {
impl PrefixKey {
/// Builds the prefix key for an address (v4 maps to /24 in the first four
/// bytes, v6 maps to /64).
///
/// The address is normalised through [`prefix_of`] first, so an
/// IPv4-mapped IPv6 source (`::ffff:a.b.c.d`) produces exactly the same key
/// as the plain IPv4 address. XDP `prefix_stats` entries and the user-space
/// tracker therefore agree on the bucket of a dual-stack client.
#[must_use]
pub fn new(ip: IpAddr) -> Self {
let mut key = Self::default();
match ip {
match prefix_of(ip) {
IpAddr::V4(v4) => {
key.family = PREFIX_FAMILY_V4;
key.addr[..4].copy_from_slice(&masked_v4(v4).octets());
key.addr[..4].copy_from_slice(&v4.octets());
},
IpAddr::V6(v6) => {
key.family = PREFIX_FAMILY_V6;
key.addr.copy_from_slice(&masked_v6_segments(v6));
key.addr.copy_from_slice(&v6.octets());
},
}
key