fix(core): close auth bypass, hangs and memory leaks found by review
Security:
- An empty API_PASSWORD passed validation (only the literal "changeme" was
rejected) and verify_password("", "") returned true, so a login with an
empty password minted an admin token. Require >= 16 characters and >= 32
non-whitespace bytes for JWT_SECRET; verify rejects an empty stored secret.
- syn_challenge_enabled required a real challenge_secret_hex: the public
placeholder from config.h is now rejected in every spelling.
- docker-compose: Redis/ClickHouse/Grafana are loopback-only and every
password uses ${VAR:?} instead of a default. An exposed Redis was a way to
lift bans and publish unbans to the whole fleet with one redis-cli.
- Empty CORS_ORIGIN no longer means "allow any origin".
- ban_cidr / userspace blacklist validate and mask prefixes; CIDR bans are
stored as ranges, so a /24 or /64 ban actually covers the subnet (before,
only the network address was blocked).
Availability:
- ClickHouse client has connect/request timeouts and no mutex held across
await: a hung analytics backend can no longer freeze the ban path. Events
go through a bounded buffer with a drop counter.
- Redis uses a ConnectionManager with per-command timeouts; pubsub and
reconcile reset the backoff after success; publish cannot block shutdown.
- Ban durations from pubsub are clamped and use checked_add; the sync task
survives panics instead of silently dying.
- Blacklist TOCTOU (remove_if) and replace_redis no longer empties the table
during a Redis reconnect.
- RateLimiter and reputation have hard caps with incremental eviction instead
of an unbounded DashMap and a stop-the-world retain.
- The attack detector scores before updating its baseline and needs several
quiet windows to leave UnderAttack, so a sustained flood keeps banning new
offenders instead of ending after one tick.
- accept() backs off on EMFILE/ENFILE, shutdown Err is treated as shutdown,
a global semaphore bounds in-flight connections, rate limiting runs before
PoW, whitelisted IPs are never banned, and connect/handshake/relay have
timeouts with an idle budget.
- PoW uses the configured difficulty as a floor, the clamp is enforced, and
nonce framing survives split segments and coalesced handshake bytes.
- Subnet tracker memory is capped (prefixes and per-prefix sources) and the
monitor bans with the configured TTL.
CI and docs:
- CI runs the test suite with --no-default-features --features protocol-http;
make ebpf fails on errors; the secrets gate catches weak compose defaults.
- doctor skips protected endpoints without a token and exits non-zero on
failure; a drain timeout returns exit code 1.
- TODO §2b records the review findings, what is fixed and what is deferred;
configuration/testing docs describe the new XDP and secret semantics.
This commit is contained in:
parent
14dcaccdbf
commit
c1a076a4a3
57 changed files with 4193 additions and 1171 deletions
|
|
@ -132,6 +132,9 @@ pub async fn run() -> anyhow::Result<RunExit> {
|
|||
let subnet_tracker = start_subnet_tracker(&config);
|
||||
monitor::spawn(
|
||||
&config.detect.prefix,
|
||||
Duration::from_secs(config.ban.ban_duration_secs),
|
||||
blacklist.clone(),
|
||||
!config.proxy_protocol.ingress_enabled,
|
||||
subnet_tracker.clone(),
|
||||
xdp_filter.clone(),
|
||||
shutdown_rx.clone(),
|
||||
|
|
@ -190,7 +193,9 @@ pub async fn run() -> anyhow::Result<RunExit> {
|
|||
tracing::info!("Rampart edge starting on {}:{}", config.bind.address, config.bind.port);
|
||||
tracing::info!("Upstreams: {:?}", config.backend.upstreams);
|
||||
|
||||
let adjuster = Arc::new(Mutex::new(DifficultyAdjuster::default()));
|
||||
// The configured difficulty is the floor and the starting point; the
|
||||
// adjuster only raises it under load.
|
||||
let adjuster = Arc::new(Mutex::new(DifficultyAdjuster::with_floor(config.pow.difficulty)));
|
||||
let gateway = Arc::new(Gateway::new(
|
||||
config.clone(),
|
||||
rate_limiter,
|
||||
|
|
|
|||
|
|
@ -18,11 +18,14 @@ pub enum RunExit {
|
|||
|
||||
impl RunExit {
|
||||
/// Process exit code for this outcome.
|
||||
///
|
||||
/// A forced drain timeout loses connections that did not finish in time, so
|
||||
/// it must not look like a clean shutdown to supervisors and scripts.
|
||||
#[must_use]
|
||||
pub const fn code(self) -> i32 {
|
||||
match self {
|
||||
Self::Ok => 0,
|
||||
Self::DrainTimeout => 0,
|
||||
Self::DrainTimeout => 1,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -53,15 +56,3 @@ async fn wait_for_signal() {
|
|||
_ = term.recv() => {}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::RunExit;
|
||||
|
||||
#[test]
|
||||
fn run_exit_codes_match_previous_process_exit() {
|
||||
// Both the clean path and the former drain timeout (`process::exit(0)`) exited with code 0.
|
||||
assert_eq!(RunExit::Ok.code(), 0);
|
||||
assert_eq!(RunExit::DrainTimeout.code(), 0);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,17 +1,14 @@
|
|||
//! Rampart manager daemon: HTTP API in front of the redis state store.
|
||||
|
||||
use axum::{
|
||||
Router,
|
||||
http::HeaderValue,
|
||||
middleware,
|
||||
Router, middleware,
|
||||
routing::{get, post},
|
||||
};
|
||||
use dashmap::DashMap;
|
||||
#[cfg(feature = "store-redis")]
|
||||
use rampart::manager::sync;
|
||||
use rampart::manager::{AppState, api, auth};
|
||||
use rampart::manager::{AppState, api, auth, security};
|
||||
use std::sync::Arc;
|
||||
use tower_http::cors::CorsLayer;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
|
||||
#[tokio::main]
|
||||
|
|
@ -32,9 +29,7 @@ async fn main() -> anyhow::Result<()> {
|
|||
let redis_client = redis::Client::open(redis_url)?;
|
||||
|
||||
let jwt_secret = std::env::var("JWT_SECRET").map_err(|_| anyhow::anyhow!("JWT_SECRET must be set"))?;
|
||||
if jwt_secret.len() < 32 {
|
||||
return Err(anyhow::anyhow!("JWT_SECRET must be at least 32 bytes"));
|
||||
}
|
||||
security::validate_jwt_secret(&jwt_secret)?;
|
||||
let jwt_audience = std::env::var("JWT_AUDIENCE").unwrap_or_else(|_| "rampart".to_string());
|
||||
let jwt_expiration = std::env::var("JWT_EXPIRATION_SECS")
|
||||
.unwrap_or_else(|_| "86400".to_string())
|
||||
|
|
@ -42,9 +37,7 @@ async fn main() -> anyhow::Result<()> {
|
|||
.map_err(|_| anyhow::anyhow!("JWT_EXPIRATION_SECS must be a valid u64"))?;
|
||||
|
||||
let api_password = std::env::var("API_PASSWORD").map_err(|_| anyhow::anyhow!("API_PASSWORD must be set"))?;
|
||||
if api_password == "changeme" {
|
||||
return Err(anyhow::anyhow!("API_PASSWORD must not be the default 'changeme'"));
|
||||
}
|
||||
security::validate_api_password(&api_password)?;
|
||||
|
||||
let state = Arc::new(AppState {
|
||||
#[cfg(feature = "store-redis")]
|
||||
|
|
@ -78,17 +71,13 @@ async fn main() -> anyhow::Result<()> {
|
|||
.route("/api/v1/nodes", get(api::inventory::nodes::list_nodes));
|
||||
let protected = protected.route_layer(middleware::from_fn_with_state(state.clone(), auth::auth_middleware));
|
||||
|
||||
let cors = match std::env::var("CORS_ORIGIN") {
|
||||
Ok(origin) if origin.is_empty() || origin == "*" => CorsLayer::new().allow_origin(tower_http::cors::Any),
|
||||
Ok(origin) => CorsLayer::new().allow_origin(HeaderValue::from_str(&origin)?),
|
||||
Err(_) => CorsLayer::new().allow_origin(HeaderValue::from_static("http://localhost:5173")),
|
||||
};
|
||||
let cors_policy = security::CorsPolicy::parse(std::env::var("CORS_ORIGIN").ok().as_deref())?;
|
||||
tracing::info!("CORS policy: {cors_policy:?}");
|
||||
|
||||
let app = Router::new()
|
||||
.merge(public)
|
||||
.merge(protected)
|
||||
.layer(cors)
|
||||
.with_state(state);
|
||||
let mut app = Router::new().merge(public).merge(protected).with_state(state);
|
||||
if let Some(cors) = cors_policy.cors_layer() {
|
||||
app = app.layer(cors);
|
||||
}
|
||||
|
||||
let addr = "0.0.0.0:8080";
|
||||
tracing::info!("Manager API listening on {addr}");
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ pub async fn run(key: Option<String>, value: Option<String>) -> anyhow::Result<(
|
|||
println!();
|
||||
println!("Example keys:");
|
||||
println!(" workers.count");
|
||||
println!(" limits.rate_limit_login_pps");
|
||||
println!(" limits.rate_limit_pps");
|
||||
println!(" limits.max_connections_per_ip");
|
||||
Ok(())
|
||||
},
|
||||
|
|
|
|||
|
|
@ -2,6 +2,11 @@
|
|||
|
||||
use crate::xdp::{EnvironmentReport, FilesystemProbe, SystemProbe};
|
||||
|
||||
/// Environment variable holding a ready-to-use manager bearer token.
|
||||
const TOKEN_ENV: &str = "RAMPART_TOKEN";
|
||||
/// Environment variable holding the manager API password used to log in.
|
||||
const PASSWORD_ENV: &str = "RAMPART_API_PASSWORD";
|
||||
|
||||
/// Finds the interface that owns the default route in `/proc/net/route`
|
||||
/// (destination `00000000`). Works without root.
|
||||
fn default_route_interface(probe: &impl SystemProbe) -> Option<String> {
|
||||
|
|
@ -12,6 +17,51 @@ fn default_route_interface(probe: &impl SystemProbe) -> Option<String> {
|
|||
})
|
||||
}
|
||||
|
||||
/// Obtains a bearer token for the protected manager endpoints.
|
||||
///
|
||||
/// `RAMPART_TOKEN` is used as is; otherwise `RAMPART_API_PASSWORD` is exchanged
|
||||
/// for a token via `/api/v1/auth/login`. Protected endpoints are never called
|
||||
/// anonymously, so `None` means the caller has to be told to provide a
|
||||
/// credential instead of seeing a misleading failure.
|
||||
async fn manager_token(client: &reqwest::Client, manager_url: &str) -> Option<String> {
|
||||
if let Ok(token) = std::env::var(TOKEN_ENV) {
|
||||
if !token.trim().is_empty() {
|
||||
return Some(token);
|
||||
}
|
||||
}
|
||||
let password = std::env::var(PASSWORD_ENV).ok().filter(|value| !value.is_empty())?;
|
||||
let response = client
|
||||
.post(format!("{manager_url}/api/v1/auth/login"))
|
||||
.json(&serde_json::json!({"password": password}))
|
||||
.send()
|
||||
.await
|
||||
.ok()?;
|
||||
if !response.status().is_success() {
|
||||
println!("[WARN] Manager login rejected ({})", response.status());
|
||||
return None;
|
||||
}
|
||||
let body = response.json::<serde_json::Value>().await.ok()?;
|
||||
body["token"].as_str().map(str::to_owned)
|
||||
}
|
||||
|
||||
/// Probes one protected manager endpoint with a bearer token.
|
||||
async fn check_protected(client: &reqwest::Client, manager_url: &str, path: &str, label: &str, token: Option<&str>) {
|
||||
let Some(token) = token else {
|
||||
println!("[SKIP] {label}: set {TOKEN_ENV} or {PASSWORD_ENV} to check it");
|
||||
return;
|
||||
};
|
||||
match client
|
||||
.get(format!("{manager_url}{path}"))
|
||||
.bearer_auth(token)
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
Ok(resp) if resp.status().is_success() => println!("[OK] {label}"),
|
||||
Ok(resp) => println!("[WARN] {label} unavailable ({})", resp.status()),
|
||||
Err(e) => println!("[WARN] {label} unavailable: {e}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Network/XDP section: what the environment exposes without root.
|
||||
fn print_network_section(all_ok: &mut bool) {
|
||||
println!("\nNetwork / XDP");
|
||||
|
|
@ -60,6 +110,10 @@ fn print_network_section(all_ok: &mut bool) {
|
|||
}
|
||||
|
||||
/// Runs every diagnostic and prints the report.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error when any check failed, so scripts and CI see a non-zero
|
||||
/// exit status instead of a report that only looks alarming.
|
||||
pub async fn run() -> anyhow::Result<()> {
|
||||
println!("Rampart Diagnostics");
|
||||
println!("=====================\n");
|
||||
|
|
@ -67,8 +121,9 @@ pub async fn run() -> anyhow::Result<()> {
|
|||
let mut all_ok = true;
|
||||
|
||||
let manager_url = std::env::var("RAMPART_MANAGER").unwrap_or_else(|_| "http://localhost:8080".to_string());
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
match reqwest::get(format!("{manager_url}/api/v1/health")).await {
|
||||
match client.get(format!("{manager_url}/api/v1/health")).send().await {
|
||||
Ok(resp) if resp.status().is_success() => {
|
||||
println!("[OK] Manager API");
|
||||
},
|
||||
|
|
@ -78,32 +133,31 @@ pub async fn run() -> anyhow::Result<()> {
|
|||
},
|
||||
}
|
||||
|
||||
match reqwest::get(format!("{manager_url}/api/v1/blacklist")).await {
|
||||
Ok(resp) if resp.status().is_success() => {
|
||||
println!("[OK] Blacklist API");
|
||||
},
|
||||
_ => {
|
||||
println!("[WARN] Blacklist API unavailable");
|
||||
},
|
||||
}
|
||||
|
||||
match reqwest::get(format!("{manager_url}/api/v1/servers")).await {
|
||||
Ok(resp) if resp.status().is_success() => {
|
||||
println!("[OK] Servers API");
|
||||
},
|
||||
_ => {
|
||||
println!("[WARN] Servers API unavailable");
|
||||
},
|
||||
}
|
||||
let token = manager_token(&client, &manager_url).await;
|
||||
check_protected(
|
||||
&client,
|
||||
&manager_url,
|
||||
"/api/v1/blacklist",
|
||||
"Blacklist API",
|
||||
token.as_deref(),
|
||||
)
|
||||
.await;
|
||||
check_protected(
|
||||
&client,
|
||||
&manager_url,
|
||||
"/api/v1/servers",
|
||||
"Servers API",
|
||||
token.as_deref(),
|
||||
)
|
||||
.await;
|
||||
|
||||
print_network_section(&mut all_ok);
|
||||
|
||||
println!();
|
||||
if all_ok {
|
||||
println!("All checks passed.");
|
||||
} else {
|
||||
if !all_ok {
|
||||
println!("Some checks failed. Run with --verbose for details.");
|
||||
anyhow::bail!("doctor: one or more checks failed");
|
||||
}
|
||||
|
||||
println!("All checks passed.");
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,6 +19,14 @@ pub use sections::{
|
|||
use serde::Deserialize;
|
||||
use std::fs;
|
||||
|
||||
/// Upper bound of `ban.ban_duration_secs`: one year.
|
||||
const MAX_BAN_DURATION_SECS: u64 = 31_536_000;
|
||||
/// Upper bound of `pow.difficulty`.
|
||||
const MAX_POW_DIFFICULTY: u8 = 10;
|
||||
/// Placeholder `xdp.challenge_secret_hex` values baked into the XDP defaults
|
||||
/// (`xdp/core/config.h` and `XdpGlobals::default()`; both spellings exist).
|
||||
const PLACEHOLDER_CHALLENGE_SECRETS: [&str; 2] = ["a5a5a5a5a5a5a5a5", "a5a55a5a5a5aa5a5"];
|
||||
|
||||
#[derive(Debug, Clone, Default, Deserialize)]
|
||||
pub struct Config {
|
||||
#[serde(default)]
|
||||
|
|
@ -90,9 +98,38 @@ impl Config {
|
|||
if self.detect.prefix.enabled && self.detect.prefix.syn_threshold == 0 {
|
||||
anyhow::bail!("detect.prefix.syn_threshold must be positive when enabled");
|
||||
}
|
||||
if self.xdp.syn_challenge_enabled {
|
||||
let secret = self.xdp.challenge_secret_hex.as_deref().ok_or_else(|| {
|
||||
anyhow::anyhow!(
|
||||
"xdp.syn_challenge_enabled requires xdp.challenge_secret_hex; \
|
||||
generate one with `openssl rand -hex 8`"
|
||||
)
|
||||
})?;
|
||||
if is_placeholder_challenge_secret(secret) {
|
||||
anyhow::bail!(
|
||||
"xdp.challenge_secret_hex is the built-in placeholder; \
|
||||
generate a random secret with `openssl rand -hex 8`"
|
||||
);
|
||||
}
|
||||
}
|
||||
if self.limits.max_connections_per_ip == 0 {
|
||||
anyhow::bail!("limits.max_connections_per_ip must be positive");
|
||||
}
|
||||
if !self.limits.rate_limit_pps.is_finite() || self.limits.rate_limit_pps <= 0.0 {
|
||||
anyhow::bail!("limits.rate_limit_pps must be a finite number greater than 0");
|
||||
}
|
||||
if !self.limits.rate_limit_burst.is_finite() || self.limits.rate_limit_burst < 1.0 {
|
||||
anyhow::bail!("limits.rate_limit_burst must be a finite number at least 1");
|
||||
}
|
||||
if self.ban.ban_duration_secs == 0 || self.ban.ban_duration_secs > MAX_BAN_DURATION_SECS {
|
||||
anyhow::bail!("ban.ban_duration_secs must be between 1 and {MAX_BAN_DURATION_SECS}");
|
||||
}
|
||||
if self.pow.difficulty == 0 || self.pow.difficulty > MAX_POW_DIFFICULTY {
|
||||
anyhow::bail!("pow.difficulty must be between 1 and {MAX_POW_DIFFICULTY}");
|
||||
}
|
||||
if self.workers.count == 0 {
|
||||
anyhow::bail!("workers.count must be at least 1: otherwise no accept loops are started");
|
||||
}
|
||||
if self.proxy_protocol.ingress_enabled {
|
||||
if self.proxy_protocol.trusted_cidrs.is_empty() {
|
||||
anyhow::bail!("proxy_protocol.ingress_enabled requires at least one trusted_cidrs entry");
|
||||
|
|
@ -138,109 +175,17 @@ impl Config {
|
|||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn empty_config_uses_defaults() {
|
||||
let config = Config::parse_str("").expect("empty config should parse");
|
||||
assert_eq!(config.bind.address, "0.0.0.0");
|
||||
assert_eq!(config.bind.port, 25565);
|
||||
assert_eq!(config.backend.upstreams, vec!["127.0.0.1:25566"]);
|
||||
assert!(!config.pow.enabled);
|
||||
assert!(!config.xdp.enabled);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn full_config_parses() {
|
||||
let config = Config::parse_str(
|
||||
r#"
|
||||
[bind]
|
||||
address = "10.0.0.1"
|
||||
port = 443
|
||||
|
||||
[backend]
|
||||
upstreams = ["192.168.1.10:8080", "192.168.1.11:8080"]
|
||||
|
||||
[pow]
|
||||
enabled = true
|
||||
difficulty = 6
|
||||
|
||||
whitelist = ["203.0.113.7"]
|
||||
"#,
|
||||
)
|
||||
.expect("config should parse");
|
||||
assert_eq!(config.bind.port, 443);
|
||||
assert_eq!(config.backend.upstreams.len(), 2);
|
||||
assert!(config.pow.enabled);
|
||||
assert_eq!(config.pow.difficulty, 6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_upstreams_rejected() {
|
||||
let result = Config::parse_str(
|
||||
r#"
|
||||
[backend]
|
||||
upstreams = []
|
||||
"#,
|
||||
);
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_upstream_rejected() {
|
||||
let result = Config::parse_str(
|
||||
r#"
|
||||
[backend]
|
||||
upstreams = ["not-an-addr"]
|
||||
"#,
|
||||
);
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_whitelist_entry_rejected() {
|
||||
let result = Config::parse_str("whitelist = [\"999.999.1.1\"]");
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prefix_detect_defaults_disabled() {
|
||||
let config = Config::parse_str("").expect("empty config should parse");
|
||||
assert!(!config.detect.prefix.enabled);
|
||||
assert_eq!(config.detect.prefix.syn_threshold, 500);
|
||||
assert_eq!(config.detect.prefix.window_secs, 10);
|
||||
assert_eq!(config.detect.prefix.min_unique_sources, 16);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prefix_detect_section_parses() {
|
||||
let config = Config::parse_str(
|
||||
r#"
|
||||
[detect.prefix]
|
||||
enabled = true
|
||||
syn_threshold = 1000
|
||||
window_secs = 5
|
||||
min_unique_sources = 8
|
||||
"#,
|
||||
)
|
||||
.expect("detect.prefix section should parse");
|
||||
assert!(config.detect.prefix.enabled);
|
||||
assert_eq!(config.detect.prefix.syn_threshold, 1000);
|
||||
assert_eq!(config.detect.prefix.window_secs, 5);
|
||||
assert_eq!(config.detect.prefix.min_unique_sources, 8);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prefix_detect_zero_threshold_rejected_when_enabled() {
|
||||
let result = Config::parse_str(
|
||||
r#"
|
||||
[detect.prefix]
|
||||
enabled = true
|
||||
syn_threshold = 0
|
||||
"#,
|
||||
);
|
||||
assert!(result.is_err());
|
||||
}
|
||||
/// Returns whether `secret` spells a built-in placeholder challenge secret.
|
||||
///
|
||||
/// Hex digits are compared case-insensitively, with an optional `0x` prefix and
|
||||
/// `_`, `:` and `-` separators ignored, so every spelling of the placeholder is
|
||||
/// caught.
|
||||
fn is_placeholder_challenge_secret(secret: &str) -> bool {
|
||||
let normalized = secret
|
||||
.trim()
|
||||
.trim_start_matches("0x")
|
||||
.trim_start_matches("0X")
|
||||
.replace(['_', ':', '-'], "")
|
||||
.to_ascii_lowercase();
|
||||
PLACEHOLDER_CHALLENGE_SECRETS.contains(&normalized.as_str())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,10 +4,20 @@ use crate::metrics;
|
|||
use std::collections::VecDeque;
|
||||
use std::time::Instant;
|
||||
|
||||
/// Lowest usable difficulty: 0 disables the PoW gate before the adjuster runs.
|
||||
pub const MIN_DIFFICULTY: u8 = 1;
|
||||
|
||||
/// Highest adaptive difficulty. Every extra hex digit quadruples solver work.
|
||||
pub const MAX_DIFFICULTY: u8 = 12;
|
||||
|
||||
/// Difficulty the adjuster climbs to under the heaviest observed load.
|
||||
const DEFAULT_MAX_DIFFICULTY: u8 = 10;
|
||||
|
||||
/// Adapts the PoW difficulty to the current connection rate.
|
||||
///
|
||||
/// The rate is counted over a sliding one-second window and the resulting
|
||||
/// difficulty is clamped to `[min, max]`.
|
||||
/// difficulty is clamped to `[min, max]`. The lower bound is also the floor and
|
||||
/// the starting difficulty, so a configured value takes effect immediately.
|
||||
pub struct DifficultyAdjuster {
|
||||
window: VecDeque<Instant>,
|
||||
min: u8,
|
||||
|
|
@ -22,17 +32,36 @@ impl Default for DifficultyAdjuster {
|
|||
}
|
||||
|
||||
impl DifficultyAdjuster {
|
||||
/// Creates an adjuster clamped to `min.max(4)` and `max.min(10)`.
|
||||
/// Creates an adjuster with the explicit `[min, max]` range.
|
||||
///
|
||||
/// Inverted bounds are swapped, and both ends are clamped to
|
||||
/// [`MIN_DIFFICULTY`]..=[`MAX_DIFFICULTY`]. The lower bound doubles as the
|
||||
/// starting difficulty.
|
||||
#[must_use]
|
||||
pub fn new(min: u8, max: u8) -> Self {
|
||||
let (min, max) = if min > max { (max, min) } else { (min, max) };
|
||||
let min = min.clamp(MIN_DIFFICULTY, MAX_DIFFICULTY);
|
||||
let max = max.clamp(min, MAX_DIFFICULTY);
|
||||
Self {
|
||||
window: VecDeque::new(),
|
||||
min: min.max(4),
|
||||
max: max.min(10),
|
||||
current: min.max(4),
|
||||
min,
|
||||
max,
|
||||
current: min,
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates an adjuster whose floor and starting point is the configured PoW
|
||||
/// difficulty.
|
||||
///
|
||||
/// The ceiling stays at the default maximum, or rises to the configured
|
||||
/// value when that value is higher. Values outside
|
||||
/// [`MIN_DIFFICULTY`]..=[`MAX_DIFFICULTY`] are clamped.
|
||||
#[must_use]
|
||||
pub fn with_floor(difficulty: u8) -> Self {
|
||||
let floor = difficulty.clamp(MIN_DIFFICULTY, MAX_DIFFICULTY);
|
||||
Self::new(floor, floor.max(DEFAULT_MAX_DIFFICULTY))
|
||||
}
|
||||
|
||||
/// Records one connection and recomputes the difficulty for the window.
|
||||
pub fn record_connection(&mut self) {
|
||||
let now = Instant::now();
|
||||
|
|
@ -64,33 +93,18 @@ impl DifficultyAdjuster {
|
|||
self.current
|
||||
}
|
||||
|
||||
fn compute_difficulty(&self) -> u8 {
|
||||
/// Rate step before it is clamped to the configured range.
|
||||
fn rate_difficulty(&self) -> u8 {
|
||||
match self.window.len() {
|
||||
cps if cps > 500 => self.max.max(self.min),
|
||||
cps if cps > 500 => MAX_DIFFICULTY,
|
||||
cps if cps > 200 => 8,
|
||||
cps if cps > 50 => 6,
|
||||
_ => self.min,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn adjuster_raises_difficulty_under_load() {
|
||||
let mut adjuster = DifficultyAdjuster::new(4, 10);
|
||||
for _ in 0..600 {
|
||||
adjuster.record_connection();
|
||||
}
|
||||
assert_eq!(adjuster.current_difficulty(), 10);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn adjuster_stays_minimal_when_idle() {
|
||||
let mut adjuster = DifficultyAdjuster::new(4, 10);
|
||||
adjuster.record_connection();
|
||||
assert_eq!(adjuster.current_difficulty(), 4);
|
||||
fn compute_difficulty(&self) -> u8 {
|
||||
// The rate ladder is global; the configured range is authoritative.
|
||||
self.rate_difficulty().clamp(self.min, self.max)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,5 +4,5 @@
|
|||
mod difficulty;
|
||||
mod pow;
|
||||
|
||||
pub use difficulty::DifficultyAdjuster;
|
||||
pub use difficulty::{DifficultyAdjuster, MAX_DIFFICULTY, MIN_DIFFICULTY};
|
||||
pub use pow::{Challenge, enforce, solve};
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
//! Universal SHA-256 hashcash challenge generation, solver and verifier.
|
||||
|
||||
use rand::Rng;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::net::IpAddr;
|
||||
|
|
@ -5,11 +7,15 @@ use std::time::{Duration, Instant};
|
|||
use subtle::ConstantTimeEq;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpStream;
|
||||
use tokio::time::timeout;
|
||||
|
||||
const CHALLENGE_TTL_SECS: u64 = 30;
|
||||
const MAX_NONCE_LEN: usize = 64;
|
||||
const ALLOWED_HEX_PREFIX: [u8; 4] = *b"0123";
|
||||
|
||||
/// Maximum time a peer gets to read the challenge and return a nonce.
|
||||
const NONCE_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
|
||||
/// Active PoW challenge issued to one connection.
|
||||
pub struct Challenge {
|
||||
token: [u8; 32],
|
||||
|
|
@ -32,7 +38,9 @@ impl Challenge {
|
|||
}
|
||||
|
||||
/// Builds a challenge with an explicit token.
|
||||
///
|
||||
/// Intended for deterministic tests, where a random token makes the outcome
|
||||
/// hard to predict.
|
||||
#[must_use]
|
||||
pub fn with_token(token: [u8; 32], difficulty: u8) -> Self {
|
||||
Self {
|
||||
|
|
@ -104,8 +112,14 @@ pub fn solve(challenge: &str, difficulty: u8) -> Option<String> {
|
|||
/// Runs the textual PoW gate on `stream`: issues a challenge and checks the
|
||||
/// answer.
|
||||
///
|
||||
/// Wire format: the server writes the hex challenge followed by `\n`; the
|
||||
/// client answers with the nonce followed by `\n`. A reply terminated by EOF is
|
||||
/// also accepted for compatibility with clients that close their write half
|
||||
/// after sending the nonce. Anything the client sent after the delimiter stays
|
||||
/// in the socket buffer and is seen by the protocol handshake that runs next.
|
||||
///
|
||||
/// # Errors
|
||||
/// Only I/O failures are reported; a wrong solution is `Ok(false)`.
|
||||
/// Only I/O failures are reported; a wrong or missing solution is `Ok(false)`.
|
||||
pub async fn enforce(stream: &mut TcpStream, peer_ip: IpAddr, difficulty: u8) -> anyhow::Result<bool> {
|
||||
if difficulty == 0 {
|
||||
tracing::debug!("pow: difficulty 0, skipping for {peer_ip}");
|
||||
|
|
@ -114,22 +128,21 @@ pub async fn enforce(stream: &mut TcpStream, peer_ip: IpAddr, difficulty: u8) ->
|
|||
|
||||
let mut challenge = Challenge::generate(difficulty);
|
||||
let line = format!("{}\n", challenge.challenge_string());
|
||||
stream.write_all(line.as_bytes()).await?;
|
||||
timeout(NONCE_TIMEOUT, stream.write_all(line.as_bytes())).await??;
|
||||
|
||||
let mut buf = [0u8; MAX_NONCE_LEN + 1];
|
||||
let n = tokio::time::timeout(Duration::from_secs(10), stream.read(&mut buf)).await??;
|
||||
if n == 0 {
|
||||
tracing::debug!("pow: no response from {peer_ip}");
|
||||
let nonce = match timeout(NONCE_TIMEOUT, read_nonce_line(stream)).await {
|
||||
Ok(result) => result?,
|
||||
Err(_) => {
|
||||
tracing::debug!("pow: nonce read timed out for {peer_ip}");
|
||||
return Ok(false);
|
||||
},
|
||||
};
|
||||
let Some(nonce) = nonce else {
|
||||
tracing::debug!("pow: invalid or missing nonce from {peer_ip}");
|
||||
return Ok(false);
|
||||
}
|
||||
};
|
||||
|
||||
let nonce = std::str::from_utf8(&buf[..n.min(MAX_NONCE_LEN)]).unwrap_or("").trim();
|
||||
if nonce.is_empty() || nonce.len() > MAX_NONCE_LEN {
|
||||
tracing::debug!("pow: invalid nonce from {peer_ip}");
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let valid = challenge.verify(nonce);
|
||||
let valid = challenge.verify(&nonce);
|
||||
tracing::debug!(
|
||||
"pow: verification {} for {peer_ip}",
|
||||
if valid { "passed" } else { "failed" }
|
||||
|
|
@ -137,54 +150,43 @@ pub async fn enforce(stream: &mut TcpStream, peer_ip: IpAddr, difficulty: u8) ->
|
|||
Ok(valid)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn solve_then_verify_roundtrip() {
|
||||
let mut challenge = Challenge::generate(2);
|
||||
let nonce = solve(&challenge.challenge_string(), 2).expect("solver should find a nonce");
|
||||
assert!(challenge.verify(&nonce), "verifier must accept solver output");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verify_rejects_wrong_nonce() {
|
||||
let mut challenge = Challenge::generate(4);
|
||||
let challenge_str = challenge.challenge_string();
|
||||
|
||||
// Search for a nonce that an independently computed SHA-256 provably rejects,
|
||||
// so the test stays deterministic.
|
||||
let mut wrong_nonce = None;
|
||||
for i in 0..10_000u32 {
|
||||
let candidate = format!("bad-{i}");
|
||||
let hex_hash = hex::encode(Sha256::digest(format!("{challenge_str}{candidate}").as_bytes()));
|
||||
if !hex_hash
|
||||
.as_bytes()
|
||||
.iter()
|
||||
.take(4)
|
||||
.all(|c| ALLOWED_HEX_PREFIX.contains(c))
|
||||
{
|
||||
wrong_nonce = Some(candidate);
|
||||
break;
|
||||
}
|
||||
/// Reads a newline-terminated nonce without consuming the bytes that follow the
|
||||
/// delimiter.
|
||||
///
|
||||
/// `peek` inspects the buffer first; only the nonce and its `\n` are consumed, so
|
||||
/// a client that coalesces the nonce and the handshake into one TCP segment is
|
||||
/// not truncated. A nonce split across segments is accumulated until the
|
||||
/// delimiter arrives or the connection reports EOF.
|
||||
///
|
||||
/// Returns `None` for an empty, oversized or non-UTF-8 reply.
|
||||
async fn read_nonce_line(stream: &mut TcpStream) -> anyhow::Result<Option<String>> {
|
||||
let mut buf = [0u8; MAX_NONCE_LEN + 1];
|
||||
let mut nonce = Vec::with_capacity(MAX_NONCE_LEN);
|
||||
loop {
|
||||
let n = stream.peek(&mut buf).await?;
|
||||
if n == 0 {
|
||||
// EOF without a delimiter: accept the bytes received so far.
|
||||
break;
|
||||
}
|
||||
let wrong_nonce = wrong_nonce.expect("a failing nonce must exist among candidates");
|
||||
assert!(!challenge.verify(&wrong_nonce));
|
||||
if let Some(pos) = buf[..n].iter().position(|&b| b == b'\n') {
|
||||
if nonce.len() + pos > MAX_NONCE_LEN {
|
||||
return Ok(None);
|
||||
}
|
||||
nonce.extend_from_slice(&buf[..pos]);
|
||||
// Consume exactly the nonce line; the handshake bytes stay buffered.
|
||||
stream.read_exact(&mut buf[..=pos]).await?;
|
||||
break;
|
||||
}
|
||||
if nonce.len() + n > MAX_NONCE_LEN {
|
||||
return Ok(None);
|
||||
}
|
||||
stream.read_exact(&mut buf[..n]).await?;
|
||||
nonce.extend_from_slice(&buf[..n]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verify_rejects_replay() {
|
||||
let mut challenge = Challenge::generate(2);
|
||||
let nonce = solve(&challenge.challenge_string(), 2).expect("solved");
|
||||
assert!(challenge.verify(&nonce));
|
||||
assert!(!challenge.verify(&nonce), "challenge must be single-use");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verify_rejects_oversized_nonce() {
|
||||
let mut challenge = Challenge::generate(2);
|
||||
let big_nonce = "0".repeat(MAX_NONCE_LEN + 1);
|
||||
assert!(!challenge.verify(&big_nonce));
|
||||
let text = std::str::from_utf8(&nonce).unwrap_or("").trim();
|
||||
if text.is_empty() || text.len() > MAX_NONCE_LEN {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(text.to_string()))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,10 +4,107 @@ use crate::config::Config;
|
|||
use crate::engine::tunnel::Gateway;
|
||||
use crate::traffic::hook::TrafficHook;
|
||||
use socket2::{Domain, Socket, Type};
|
||||
use std::io::ErrorKind;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::sync::watch;
|
||||
|
||||
const ACCEPT_BACKOFF_MIN: Duration = Duration::from_millis(50);
|
||||
const ACCEPT_BACKOFF_MAX: Duration = Duration::from_millis(500);
|
||||
/// Consecutive accept failures between full-rate ERROR logs.
|
||||
///
|
||||
/// Every intervening failure is still reported at `debug`, so the log stays
|
||||
/// diagnosable without amplifying a spin into a disk-filling storm.
|
||||
const ACCEPT_ERROR_LOG_EVERY: u32 = 32;
|
||||
|
||||
/// Linux errno values that mean "the listener is temporarily unable to accept".
|
||||
///
|
||||
/// `libc` is an optional dependency of the `xdp` feature, so the values are
|
||||
/// spelled out here; they are ABI constants on the platforms Rampart targets.
|
||||
#[cfg(target_os = "linux")]
|
||||
mod errno {
|
||||
pub const ENOMEM: i32 = 12;
|
||||
pub const ENFILE: i32 = 23;
|
||||
pub const EMFILE: i32 = 24;
|
||||
pub const ENOBUFS: i32 = 105;
|
||||
}
|
||||
|
||||
/// Exponential backoff for transient `accept()` failures.
|
||||
///
|
||||
/// File-descriptor exhaustion (`EMFILE`/`ENFILE`) makes `accept()` fail
|
||||
/// synchronously and forever: a plain `continue` turns every worker into a
|
||||
/// 100% CPU spin that also floods the log. Each transient failure therefore
|
||||
/// yields a mandatory sleep, growing from `ACCEPT_BACKOFF_MIN` to
|
||||
/// `ACCEPT_BACKOFF_MAX` and resetting after the next successful accept.
|
||||
pub struct AcceptBackoff {
|
||||
delay: Duration,
|
||||
failures: u32,
|
||||
}
|
||||
|
||||
impl Default for AcceptBackoff {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl AcceptBackoff {
|
||||
#[must_use]
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
delay: Duration::ZERO,
|
||||
failures: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns how long the accept loop must sleep before retrying `error`.
|
||||
///
|
||||
/// `None` means the error is not a capacity or readiness condition
|
||||
/// (`ECONNABORTED`, `EPROTO`, ...): the connection was abandoned by the peer
|
||||
/// or the kernel, so an immediate retry is correct.
|
||||
pub fn delay(&mut self, error: &std::io::Error) -> Option<Duration> {
|
||||
if !is_transient(error) {
|
||||
return None;
|
||||
}
|
||||
self.failures = self.failures.saturating_add(1);
|
||||
self.delay = if self.delay.is_zero() {
|
||||
ACCEPT_BACKOFF_MIN
|
||||
} else {
|
||||
(self.delay * 2).min(ACCEPT_BACKOFF_MAX)
|
||||
};
|
||||
Some(self.delay)
|
||||
}
|
||||
|
||||
/// Clears the accumulated backoff after a successful accept.
|
||||
pub fn reset(&mut self) {
|
||||
self.delay = Duration::ZERO;
|
||||
self.failures = 0;
|
||||
}
|
||||
|
||||
/// Reports whether the current failure deserves a full-rate ERROR log.
|
||||
#[must_use]
|
||||
pub fn should_log(&self) -> bool {
|
||||
self.failures == 1 || self.failures % ACCEPT_ERROR_LOG_EVERY == 0
|
||||
}
|
||||
}
|
||||
|
||||
fn is_transient(error: &std::io::Error) -> bool {
|
||||
if matches!(
|
||||
error.kind(),
|
||||
ErrorKind::WouldBlock | ErrorKind::Interrupted | ErrorKind::OutOfMemory
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
#[cfg(target_os = "linux")]
|
||||
if matches!(
|
||||
error.raw_os_error(),
|
||||
Some(errno::EMFILE | errno::ENFILE | errno::ENOBUFS | errno::ENOMEM)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Starts `workers.count` accept loops on a single address.
|
||||
///
|
||||
/// Each worker binds its own socket with `SO_REUSEPORT` and `SO_REUSEADDR`, so
|
||||
|
|
@ -58,22 +155,45 @@ async fn accept_loop(
|
|||
hook: Arc<TrafficHook>,
|
||||
mut shutdown: watch::Receiver<bool>,
|
||||
) {
|
||||
let mut backoff = AcceptBackoff::new();
|
||||
loop {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = shutdown.changed() => {
|
||||
if *shutdown.borrow() {
|
||||
tracing::info!("shutdown signal received, stopping accept loop");
|
||||
return;
|
||||
changed = shutdown.changed() => {
|
||||
match changed {
|
||||
Ok(()) if *shutdown.borrow() => {
|
||||
tracing::info!("shutdown signal received, stopping accept loop");
|
||||
return;
|
||||
},
|
||||
// An update that does not raise the flag: keep serving.
|
||||
Ok(()) => {},
|
||||
// The sender is gone. `changed()` would return `Err` immediately
|
||||
// and forever, so treat it as shutdown instead of spinning.
|
||||
Err(_) => {
|
||||
tracing::info!("shutdown channel closed, stopping accept loop");
|
||||
return;
|
||||
},
|
||||
}
|
||||
}
|
||||
result = listener.accept() => {
|
||||
let (stream, peer_addr) = match result {
|
||||
Ok(conn) => conn,
|
||||
Ok(conn) => {
|
||||
backoff.reset();
|
||||
conn
|
||||
},
|
||||
Err(e) => {
|
||||
tracing::error!("accept error: {e}");
|
||||
if let Some(delay) = backoff.delay(&e) {
|
||||
if backoff.should_log() {
|
||||
tracing::error!("accept error, backing off {delay:?}: {e}");
|
||||
} else {
|
||||
tracing::debug!("accept error, backing off {delay:?}: {e}");
|
||||
}
|
||||
tokio::time::sleep(delay).await;
|
||||
} else {
|
||||
tracing::debug!("accept error: {e}");
|
||||
}
|
||||
continue;
|
||||
}
|
||||
},
|
||||
};
|
||||
let gateway = Arc::clone(&gateway);
|
||||
let hook = Arc::clone(&hook);
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
|
||||
use crate::config::DetectPrefixConfig;
|
||||
use crate::engine::subnet_tracker::{SubnetTracker, TrackerSource};
|
||||
use crate::filter::blacklist::Blacklist;
|
||||
use crate::metrics;
|
||||
#[cfg(feature = "xdp")]
|
||||
use crate::traffic::prefix::XdpPrefixStats;
|
||||
|
|
@ -30,6 +31,22 @@ impl Source {
|
|||
Source::Tracker(tracker) => tracker.snapshot().await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether the source reports distinct source addresses per prefix.
|
||||
fn counts_unique_sources(&self) -> bool {
|
||||
match self {
|
||||
#[cfg(feature = "xdp")]
|
||||
Source::Xdp(_) => false,
|
||||
Source::Tracker(_) => true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Verdict enforcement settings shared by every snapshot tick.
|
||||
struct BanPolicy {
|
||||
blacklist: Arc<Blacklist>,
|
||||
ttl: Duration,
|
||||
trusted: bool,
|
||||
}
|
||||
|
||||
/// Spawns the background subnet monitor when `[detect.prefix] enabled`.
|
||||
|
|
@ -37,8 +54,16 @@ impl Source {
|
|||
/// Source priority: the XDP map when a filter is loaded, otherwise the userspace
|
||||
/// aggregator. Without either source the monitor stays disabled and logs a
|
||||
/// warning.
|
||||
///
|
||||
/// `ban_ttl` comes from `[ban].ban_duration_secs`. `trusted_peer` is false when
|
||||
/// the observed address comes from an unverified ingress path (inbound PROXY
|
||||
/// protocol), where banning it could punish the transport proxy instead of the
|
||||
/// real client.
|
||||
pub fn spawn(
|
||||
detect: &DetectPrefixConfig,
|
||||
ban_ttl: Duration,
|
||||
blacklist: Arc<Blacklist>,
|
||||
trusted_peer: bool,
|
||||
tracker: Option<Arc<SubnetTracker>>,
|
||||
xdp: Option<Arc<Mutex<XdpFilter>>>,
|
||||
mut shutdown: watch::Receiver<bool>,
|
||||
|
|
@ -52,6 +77,12 @@ pub fn spawn(
|
|||
};
|
||||
let detector = SubnetDetector::from_config(detect);
|
||||
let window = Duration::from_secs(detect.window_secs.max(1));
|
||||
let policy = BanPolicy {
|
||||
blacklist,
|
||||
ttl: ban_ttl,
|
||||
trusted: trusted_peer,
|
||||
};
|
||||
let extreme_threshold = detect.syn_threshold.saturating_mul(4);
|
||||
|
||||
tokio::spawn(async move {
|
||||
let mut tick = tokio::time::interval(window);
|
||||
|
|
@ -66,7 +97,10 @@ pub fn spawn(
|
|||
}
|
||||
_ = tick.tick() => {
|
||||
match source.snapshot().await {
|
||||
Ok(snaps) => apply_verdicts(&detector.evaluate(&snaps), &source).await,
|
||||
Ok(snaps) => {
|
||||
let verdicts = detector.evaluate(&snaps);
|
||||
apply_verdicts(&verdicts, &source, &policy, extreme_threshold);
|
||||
},
|
||||
Err(e) => tracing::debug!("prefix stats unavailable: {e}"),
|
||||
}
|
||||
}
|
||||
|
|
@ -85,7 +119,7 @@ fn build_source(tracker: Option<Arc<SubnetTracker>>, xdp: Option<Arc<Mutex<XdpFi
|
|||
tracker.map(|t| Source::Tracker(TrackerSource { tracker: t }))
|
||||
}
|
||||
|
||||
async fn apply_verdicts(verdicts: &[PrefixVerdict], source: &Source) {
|
||||
fn apply_verdicts(verdicts: &[PrefixVerdict], source: &Source, policy: &BanPolicy, extreme_threshold: u64) {
|
||||
for pv in verdicts {
|
||||
metrics::SUBNET_VERDICTS
|
||||
.with_label_values(&[verdict_name(pv.verdict)])
|
||||
|
|
@ -97,35 +131,87 @@ async fn apply_verdicts(verdicts: &[PrefixVerdict], source: &Source) {
|
|||
verdict = verdict_name(pv.verdict),
|
||||
"subnet verdict"
|
||||
);
|
||||
if pv.verdict == SubnetVerdict::Block {
|
||||
enforce_block(source, pv).await;
|
||||
if pv.verdict == SubnetVerdict::Challenge
|
||||
&& !source.counts_unique_sources()
|
||||
&& pv.snapshot.syn_count >= extreme_threshold
|
||||
{
|
||||
// XDP does not count distinct sources, so the spoof gate can never
|
||||
// confirm `Block` on that path; make the cap visible instead of silent.
|
||||
tracing::info!(
|
||||
prefix = %pv.prefix,
|
||||
syn = pv.snapshot.syn_count,
|
||||
"spoof gate suppressed block: XDP has no source diversity data; escalating at L7 only"
|
||||
);
|
||||
}
|
||||
}
|
||||
match source {
|
||||
#[cfg(feature = "xdp")]
|
||||
Source::Xdp(xdp) if policy.trusted => ban_xdp_verdicts(xdp, verdicts, policy.ttl),
|
||||
#[cfg(feature = "xdp")]
|
||||
Source::Xdp(_) => {
|
||||
if verdicts.iter().any(|pv| pv.verdict == SubnetVerdict::Block) {
|
||||
tracing::warn!("subnet kernel bans suppressed: peer identity comes from an untrusted ingress path");
|
||||
}
|
||||
},
|
||||
Source::Tracker(_) => {
|
||||
enforce_tracker_verdicts(verdicts, &policy.blacklist, policy.ttl, policy.trusted);
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Applies userspace `Block` verdicts to the shared blacklist.
|
||||
///
|
||||
/// Returns the number of banned prefixes. When `trusted` is false the observed
|
||||
/// addresses belong to an unverified ingress path, so the verdict is only logged
|
||||
/// at warn level and no ban is installed.
|
||||
///
|
||||
/// The ban is inserted as a CIDR range, so every address inside the detected
|
||||
/// `/24` or `/64` is blocked, not just its network address.
|
||||
pub fn enforce_tracker_verdicts(
|
||||
verdicts: &[PrefixVerdict],
|
||||
blacklist: &Blacklist,
|
||||
ttl: Duration,
|
||||
trusted: bool,
|
||||
) -> usize {
|
||||
let mut banned = 0;
|
||||
for pv in verdicts.iter().filter(|pv| pv.verdict == SubnetVerdict::Block) {
|
||||
if !trusted {
|
||||
tracing::warn!(
|
||||
prefix = %pv.prefix,
|
||||
"subnet block verdict on an untrusted peer path; not adding to the blacklist"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
blacklist.add_cidr(pv.prefix, prefix_len(pv.prefix), ttl, "subnet_attack");
|
||||
banned += 1;
|
||||
tracing::warn!(
|
||||
prefix = %pv.prefix,
|
||||
ttl_secs = ttl.as_secs(),
|
||||
"subnet blocked in the userspace blacklist"
|
||||
);
|
||||
}
|
||||
banned
|
||||
}
|
||||
|
||||
#[cfg(feature = "xdp")]
|
||||
fn ban_xdp_verdicts(xdp: &XdpPrefixStats, verdicts: &[PrefixVerdict], ttl: Duration) {
|
||||
for pv in verdicts.iter().filter(|pv| pv.verdict == SubnetVerdict::Block) {
|
||||
let result = {
|
||||
let guard = xdp.filter_lock();
|
||||
guard.ban_cidr(pv.prefix, prefix_len(pv.prefix), ttl.as_secs())
|
||||
};
|
||||
match result {
|
||||
Ok(()) => tracing::warn!(
|
||||
prefix = %pv.prefix,
|
||||
ttl_secs = ttl.as_secs(),
|
||||
"subnet blocked in XDP"
|
||||
),
|
||||
Err(e) => tracing::warn!("subnet block failed for {}: {e}", pv.prefix),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn enforce_block(source: &Source, pv: &PrefixVerdict) {
|
||||
match source {
|
||||
#[cfg(feature = "xdp")]
|
||||
Source::Xdp(xdp) => {
|
||||
let result = {
|
||||
let guard = xdp.filter_lock();
|
||||
guard.ban_cidr(pv.prefix, prefix_len(pv.prefix), 3600)
|
||||
};
|
||||
if let Err(e) = result {
|
||||
tracing::warn!("subnet block failed for {}: {e}", pv.prefix);
|
||||
}
|
||||
},
|
||||
Source::Tracker(_) => {
|
||||
tracing::warn!(
|
||||
prefix = %pv.prefix,
|
||||
"block verdict without kernel enforcement; escalate at L7"
|
||||
);
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Prefix length matching the detection granularity (v4 /24, v6 /64).
|
||||
#[cfg(feature = "xdp")]
|
||||
fn prefix_len(prefix: IpAddr) -> u8 {
|
||||
match prefix {
|
||||
IpAddr::V4(_) => 24,
|
||||
|
|
|
|||
|
|
@ -4,25 +4,49 @@
|
|||
//! Fallback path of the subnet detector when XDP is disabled: it feeds
|
||||
//! [`crate::traffic::prefix::SubnetDetector`] with the same snapshot shape the
|
||||
//! kernel map produces.
|
||||
//!
|
||||
//! Memory is bounded on purpose. Both the prefix table and the per-prefix
|
||||
//! source set have hard caps, so a source rotating through a /48 (or a /64 with
|
||||
//! millions of addresses) cannot grow the tracker without limit during one
|
||||
//! detection window; over-cap events are counted and reported.
|
||||
|
||||
use crate::traffic::prefix::{PrefixSnapshot, PrefixStatsSource, prefix_of};
|
||||
use dashmap::DashMap;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::net::IpAddr;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Instant;
|
||||
|
||||
/// Upper bound on tracked prefixes per window.
|
||||
pub const DEFAULT_MAX_PREFIXES: usize = 4096;
|
||||
|
||||
/// Upper bound on distinct sources remembered per prefix.
|
||||
///
|
||||
/// 256 is well above the default `min_unique_sources = 16` spoof gate while
|
||||
/// keeping the worst-case table at a few megabytes.
|
||||
pub const DEFAULT_MAX_SOURCES_PER_PREFIX: usize = 256;
|
||||
|
||||
struct WindowState {
|
||||
syn_count: u64,
|
||||
sources: std::collections::HashSet<IpAddr>,
|
||||
sources: HashSet<IpAddr>,
|
||||
started: Instant,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct Limits {
|
||||
max_prefixes: usize,
|
||||
max_sources: usize,
|
||||
}
|
||||
|
||||
/// Aggregates new connections per prefix.
|
||||
///
|
||||
/// `take_snapshot` drains the accumulated state, so every call closes exactly
|
||||
/// one detection window.
|
||||
pub struct SubnetTracker {
|
||||
windows: DashMap<IpAddr, WindowState>,
|
||||
windows: Mutex<HashMap<IpAddr, WindowState>>,
|
||||
limits: Limits,
|
||||
dropped_prefixes: AtomicU64,
|
||||
dropped_sources: AtomicU64,
|
||||
}
|
||||
|
||||
impl Default for SubnetTracker {
|
||||
|
|
@ -34,47 +58,108 @@ impl Default for SubnetTracker {
|
|||
impl SubnetTracker {
|
||||
#[must_use]
|
||||
pub fn new() -> Self {
|
||||
Self::with_limits(DEFAULT_MAX_PREFIXES, DEFAULT_MAX_SOURCES_PER_PREFIX)
|
||||
}
|
||||
|
||||
/// Creates a tracker with explicit caps.
|
||||
///
|
||||
/// Both limits are clamped to at least one entry. The constructor mainly
|
||||
/// lets tests exercise the caps without building thousands of prefixes.
|
||||
#[must_use]
|
||||
pub fn with_limits(max_prefixes: usize, max_sources_per_prefix: usize) -> Self {
|
||||
Self {
|
||||
windows: DashMap::new(),
|
||||
windows: Mutex::new(HashMap::new()),
|
||||
limits: Limits {
|
||||
max_prefixes: max_prefixes.max(1),
|
||||
max_sources: max_sources_per_prefix.max(1),
|
||||
},
|
||||
dropped_prefixes: AtomicU64::new(0),
|
||||
dropped_sources: AtomicU64::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
/// Records a new inbound connection (userspace equivalent of a SYN).
|
||||
///
|
||||
/// Call it only after the source passed the blacklist check, so banned
|
||||
/// sources cannot allocate tracker memory.
|
||||
pub fn record_connection(&self, ip: IpAddr) {
|
||||
let prefix = prefix_of(ip);
|
||||
let mut state = self.windows.entry(prefix).or_insert_with(|| WindowState {
|
||||
syn_count: 0,
|
||||
sources: std::collections::HashSet::new(),
|
||||
started: Instant::now(),
|
||||
});
|
||||
state.syn_count = state.syn_count.saturating_add(1);
|
||||
state.sources.insert(ip);
|
||||
let mut windows = self.windows.lock().expect("subnet tracker lock poisoned");
|
||||
if let Some(state) = windows.get_mut(&prefix) {
|
||||
state.syn_count = state.syn_count.saturating_add(1);
|
||||
Self::remember_source(state, ip, self.limits.max_sources, &self.dropped_sources);
|
||||
return;
|
||||
}
|
||||
if windows.len() >= self.limits.max_prefixes {
|
||||
self.dropped_prefixes.fetch_add(1, Ordering::Relaxed);
|
||||
return;
|
||||
}
|
||||
let mut sources = HashSet::new();
|
||||
sources.insert(ip);
|
||||
windows.insert(
|
||||
prefix,
|
||||
WindowState {
|
||||
syn_count: 1,
|
||||
sources,
|
||||
started: Instant::now(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/// Takes and resets the aggregates of the current window.
|
||||
#[must_use]
|
||||
pub fn take_snapshot(&self) -> Vec<(IpAddr, PrefixSnapshot)> {
|
||||
let mut out = Vec::new();
|
||||
// Drain every key: the next window starts from an empty map, so memory does not
|
||||
// grow when an attacker spoofs many distinct prefixes.
|
||||
self.windows.retain(|prefix, state| {
|
||||
if state.syn_count == 0 {
|
||||
return false;
|
||||
}
|
||||
out.push((
|
||||
*prefix,
|
||||
PrefixSnapshot {
|
||||
// Userspace only sees established connections, so `pkt_count` mirrors the same
|
||||
// flow as `syn_count`.
|
||||
pkt_count: state.syn_count,
|
||||
syn_count: state.syn_count,
|
||||
last_seen_ns: state.started.elapsed().as_nanos() as u64,
|
||||
unique_sources: state.sources.len() as u64,
|
||||
},
|
||||
));
|
||||
false
|
||||
});
|
||||
out
|
||||
let windows = std::mem::take(&mut *self.windows.lock().expect("subnet tracker lock poisoned"));
|
||||
let dropped_prefixes = self.dropped_prefixes.load(Ordering::Relaxed);
|
||||
let dropped_sources = self.dropped_sources.load(Ordering::Relaxed);
|
||||
if dropped_prefixes > 0 || dropped_sources > 0 {
|
||||
tracing::debug!(
|
||||
dropped_prefixes,
|
||||
dropped_sources,
|
||||
"subnet tracker caps reached; aggregates are bounded"
|
||||
);
|
||||
}
|
||||
windows
|
||||
.into_iter()
|
||||
.filter(|(_, state)| state.syn_count > 0)
|
||||
.map(|(prefix, state)| {
|
||||
(
|
||||
prefix,
|
||||
PrefixSnapshot {
|
||||
// Userspace only sees established connections, so `pkt_count` mirrors the
|
||||
// same flow as `syn_count`.
|
||||
pkt_count: state.syn_count,
|
||||
syn_count: state.syn_count,
|
||||
last_seen_ns: state.started.elapsed().as_nanos() as u64,
|
||||
unique_sources: state.sources.len() as u64,
|
||||
},
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Number of connections from new prefixes dropped because the prefix cap
|
||||
/// was reached.
|
||||
#[must_use]
|
||||
pub fn dropped_prefixes(&self) -> u64 {
|
||||
self.dropped_prefixes.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Number of connections from new source addresses dropped because the
|
||||
/// per-prefix source cap was reached.
|
||||
#[must_use]
|
||||
pub fn dropped_sources(&self) -> u64 {
|
||||
self.dropped_sources.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
fn remember_source(state: &mut WindowState, ip: IpAddr, cap: usize, dropped: &AtomicU64) {
|
||||
if state.sources.len() < cap {
|
||||
state.sources.insert(ip);
|
||||
return;
|
||||
}
|
||||
if !state.sources.contains(&ip) {
|
||||
dropped.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -88,48 +173,3 @@ impl PrefixStatsSource for TrackerSource {
|
|||
Ok(self.tracker.take_snapshot())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::net::{Ipv4Addr, Ipv6Addr};
|
||||
|
||||
fn v4(octets: [u8; 4]) -> IpAddr {
|
||||
IpAddr::V4(Ipv4Addr::from(octets))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn groups_connections_by_prefix() {
|
||||
let tracker = SubnetTracker::new();
|
||||
tracker.record_connection(v4([203, 0, 113, 1]));
|
||||
tracker.record_connection(v4([203, 0, 113, 2]));
|
||||
tracker.record_connection(v4([198, 51, 100, 9]));
|
||||
let snaps = tracker.take_snapshot();
|
||||
assert_eq!(snaps.len(), 2);
|
||||
let by_prefix: std::collections::HashMap<IpAddr, PrefixSnapshot> = snaps.into_iter().collect();
|
||||
assert_eq!(by_prefix[&v4([203, 0, 113, 0])].syn_count, 2);
|
||||
assert_eq!(by_prefix[&v4([203, 0, 113, 0])].unique_sources, 2);
|
||||
assert_eq!(by_prefix[&v4([198, 51, 100, 0])].syn_count, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn snapshot_drains_window() {
|
||||
let tracker = SubnetTracker::new();
|
||||
tracker.record_connection(v4([10, 0, 0, 1]));
|
||||
assert_eq!(tracker.take_snapshot().len(), 1);
|
||||
assert!(tracker.take_snapshot().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mapped_addresses_share_v4_prefix() {
|
||||
let tracker = SubnetTracker::new();
|
||||
tracker.record_connection(v4([203, 0, 113, 5]));
|
||||
// ::ffff:203.0.113.6
|
||||
let mapped = Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xcb00, 0x7106);
|
||||
tracker.record_connection(IpAddr::V6(mapped));
|
||||
let snaps = tracker.take_snapshot();
|
||||
assert_eq!(snaps.len(), 1);
|
||||
assert_eq!(snaps[0].0, v4([203, 0, 113, 0]));
|
||||
assert_eq!(snaps[0].1.unique_sources, 2);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,16 @@
|
|||
//! Per-connection pipeline of the edge engine: filters, PoW gate, protocol
|
||||
//! handshake and upstream relay.
|
||||
//!
|
||||
//! The module root owns gateway construction and the entry point; the stages
|
||||
//! live in `pipeline` (filters, PoW, handshake, upstream), `admission`
|
||||
//! (reputation escalation, bans and events) and `relay` (idle-bounded relay).
|
||||
|
||||
mod admission;
|
||||
mod challenge;
|
||||
mod pipeline;
|
||||
mod relay;
|
||||
|
||||
pub use relay::RELAY_IDLE_TIMEOUT;
|
||||
|
||||
#[cfg(feature = "protocol-minecraft")]
|
||||
use super::proxy;
|
||||
|
|
@ -12,17 +21,14 @@ use crate::engine::subnet::tracker::SubnetTracker;
|
|||
use crate::filter::blacklist::Blacklist;
|
||||
use crate::filter::connection_limit::ConnectionLimiter;
|
||||
use crate::filter::rate_limit::RateLimiter;
|
||||
use crate::metrics;
|
||||
use crate::protocol::ProtocolRegistry;
|
||||
use crate::store::clickhouse::{ClickHouseEvent, ClickHouseWriter};
|
||||
use crate::store::clickhouse::ClickHouseWriter;
|
||||
use crate::traffic::reputation::IpReputation;
|
||||
use crate::xdp::XdpFilter;
|
||||
use std::collections::HashSet;
|
||||
use std::net::IpAddr;
|
||||
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
|
||||
use std::sync::atomic::AtomicU64;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
use tokio::net::TcpStream;
|
||||
use tokio::sync::Mutex as TokioMutex;
|
||||
|
||||
|
|
@ -55,7 +61,7 @@ impl Gateway {
|
|||
allowed_1s,
|
||||
registry,
|
||||
subnet_tracker,
|
||||
upstream_cursor: AtomicUsize::new(0),
|
||||
upstream_cursor: std::sync::atomic::AtomicUsize::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -69,25 +75,7 @@ impl Gateway {
|
|||
let client_addr = if self.config.proxy_protocol.ingress_enabled {
|
||||
#[cfg(feature = "protocol-minecraft")]
|
||||
{
|
||||
if !self
|
||||
.config
|
||||
.proxy_protocol
|
||||
.trusted_cidrs
|
||||
.iter()
|
||||
.any(|cidr| cidr.contains(peer_addr.ip()))
|
||||
{
|
||||
return GatewayOutcome {
|
||||
client_ip: None,
|
||||
result: Err(anyhow::anyhow!("untrusted peer attempted inbound PROXY protocol")),
|
||||
};
|
||||
}
|
||||
match proxy::read_v2_header(
|
||||
&mut client,
|
||||
Duration::from_millis(self.config.proxy_protocol.ingress_timeout_ms),
|
||||
self.config.proxy_protocol.max_header_bytes,
|
||||
)
|
||||
.await
|
||||
{
|
||||
match proxy::resolve_client_addr(&self.config.proxy_protocol, &mut client, peer_addr).await {
|
||||
Ok(client_addr) => client_addr,
|
||||
Err(error) => {
|
||||
return GatewayOutcome {
|
||||
|
|
@ -114,122 +102,4 @@ impl Gateway {
|
|||
result,
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_client(
|
||||
&self,
|
||||
mut client: TcpStream,
|
||||
peer_addr: std::net::SocketAddr,
|
||||
client_addr: std::net::SocketAddr,
|
||||
) -> anyhow::Result<()> {
|
||||
let peer_ip = client_addr.ip();
|
||||
|
||||
if let Some(tracker) = &self.subnet_tracker {
|
||||
tracker.record_connection(peer_ip);
|
||||
}
|
||||
|
||||
if self.blacklist.is_blocked(peer_ip) {
|
||||
metrics::CONNECTIONS_TOTAL.with_label_values(&["blocked"]).inc();
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let Some(_connection_permit) = self.connection_limiter.try_acquire(peer_ip) else {
|
||||
self.block_rate_limit(peer_ip).await;
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
if !self.pass_pow_gate(&mut client, peer_ip).await? {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if !self.rate_limiter.check(peer_ip) {
|
||||
self.block_rate_limit(peer_ip).await;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let handler = self.registry.primary()?;
|
||||
let handshake = match handler.handshake(&mut client).await {
|
||||
Ok(handshake) => handshake,
|
||||
Err(e) => {
|
||||
tracing::debug!("handshake rejected from {peer_addr}: {e}");
|
||||
metrics::CONNECTIONS_TOTAL.with_label_values(&["blocked"]).inc();
|
||||
return Ok(());
|
||||
},
|
||||
};
|
||||
|
||||
metrics::CONNECTIONS_TOTAL.with_label_values(&["allowed"]).inc();
|
||||
self.allowed_1s.fetch_add(1, Ordering::Relaxed);
|
||||
self.reputation.record_good(peer_ip);
|
||||
|
||||
let mut backend = TcpStream::connect(self.next_upstream(&handshake.upstream.addr)).await?;
|
||||
#[cfg(feature = "protocol-minecraft")]
|
||||
if self.config.protocol.minecraft.enabled && self.config.protocol.minecraft.send_proxy_protocol {
|
||||
let source = client_addr;
|
||||
let destination = client.local_addr()?;
|
||||
backend.write_all(&proxy::protocol_v2(source, destination)?).await?;
|
||||
}
|
||||
if !handshake.initial_backend_data.is_empty() {
|
||||
backend.write_all(&handshake.initial_backend_data).await?;
|
||||
}
|
||||
tokio::io::copy_bidirectional(&mut client, &mut backend).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn next_upstream<'a>(&'a self, negotiated: &'a str) -> &'a str {
|
||||
let upstreams = &self.config.backend.upstreams;
|
||||
if upstreams.len() > 1 {
|
||||
let idx = self.upstream_cursor.fetch_add(1, Ordering::Relaxed) % upstreams.len();
|
||||
return upstreams[idx].as_str();
|
||||
}
|
||||
if let Some(first) = upstreams.first() {
|
||||
return first.as_str();
|
||||
}
|
||||
negotiated
|
||||
}
|
||||
|
||||
async fn block_rate_limit(&self, ip: IpAddr) {
|
||||
metrics::RATE_LIMIT_HITS.with_label_values(&["hit"]).inc();
|
||||
metrics::CONNECTIONS_TOTAL.with_label_values(&["blocked"]).inc();
|
||||
self.reputation.record_bad(ip);
|
||||
if self.reputation.score(ip) < -40 {
|
||||
let duration_secs = Duration::from_secs(self.config.ban.ban_duration_secs);
|
||||
self.blacklist.add(ip, duration_secs, "low_reputation");
|
||||
if !self.config.proxy_protocol.ingress_enabled {
|
||||
self.xdp_ban(ip, duration_secs.as_secs());
|
||||
}
|
||||
self.push_event("block", ip, "low_reputation").await;
|
||||
tracing::info!("low reputation ban {ip}: rate-limit abuse");
|
||||
}
|
||||
}
|
||||
|
||||
fn xdp_ban(&self, ip: IpAddr, duration_secs: u64) {
|
||||
let Some(xdp) = &self.xdp else {
|
||||
return;
|
||||
};
|
||||
let prefix_len = if ip.is_ipv4() { 32 } else { 128 };
|
||||
match xdp
|
||||
.lock()
|
||||
.expect("xdp lock poisoned")
|
||||
.ban_cidr(ip, prefix_len, duration_secs)
|
||||
{
|
||||
Ok(()) => tracing::debug!("xdp ban {ip} for {duration_secs}s"),
|
||||
Err(e) => tracing::warn!("xdp ban failed for {ip}: {e}"),
|
||||
}
|
||||
}
|
||||
|
||||
async fn push_event(&self, event_type: &str, ip: IpAddr, data_string: &str) {
|
||||
let Some(writer) = &self.clickhouse else {
|
||||
return;
|
||||
};
|
||||
let event = ClickHouseEvent {
|
||||
timestamp: chrono::Utc::now(),
|
||||
event_type: event_type.to_string(),
|
||||
ip: ip.to_string(),
|
||||
data_float: 0.0,
|
||||
data_int: 0,
|
||||
data_string: data_string.to_string(),
|
||||
};
|
||||
if let Err(e) = writer.lock().await.push(event).await {
|
||||
tracing::debug!("clickhouse push error: {e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
67
src/engine/tunnel/gateway/admission.rs
Normal file
67
src/engine/tunnel/gateway/admission.rs
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
//! Reputation escalation, bans and telemetry for dropped connections.
|
||||
|
||||
use super::Gateway;
|
||||
use crate::metrics;
|
||||
use crate::store::clickhouse::ClickHouseEvent;
|
||||
use std::net::IpAddr;
|
||||
use std::time::Duration;
|
||||
|
||||
impl Gateway {
|
||||
/// Accounts for a connection dropped by an admission filter and escalates to
|
||||
/// a ban when the source reputation is exhausted.
|
||||
///
|
||||
/// Whitelisted peers are exempt: a ban is replicated to Redis and XDP on
|
||||
/// every node, which would lock a trusted operator out of the whole cluster.
|
||||
pub(super) async fn block_rate_limit(&self, ip: IpAddr) {
|
||||
if self.whitelist.contains(&ip) {
|
||||
metrics::RATE_LIMIT_HITS.with_label_values(&["trusted"]).inc();
|
||||
return;
|
||||
}
|
||||
metrics::RATE_LIMIT_HITS.with_label_values(&["hit"]).inc();
|
||||
metrics::CONNECTIONS_TOTAL.with_label_values(&["blocked"]).inc();
|
||||
self.reputation.record_bad(ip);
|
||||
if self.reputation.score(ip) < -40 {
|
||||
let duration_secs = Duration::from_secs(self.config.ban.ban_duration_secs);
|
||||
self.blacklist.add(ip, duration_secs, "low_reputation");
|
||||
if !self.config.proxy_protocol.ingress_enabled {
|
||||
self.xdp_ban(ip, duration_secs.as_secs());
|
||||
}
|
||||
self.push_event("block", ip, "low_reputation").await;
|
||||
tracing::info!("low reputation ban {ip}: rate-limit abuse");
|
||||
}
|
||||
}
|
||||
|
||||
/// Mirrors a ban into the XDP map, if the kernel filter is attached.
|
||||
fn xdp_ban(&self, ip: IpAddr, duration_secs: u64) {
|
||||
let Some(xdp) = &self.xdp else {
|
||||
return;
|
||||
};
|
||||
let prefix_len = if ip.is_ipv4() { 32 } else { 128 };
|
||||
match xdp
|
||||
.lock()
|
||||
.expect("xdp lock poisoned")
|
||||
.ban_cidr(ip, prefix_len, duration_secs)
|
||||
{
|
||||
Ok(()) => tracing::debug!("xdp ban {ip} for {duration_secs}s"),
|
||||
Err(e) => tracing::warn!("xdp ban failed for {ip}: {e}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Forwards an event to ClickHouse when the writer is configured.
|
||||
async fn push_event(&self, event_type: &str, ip: IpAddr, data_string: &str) {
|
||||
let Some(writer) = &self.clickhouse else {
|
||||
return;
|
||||
};
|
||||
let event = ClickHouseEvent {
|
||||
timestamp: chrono::Utc::now(),
|
||||
event_type: event_type.to_string(),
|
||||
ip: ip.to_string(),
|
||||
data_float: 0.0,
|
||||
data_int: 0,
|
||||
data_string: data_string.to_string(),
|
||||
};
|
||||
if let Err(e) = writer.lock().await.push(event).await {
|
||||
tracing::debug!("clickhouse push error: {e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -31,6 +31,10 @@ impl Gateway {
|
|||
adjuster.record_connection();
|
||||
adjuster.current_difficulty()
|
||||
};
|
||||
// The configured difficulty is a hard floor even if the adjuster was
|
||||
// constructed without it; recording the connection above still adapts
|
||||
// the value for later peers.
|
||||
let diff = diff.max(pow_config.difficulty);
|
||||
let result = enforce_pow(client, peer_ip, diff).await?;
|
||||
if !result {
|
||||
metrics::POW_CHALLENGES_TOTAL.with_label_values(&["failed"]).inc();
|
||||
|
|
|
|||
208
src/engine/tunnel/gateway/pipeline.rs
Normal file
208
src/engine/tunnel/gateway/pipeline.rs
Normal file
|
|
@ -0,0 +1,208 @@
|
|||
//! Filter order and upstream establishment for one accepted connection.
|
||||
|
||||
use super::Gateway;
|
||||
use super::RELAY_IDLE_TIMEOUT;
|
||||
use crate::metrics;
|
||||
use std::sync::OnceLock;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::Duration;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
use tokio::net::TcpStream;
|
||||
use tokio::sync::Semaphore;
|
||||
|
||||
/// Multiplier applied to the per-IP limit when sizing the global connection
|
||||
/// budget: `4 x workers x max_connections_per_ip` in-flight connections.
|
||||
const GLOBAL_BUDGET_FACTOR: usize = 4;
|
||||
/// Ceiling of the global budget, so a large worker count cannot exhaust the
|
||||
/// file-descriptor table of the host.
|
||||
const GLOBAL_BUDGET_MAX: usize = 100_000;
|
||||
/// One in every `BUDGET_LOG_EVERY` budget rejections is logged.
|
||||
const BUDGET_LOG_EVERY: u64 = 128;
|
||||
|
||||
/// Process-wide in-flight connection budget.
|
||||
///
|
||||
/// Initialised on first use from the worker count and the per-IP limit of the
|
||||
/// gateway configuration. A process-wide value is intentional: the budget caps
|
||||
/// descriptors and tasks for the whole edge node, and there is exactly one
|
||||
/// gateway per process in production.
|
||||
static GLOBAL_BUDGET: OnceLock<Semaphore> = OnceLock::new();
|
||||
/// Total connections rejected by the global budget, used for log rate limiting.
|
||||
static BUDGET_REJECTIONS: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
impl Gateway {
|
||||
/// Returns the process-wide in-flight connection budget.
|
||||
fn global_budget(&self) -> &'static Semaphore {
|
||||
GLOBAL_BUDGET.get_or_init(|| {
|
||||
let workers = self.config.workers.count.max(1);
|
||||
let per_ip = self.config.limits.max_connections_per_ip.max(1) as usize;
|
||||
let capacity = workers
|
||||
.saturating_mul(per_ip)
|
||||
.saturating_mul(GLOBAL_BUDGET_FACTOR)
|
||||
.clamp(1, GLOBAL_BUDGET_MAX);
|
||||
Semaphore::new(capacity)
|
||||
})
|
||||
}
|
||||
|
||||
/// Runs the filter stages, PoW gate, protocol handshake and relay.
|
||||
///
|
||||
/// # Errors
|
||||
/// Only upstream I/O failures are surfaced; every filter drop is `Ok(())`.
|
||||
pub(super) async fn handle_client(
|
||||
&self,
|
||||
mut client: TcpStream,
|
||||
peer_addr: std::net::SocketAddr,
|
||||
client_addr: std::net::SocketAddr,
|
||||
) -> anyhow::Result<()> {
|
||||
let peer_ip = client_addr.ip();
|
||||
let trusted = self.whitelist.contains(&peer_ip);
|
||||
|
||||
// Trusted peers bypass every ingress filter and are never auto-banned:
|
||||
// blacklist entries and XDP/Redis bans propagate across all nodes, so a
|
||||
// whitelisted address must never enter that path.
|
||||
if !trusted {
|
||||
if self.blacklist.is_blocked(peer_ip) {
|
||||
metrics::CONNECTIONS_TOTAL.with_label_values(&["blocked"]).inc();
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Feed the subnet detector only after the ban decision: a source we
|
||||
// already block must not allocate tracker memory.
|
||||
if let Some(tracker) = &self.subnet_tracker {
|
||||
tracker.record_connection(peer_ip);
|
||||
}
|
||||
|
||||
// The cheap token check runs before the expensive PoW exchange: a
|
||||
// client that never solves the challenge must still exhaust its own
|
||||
// rate budget instead of holding a task for the PoW timeout unpaid.
|
||||
if !self.rate_limiter.check(peer_ip) {
|
||||
self.block_rate_limit(peer_ip).await;
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
|
||||
// Global budget: `try_acquire` keeps an overloaded node responsive (the
|
||||
// accept loop never queues unbounded per-connection tasks) and the
|
||||
// connection is closed immediately on refusal.
|
||||
let _global_permit = if trusted {
|
||||
None
|
||||
} else {
|
||||
match self.global_budget().try_acquire() {
|
||||
Ok(permit) => Some(permit),
|
||||
Err(_) => {
|
||||
metrics::CONNECTIONS_TOTAL.with_label_values(&["overloaded"]).inc();
|
||||
let seen = BUDGET_REJECTIONS.fetch_add(1, Ordering::Relaxed);
|
||||
if seen % BUDGET_LOG_EVERY == 0 {
|
||||
tracing::warn!(
|
||||
"global connection budget exhausted after {seen} rejections, dropping {peer_ip}"
|
||||
);
|
||||
}
|
||||
return Ok(());
|
||||
},
|
||||
}
|
||||
};
|
||||
|
||||
let Some(_connection_permit) = self.connection_limiter.try_acquire(peer_ip) else {
|
||||
if trusted {
|
||||
tracing::debug!("per-IP connection limit reached for trusted {peer_ip}");
|
||||
} else {
|
||||
self.block_rate_limit(peer_ip).await;
|
||||
}
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
if !self.pass_pow_gate(&mut client, peer_ip).await? {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let handshake_budget = Duration::from_secs(self.config.limits.handshake_timeout_secs.max(1));
|
||||
let handler = self.registry.primary()?;
|
||||
let handshake = match tokio::time::timeout(handshake_budget, handler.handshake(&mut client)).await {
|
||||
Ok(Ok(handshake)) => handshake,
|
||||
Ok(Err(e)) => {
|
||||
tracing::debug!("handshake rejected from {peer_addr}: {e}");
|
||||
metrics::CONNECTIONS_TOTAL.with_label_values(&["blocked"]).inc();
|
||||
return Ok(());
|
||||
},
|
||||
Err(_) => {
|
||||
tracing::debug!("handshake timed out after {handshake_budget:?} from {peer_addr}");
|
||||
metrics::CONNECTIONS_TOTAL.with_label_values(&["blocked"]).inc();
|
||||
return Ok(());
|
||||
},
|
||||
};
|
||||
|
||||
metrics::CONNECTIONS_TOTAL.with_label_values(&["allowed"]).inc();
|
||||
self.allowed_1s.fetch_add(1, Ordering::Relaxed);
|
||||
self.reputation.record_good(peer_ip);
|
||||
|
||||
let mut backend = self
|
||||
.connect_upstream(&handshake.upstream.addr, handshake_budget)
|
||||
.await?;
|
||||
self.send_preamble(
|
||||
&mut client,
|
||||
&mut backend,
|
||||
client_addr,
|
||||
&handshake.initial_backend_data,
|
||||
handshake_budget,
|
||||
)
|
||||
.await?;
|
||||
self.relay_with_idle(client, backend, RELAY_IDLE_TIMEOUT).await
|
||||
}
|
||||
|
||||
/// Opens the upstream connection under the handshake timeout.
|
||||
///
|
||||
/// # Errors
|
||||
/// Fails when the connect is refused or exceeds `budget`.
|
||||
async fn connect_upstream(&self, negotiated: &str, budget: Duration) -> anyhow::Result<TcpStream> {
|
||||
let upstream = self.next_upstream(negotiated);
|
||||
match tokio::time::timeout(budget, TcpStream::connect(upstream)).await {
|
||||
Ok(Ok(stream)) => Ok(stream),
|
||||
Ok(Err(e)) => Err(anyhow::anyhow!("upstream connect to {upstream} failed: {e}")),
|
||||
Err(_) => Err(anyhow::anyhow!(
|
||||
"upstream connect to {upstream} timed out after {budget:?}"
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Writes the PROXY protocol header and the protocol's replay bytes to the
|
||||
/// backend under the handshake timeout.
|
||||
///
|
||||
/// # Errors
|
||||
/// Fails on address resolution and on a write that exceeds `budget`.
|
||||
#[allow(unused_variables)]
|
||||
async fn send_preamble(
|
||||
&self,
|
||||
client: &mut TcpStream,
|
||||
backend: &mut TcpStream,
|
||||
client_addr: std::net::SocketAddr,
|
||||
initial_data: &[u8],
|
||||
budget: Duration,
|
||||
) -> anyhow::Result<()> {
|
||||
let mut preamble = Vec::new();
|
||||
#[cfg(feature = "protocol-minecraft")]
|
||||
if self.config.protocol.minecraft.enabled && self.config.protocol.minecraft.send_proxy_protocol {
|
||||
let destination = client.local_addr()?;
|
||||
preamble.extend_from_slice(&super::super::proxy::protocol_v2(client_addr, destination)?);
|
||||
}
|
||||
preamble.extend_from_slice(initial_data);
|
||||
if preamble.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
match tokio::time::timeout(budget, backend.write_all(&preamble)).await {
|
||||
Ok(Ok(())) => Ok(()),
|
||||
Ok(Err(e)) => Err(e.into()),
|
||||
Err(_) => Err(anyhow::anyhow!("upstream preamble write timed out after {budget:?}")),
|
||||
}
|
||||
}
|
||||
|
||||
fn next_upstream<'a>(&'a self, negotiated: &'a str) -> &'a str {
|
||||
let upstreams = &self.config.backend.upstreams;
|
||||
if upstreams.len() > 1 {
|
||||
let idx = self.upstream_cursor.fetch_add(1, Ordering::Relaxed) % upstreams.len();
|
||||
return upstreams[idx].as_str();
|
||||
}
|
||||
if let Some(first) = upstreams.first() {
|
||||
return first.as_str();
|
||||
}
|
||||
negotiated
|
||||
}
|
||||
}
|
||||
75
src/engine/tunnel/gateway/relay.rs
Normal file
75
src/engine/tunnel/gateway/relay.rs
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
//! Idle-aware bidirectional relay between the accepted client and the backend.
|
||||
|
||||
use super::Gateway;
|
||||
use std::io;
|
||||
use std::time::Duration;
|
||||
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
|
||||
use tokio::net::TcpStream;
|
||||
|
||||
const RELAY_BUFFER_BYTES: usize = 16 * 1024;
|
||||
|
||||
/// Idle budget of the client/backend relay.
|
||||
///
|
||||
/// Minecraft keeps one connection alive for a whole session and a legitimately
|
||||
/// silent gap (menus, chunk loading, AFK timeouts) can last minutes, so 300s is
|
||||
/// the safe default: it never cuts a real session and still reaps dead peers.
|
||||
/// The value is a constant because `[limits]` is owned by a parallel change; it
|
||||
/// is plumbed as a parameter through [`Gateway::relay_with_idle`] so a config
|
||||
/// field can be wired in without touching the relay itself.
|
||||
pub const RELAY_IDLE_TIMEOUT: Duration = Duration::from_secs(300);
|
||||
|
||||
impl Gateway {
|
||||
/// Relays bytes in both directions until either side closes, an I/O error
|
||||
/// occurs or a direction stays silent for `idle`.
|
||||
///
|
||||
/// The idle budget restarts on every successful read/write, so a long-lived
|
||||
/// session with steady traffic is never cut, while a dead or half-open peer
|
||||
/// is reaped instead of pinning two descriptors forever. Both directions run
|
||||
/// under one `select!`: when one pump finishes the other is dropped and both
|
||||
/// sockets close, preserving the cancellation semantics of
|
||||
/// `copy_bidirectional`.
|
||||
///
|
||||
/// `idle` is a parameter rather than a constant so tests can exercise the
|
||||
/// timeout with a short budget and per-upstream tuning can override it.
|
||||
///
|
||||
/// # Errors
|
||||
/// Fails on the first I/O error and on an idle timeout; a clean half-close is
|
||||
/// `Ok`.
|
||||
pub async fn relay_with_idle(&self, client: TcpStream, backend: TcpStream, idle: Duration) -> anyhow::Result<()> {
|
||||
let (client_read, mut client_write) = client.into_split();
|
||||
let (backend_read, mut backend_write) = backend.into_split();
|
||||
tokio::select! {
|
||||
result = pump(client_read, &mut backend_write, idle) => result?,
|
||||
result = pump(backend_read, &mut client_write, idle) => result?,
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Copies one direction, bounded by `idle` between chunks.
|
||||
///
|
||||
/// # Errors
|
||||
/// Fails on a read/write error and when a direction makes no progress within
|
||||
/// `idle`.
|
||||
async fn pump<R, W>(mut source: R, mut sink: W, idle: Duration) -> io::Result<()>
|
||||
where
|
||||
R: AsyncRead + Unpin,
|
||||
W: AsyncWrite + Unpin,
|
||||
{
|
||||
let mut buffer = vec![0u8; RELAY_BUFFER_BYTES];
|
||||
loop {
|
||||
let read = tokio::time::timeout(idle, source.read(&mut buffer))
|
||||
.await
|
||||
.map_err(|_| idle_error())??;
|
||||
if read == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
tokio::time::timeout(idle, sink.write_all(&buffer[..read]))
|
||||
.await
|
||||
.map_err(|_| idle_error())??;
|
||||
}
|
||||
}
|
||||
|
||||
fn idle_error() -> io::Error {
|
||||
io::Error::new(io::ErrorKind::TimedOut, "relay idle timeout")
|
||||
}
|
||||
|
|
@ -1,10 +1,38 @@
|
|||
//! Encoding of upstream PROXY protocol v2 headers.
|
||||
//! PROXY protocol v2 support: header encoding for the backend and strict
|
||||
//! parsing of headers injected by trusted ingress peers.
|
||||
|
||||
use crate::config::ProxyProtocolConfig;
|
||||
use std::net::SocketAddr;
|
||||
use std::time::Duration;
|
||||
use tokio::net::TcpStream;
|
||||
|
||||
mod inbound;
|
||||
pub(super) use inbound::read_v2_header;
|
||||
|
||||
/// Resolves the effective client address from an inbound PROXY v2 header.
|
||||
///
|
||||
/// Only peers inside `config.trusted_cidrs` may inject a header; anyone else
|
||||
/// is rejected instead of being believed. The whole header must arrive within
|
||||
/// `config.ingress_timeout_ms` and fit in `config.max_header_bytes`.
|
||||
///
|
||||
/// # Errors
|
||||
/// Fails when the peer is not trusted, on timeout and on any header parse error.
|
||||
pub(super) async fn resolve_client_addr(
|
||||
config: &ProxyProtocolConfig,
|
||||
client: &mut TcpStream,
|
||||
peer_addr: SocketAddr,
|
||||
) -> anyhow::Result<SocketAddr> {
|
||||
if !config.trusted_cidrs.iter().any(|cidr| cidr.contains(peer_addr.ip())) {
|
||||
anyhow::bail!("untrusted peer attempted inbound PROXY protocol");
|
||||
}
|
||||
read_v2_header(
|
||||
client,
|
||||
Duration::from_millis(config.ingress_timeout_ms),
|
||||
config.max_header_bytes,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Encodes a PROXY protocol v2 TCP header for the backend.
|
||||
///
|
||||
/// The backend must accept connections only from trusted Rampart nodes, since
|
||||
|
|
|
|||
|
|
@ -1,4 +1,15 @@
|
|||
//! In-memory ban tables.
|
||||
//!
|
||||
//! Two tables are kept: bans detected locally and bans mirrored from Redis.
|
||||
//! Expired entries are removed lazily on lookup and periodically by
|
||||
//! [`Blacklist::clear_expired`]. Deadlines are monotonic and clamped, so a
|
||||
//! hostile expiry can never overflow or panic.
|
||||
|
||||
mod prefix;
|
||||
|
||||
use self::prefix::PrefixBans;
|
||||
use dashmap::DashMap;
|
||||
use std::collections::HashSet;
|
||||
use std::net::IpAddr;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
|
@ -6,9 +17,14 @@ use tokio::sync::broadcast;
|
|||
|
||||
const LOCAL_BAN_EVENT_CAPACITY: usize = 4096;
|
||||
|
||||
/// Upper bound for a single ban; longer requests are clamped.
|
||||
pub const MAX_BAN_DURATION_SECS: u64 = 31_536_000;
|
||||
|
||||
/// A locally detected ban, forwarded to the Redis publisher.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct LocalBanEvent {
|
||||
pub ip: IpAddr,
|
||||
/// Wall-clock expiry as Unix seconds.
|
||||
pub expires_at_unix: i64,
|
||||
pub reason: String,
|
||||
}
|
||||
|
|
@ -18,9 +34,11 @@ struct BanEntry {
|
|||
_reason: String,
|
||||
}
|
||||
|
||||
/// Combined view of local and Redis-mirrored bans.
|
||||
pub struct Blacklist {
|
||||
entries: Arc<DashMap<IpAddr, BanEntry>>,
|
||||
redis_entries: Arc<DashMap<IpAddr, BanEntry>>,
|
||||
prefixes: PrefixBans,
|
||||
local_ban_events: broadcast::Sender<LocalBanEvent>,
|
||||
}
|
||||
|
||||
|
|
@ -31,29 +49,53 @@ impl Default for Blacklist {
|
|||
}
|
||||
|
||||
impl Blacklist {
|
||||
/// Creates an empty blacklist.
|
||||
pub fn new() -> Self {
|
||||
let (local_ban_events, _) = broadcast::channel(LOCAL_BAN_EVENT_CAPACITY);
|
||||
Self {
|
||||
entries: Arc::new(DashMap::new()),
|
||||
redis_entries: Arc::new(DashMap::new()),
|
||||
prefixes: PrefixBans::new(),
|
||||
local_ban_events,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns whether `ip` is banned: exact match or inside a CIDR ban.
|
||||
pub fn is_blocked(&self, ip: IpAddr) -> bool {
|
||||
Self::is_blocked_in(&self.entries, ip) || Self::is_blocked_in(&self.redis_entries, ip)
|
||||
let ip = normalize(ip);
|
||||
Self::is_blocked_in(&self.entries, ip)
|
||||
|| Self::is_blocked_in(&self.redis_entries, ip)
|
||||
|| self.prefixes.is_blocked(ip, Instant::now())
|
||||
}
|
||||
|
||||
/// Adds a CIDR ban covering `ip/prefix_len` and broadcasts its network
|
||||
/// address, so other nodes mirror at least that host.
|
||||
///
|
||||
/// Invalid prefix lengths (`0`, `> 32` for IPv4, `> 128` for IPv6) are
|
||||
/// ignored: a `/0` ban would blackhole the whole address family.
|
||||
pub fn add_cidr(&self, ip: IpAddr, prefix_len: u8, duration: Duration, reason: &str) {
|
||||
let duration = clamp_duration(duration);
|
||||
let Some(start) = self.prefixes.add(ip, prefix_len, deadline(duration)) else {
|
||||
tracing::warn!(%ip, prefix_len, "ignoring CIDR ban with an invalid prefix length");
|
||||
return;
|
||||
};
|
||||
let expires_at_unix = now_unix().saturating_add(duration.as_secs()).min(i64::MAX as u64) as i64;
|
||||
let _ = self.local_ban_events.send(LocalBanEvent {
|
||||
ip: start,
|
||||
expires_at_unix,
|
||||
reason: reason.to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
/// Adds a local ban and broadcasts it for the Redis publisher.
|
||||
pub fn add(&self, ip: IpAddr, duration: Duration, reason: &str) {
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs();
|
||||
let expires_at_unix = now.saturating_add(duration.as_secs()).min(i64::MAX as u64) as i64;
|
||||
let ip = normalize(ip);
|
||||
let duration = clamp_duration(duration);
|
||||
let expires_at_unix = now_unix().saturating_add(duration.as_secs()).min(i64::MAX as u64) as i64;
|
||||
self.entries.insert(
|
||||
ip,
|
||||
BanEntry {
|
||||
expires: Instant::now() + duration,
|
||||
expires: deadline(duration),
|
||||
_reason: reason.to_string(),
|
||||
},
|
||||
);
|
||||
|
|
@ -64,120 +106,130 @@ impl Blacklist {
|
|||
});
|
||||
}
|
||||
|
||||
/// Subscribes to locally detected bans.
|
||||
pub fn subscribe_local_bans(&self) -> broadcast::Receiver<LocalBanEvent> {
|
||||
self.local_ban_events.subscribe()
|
||||
}
|
||||
|
||||
/// Returns all active local bans.
|
||||
pub fn local_snapshot(&self) -> Vec<LocalBanEvent> {
|
||||
let now = Instant::now();
|
||||
let now_unix = now_unix();
|
||||
self.entries
|
||||
.iter()
|
||||
.filter(|entry| entry.expires > Instant::now())
|
||||
.filter(|entry| entry.expires > now)
|
||||
.map(|entry| LocalBanEvent {
|
||||
ip: *entry.key(),
|
||||
expires_at_unix: std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs()
|
||||
.saturating_add(entry.expires.saturating_duration_since(Instant::now()).as_secs())
|
||||
expires_at_unix: now_unix
|
||||
.saturating_add(entry.expires.saturating_duration_since(now).as_secs())
|
||||
.min(i64::MAX as u64) as i64,
|
||||
reason: entry._reason.clone(),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Removes a local ban.
|
||||
pub fn remove(&self, ip: IpAddr) {
|
||||
self.entries.remove(&ip);
|
||||
self.entries.remove(&normalize(ip));
|
||||
}
|
||||
|
||||
/// Adds a Redis-mirrored ban.
|
||||
pub fn add_redis(&self, ip: IpAddr, duration: Duration, reason: &str) {
|
||||
let ip = normalize(ip);
|
||||
let duration = clamp_duration(duration);
|
||||
self.redis_entries.insert(
|
||||
ip,
|
||||
BanEntry {
|
||||
expires: Instant::now() + duration,
|
||||
expires: deadline(duration),
|
||||
_reason: reason.to_string(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/// Removes a Redis-mirrored ban.
|
||||
pub fn remove_redis(&self, ip: IpAddr) {
|
||||
self.redis_entries.remove(&ip);
|
||||
self.redis_entries.remove(&normalize(ip));
|
||||
}
|
||||
|
||||
/// Replaces the Redis-managed ban set, preserving locally detected bans.
|
||||
///
|
||||
/// `redis_entries` is rebuilt from `entries` wholesale, so a ban that
|
||||
/// disappeared upstream stops applying after this call.
|
||||
/// New entries are inserted before stale ones are removed, so a reconnect
|
||||
/// never exposes a window without bans. A ban that disappeared upstream
|
||||
/// stops applying after this call.
|
||||
pub fn replace_redis<I>(&self, entries: I)
|
||||
where
|
||||
I: IntoIterator<Item = (IpAddr, Duration, String)>,
|
||||
{
|
||||
self.redis_entries.clear();
|
||||
let mut fresh = HashSet::new();
|
||||
for (ip, duration, reason) in entries {
|
||||
fresh.insert(ip);
|
||||
self.add_redis(ip, duration, &reason);
|
||||
}
|
||||
self.redis_entries.retain(|ip, _| fresh.contains(ip));
|
||||
}
|
||||
|
||||
/// Total number of tracked bans.
|
||||
pub fn len(&self) -> usize {
|
||||
self.entries.len() + self.redis_entries.len()
|
||||
self.entries.len() + self.redis_entries.len() + self.prefixes.len()
|
||||
}
|
||||
|
||||
/// Returns whether all ban tables are empty.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.entries.is_empty() && self.redis_entries.is_empty()
|
||||
self.len() == 0
|
||||
}
|
||||
|
||||
/// Drops expired entries from all tables.
|
||||
pub fn clear_expired(&self) {
|
||||
self.entries.retain(|_, entry| entry.expires > Instant::now());
|
||||
self.redis_entries.retain(|_, entry| entry.expires > Instant::now());
|
||||
let now = Instant::now();
|
||||
self.entries.retain(|_, entry| entry.expires > now);
|
||||
self.redis_entries.retain(|_, entry| entry.expires > now);
|
||||
self.prefixes.clear_expired(now);
|
||||
}
|
||||
|
||||
/// Checks `entries`, lazily removing `ip` only if it is still expired.
|
||||
///
|
||||
/// The conditional removal avoids a TOCTOU race in which a concurrent
|
||||
/// re-ban would be deleted right after the expired entry was observed.
|
||||
fn is_blocked_in(entries: &DashMap<IpAddr, BanEntry>, ip: IpAddr) -> bool {
|
||||
if let Some(entry) = entries.get(&ip) {
|
||||
if entry.expires > Instant::now() {
|
||||
return true;
|
||||
}
|
||||
drop(entry);
|
||||
entries.remove(&ip);
|
||||
let now = Instant::now();
|
||||
if let Some(entry) = entries.get(&ip)
|
||||
&& entry.expires > now
|
||||
{
|
||||
return true;
|
||||
}
|
||||
entries.remove_if(&ip, |_, entry| entry.expires <= now);
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::net::{IpAddr, Ipv4Addr};
|
||||
|
||||
fn ip(octets: [u8; 4]) -> IpAddr {
|
||||
IpAddr::V4(Ipv4Addr::from(octets))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_blacklist_block() {
|
||||
let bl = Blacklist::new();
|
||||
bl.add(ip([1, 2, 3, 4]), Duration::from_secs(60), "test");
|
||||
assert!(bl.is_blocked(ip([1, 2, 3, 4])));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_blacklist_not_blocked() {
|
||||
let bl = Blacklist::new();
|
||||
bl.add(ip([1, 2, 3, 4]), Duration::from_secs(60), "test");
|
||||
assert!(!bl.is_blocked(ip([5, 6, 7, 8])));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_blacklist_expired() {
|
||||
let bl = Blacklist::new();
|
||||
bl.add(ip([1, 2, 3, 4]), Duration::from_millis(1), "test");
|
||||
std::thread::sleep(Duration::from_millis(2));
|
||||
assert!(!bl.is_blocked(ip([1, 2, 3, 4])));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_blacklist_remove() {
|
||||
let bl = Blacklist::new();
|
||||
bl.add(ip([1, 2, 3, 4]), Duration::from_secs(60), "test");
|
||||
bl.remove(ip([1, 2, 3, 4]));
|
||||
assert!(!bl.is_blocked(ip([1, 2, 3, 4])));
|
||||
/// Collapses IPv4-mapped IPv6 addresses to IPv4, so one client never occupies
|
||||
/// two different keys in the ban tables.
|
||||
fn normalize(ip: IpAddr) -> IpAddr {
|
||||
match ip {
|
||||
IpAddr::V6(v6) => v6.to_ipv4_mapped().map_or(ip, IpAddr::V4),
|
||||
IpAddr::V4(_) => ip,
|
||||
}
|
||||
}
|
||||
|
||||
/// Clamps a requested duration to [`MAX_BAN_DURATION_SECS`].
|
||||
fn clamp_duration(duration: Duration) -> Duration {
|
||||
duration.min(Duration::from_secs(MAX_BAN_DURATION_SECS))
|
||||
}
|
||||
|
||||
/// Returns the monotonic deadline for an already clamped duration.
|
||||
///
|
||||
/// Overflow cannot occur for clamped values, but is handled defensively so a
|
||||
/// malformed duration can never panic the data plane.
|
||||
fn deadline(duration: Duration) -> Instant {
|
||||
Instant::now()
|
||||
.checked_add(duration)
|
||||
.unwrap_or_else(|| Instant::now() + Duration::from_secs(MAX_BAN_DURATION_SECS))
|
||||
}
|
||||
|
||||
/// Current wall-clock time as Unix seconds.
|
||||
fn now_unix() -> u64 {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs()
|
||||
}
|
||||
|
|
|
|||
119
src/filter/blacklist/prefix.rs
Normal file
119
src/filter/blacklist/prefix.rs
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
//! CIDR ban table.
|
||||
//!
|
||||
//! Exact-address bans live in two `DashMap`s in [`super`]; substrate-wide bans
|
||||
//! (`/24`, `/64`, ...) are stored here as inclusive address ranges. The table is
|
||||
//! a plain `RwLock<Vec<_>>`: entries are few (the subnet monitor bans at most a
|
||||
//! handful per tick), so a linear scan is cheaper than a trie, and the hard cap
|
||||
//! bounds the worst case under abuse.
|
||||
|
||||
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
|
||||
use std::sync::RwLock;
|
||||
use std::time::Instant;
|
||||
|
||||
/// Upper bound on distinct CIDR bans kept in memory.
|
||||
const MAX_PREFIX_BANS: usize = 256;
|
||||
|
||||
/// One CIDR ban, stored as an inclusive address range.
|
||||
struct PrefixBan {
|
||||
start: IpAddr,
|
||||
end: IpAddr,
|
||||
expires: Instant,
|
||||
}
|
||||
|
||||
/// Thread-safe set of active CIDR bans.
|
||||
pub(super) struct PrefixBans {
|
||||
entries: RwLock<Vec<PrefixBan>>,
|
||||
}
|
||||
|
||||
impl PrefixBans {
|
||||
/// Creates an empty table.
|
||||
pub(super) fn new() -> Self {
|
||||
Self {
|
||||
entries: RwLock::new(Vec::new()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Inserts `ip/prefix_len` until `expires`.
|
||||
///
|
||||
/// Returns the network address when the ban was inserted, or `None` for an
|
||||
/// invalid prefix length (`0`, `> 32` for IPv4, `> 128` for IPv6): a `/0`
|
||||
/// ban would blackhole an entire address family.
|
||||
pub(super) fn add(&self, ip: IpAddr, prefix_len: u8, expires: Instant) -> Option<IpAddr> {
|
||||
let (start, end) = cidr_range(ip, prefix_len)?;
|
||||
let now = Instant::now();
|
||||
let mut bans = self.entries.write().unwrap_or_else(|poison| poison.into_inner());
|
||||
bans.retain(|ban| ban.expires > now);
|
||||
if bans.len() >= MAX_PREFIX_BANS {
|
||||
bans.remove(0);
|
||||
}
|
||||
bans.push(PrefixBan { start, end, expires });
|
||||
Some(start)
|
||||
}
|
||||
|
||||
/// Returns whether `ip` falls into an active CIDR ban.
|
||||
pub(super) fn is_blocked(&self, ip: IpAddr, now: Instant) -> bool {
|
||||
let bans = self.entries.read().unwrap_or_else(|poison| poison.into_inner());
|
||||
bans.iter().any(|ban| ban.expires > now && range_contains(ban, ip))
|
||||
}
|
||||
|
||||
/// Drops expired CIDR bans.
|
||||
pub(super) fn clear_expired(&self, now: Instant) {
|
||||
self.entries
|
||||
.write()
|
||||
.unwrap_or_else(|poison| poison.into_inner())
|
||||
.retain(|ban| ban.expires > now);
|
||||
}
|
||||
|
||||
/// Number of tracked CIDR bans (including not-yet-reaped expired ones).
|
||||
pub(super) fn len(&self) -> usize {
|
||||
self.entries.read().unwrap_or_else(|poison| poison.into_inner()).len()
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts `ip/prefix_len` into an inclusive address range.
|
||||
///
|
||||
/// IPv4-mapped IPv6 addresses are normalised to IPv4 first, matching the prefix
|
||||
/// keys used by the subnet detector and the XDP prefix stats.
|
||||
fn cidr_range(ip: IpAddr, prefix_len: u8) -> Option<(IpAddr, IpAddr)> {
|
||||
let ip = match ip {
|
||||
IpAddr::V6(v6) => v6.to_ipv4_mapped().map_or(ip, IpAddr::V4),
|
||||
IpAddr::V4(_) => ip,
|
||||
};
|
||||
match ip {
|
||||
IpAddr::V4(v4) => {
|
||||
if prefix_len == 0 || prefix_len > 32 {
|
||||
return None;
|
||||
}
|
||||
let mask = u32::MAX << (32 - prefix_len);
|
||||
let base = u32::from(v4) & mask;
|
||||
Some((
|
||||
IpAddr::V4(Ipv4Addr::from(base)),
|
||||
IpAddr::V4(Ipv4Addr::from(base | !mask)),
|
||||
))
|
||||
},
|
||||
IpAddr::V6(v6) => {
|
||||
if prefix_len == 0 || prefix_len > 128 {
|
||||
return None;
|
||||
}
|
||||
let mask = u128::MAX << (128 - prefix_len);
|
||||
let base = u128::from(v6) & mask;
|
||||
Some((
|
||||
IpAddr::V6(Ipv6Addr::from(base)),
|
||||
IpAddr::V6(Ipv6Addr::from(base | !mask)),
|
||||
))
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns whether `ip` is inside `ban`'s inclusive range.
|
||||
fn range_contains(ban: &PrefixBan, ip: IpAddr) -> bool {
|
||||
match (ban.start, ban.end, ip) {
|
||||
(IpAddr::V4(start), IpAddr::V4(end), IpAddr::V4(addr)) => {
|
||||
u32::from(start) <= u32::from(addr) && u32::from(addr) <= u32::from(end)
|
||||
},
|
||||
(IpAddr::V6(start), IpAddr::V6(end), IpAddr::V6(addr)) => {
|
||||
u128::from(start) <= u128::from(addr) && u128::from(addr) <= u128::from(end)
|
||||
},
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
|
@ -1,155 +1,175 @@
|
|||
use dashmap::DashMap;
|
||||
//! Per-source token-bucket rate limiter for the accept path.
|
||||
//!
|
||||
//! # Capacity policy
|
||||
//!
|
||||
//! The bucket table is a hard-bounded sharded hash map: at most `max_buckets`
|
||||
//! live buckets (one million for [`RateLimiter::new`]). Admission uses an
|
||||
//! atomic reservation, so concurrent accept loops cannot overshoot the bound
|
||||
//! the way a plain `len()` check would.
|
||||
//!
|
||||
//! # Saturation policy
|
||||
//!
|
||||
//! When the table is full, [`RateLimiter::check`] fails closed for a *new*
|
||||
//! source: the connection is rejected until [`RateLimiter::sweep`] frees a
|
||||
//! slot. Existing sources keep their token state and are never refused because
|
||||
//! of the cap. Failing closed is deliberate for a protection device - the
|
||||
//! alternative (unbounded buckets, or silently allowing everything once the
|
||||
//! table is full) turns a memory exhaustion vector into a bypass.
|
||||
//!
|
||||
//! # Eviction policy
|
||||
//!
|
||||
//! [`RateLimiter::sweep`] visits eight of the sixty-four shards and drops
|
||||
//! buckets idle for longer than `eviction_idle` (ten minutes by default). Work
|
||||
//! per call is bounded, so the accept path never stalls on a full-table
|
||||
//! `retain`, and one full cycle takes eight one-minute ticks.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::collections::hash_map::RandomState;
|
||||
use std::hash::BuildHasher;
|
||||
use std::net::IpAddr;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::{Mutex, MutexGuard};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Default hard bound of the bucket table (one million sources).
|
||||
const MAX_BUCKETS: usize = 1_000_000;
|
||||
/// Default idle age after which a bucket may be reclaimed.
|
||||
const EVICTION_IDLE: Duration = Duration::from_secs(600);
|
||||
const SWEEP_INTERVAL: Duration = Duration::from_secs(60);
|
||||
/// Number of independently locked map shards.
|
||||
const SHARDS: usize = 64;
|
||||
/// Shards examined by one [`RateLimiter::sweep`] call.
|
||||
const SWEEP_SHARDS_PER_TICK: usize = 8;
|
||||
|
||||
/// Token-bucket state of one source address.
|
||||
struct Bucket {
|
||||
tokens: f64,
|
||||
last_refill: Instant,
|
||||
last_access: Instant,
|
||||
}
|
||||
|
||||
/// Per-source token buckets with a hard capacity bound.
|
||||
pub struct RateLimiter {
|
||||
buckets: Arc<DashMap<IpAddr, Bucket>>,
|
||||
shards: Box<[Mutex<HashMap<IpAddr, Bucket>>]>,
|
||||
hasher: RandomState,
|
||||
live: AtomicUsize,
|
||||
max_buckets: usize,
|
||||
max_tokens: f64,
|
||||
refill_rate: f64,
|
||||
_refill_interval: Duration,
|
||||
epoch: Instant,
|
||||
last_sweep_elapsed: AtomicU64,
|
||||
eviction_idle: Duration,
|
||||
sweep_cursor: AtomicUsize,
|
||||
}
|
||||
|
||||
impl RateLimiter {
|
||||
/// Builds a limiter with the production capacity policy.
|
||||
#[must_use]
|
||||
pub fn new(rate_per_sec: f64, burst: f64) -> Self {
|
||||
Self::with_eviction(rate_per_sec, burst, EVICTION_IDLE)
|
||||
Self::with_capacity(rate_per_sec, burst, MAX_BUCKETS, EVICTION_IDLE)
|
||||
}
|
||||
|
||||
fn with_eviction(rate_per_sec: f64, burst: f64, eviction_idle: Duration) -> Self {
|
||||
/// Builds a limiter with an explicit capacity policy.
|
||||
///
|
||||
/// `max_buckets` is the hard bound of the table and `eviction_idle` the
|
||||
/// idle age after which [`Self::sweep`] may reclaim a bucket. [`Self::new`]
|
||||
/// supplies the production defaults; this constructor exists for tests and
|
||||
/// for deployments with a tighter memory budget.
|
||||
#[must_use]
|
||||
pub fn with_capacity(rate_per_sec: f64, burst: f64, max_buckets: usize, eviction_idle: Duration) -> Self {
|
||||
Self {
|
||||
buckets: Arc::new(DashMap::new()),
|
||||
shards: (0..SHARDS).map(|_| Mutex::new(HashMap::new())).collect(),
|
||||
hasher: RandomState::new(),
|
||||
live: AtomicUsize::new(0),
|
||||
max_buckets,
|
||||
max_tokens: burst,
|
||||
refill_rate: rate_per_sec,
|
||||
_refill_interval: Duration::from_secs(1),
|
||||
epoch: Instant::now(),
|
||||
last_sweep_elapsed: AtomicU64::new(0),
|
||||
eviction_idle,
|
||||
sweep_cursor: AtomicUsize::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
/// Consumes one token for `ip`, returning `false` when the source is over
|
||||
/// its rate or the table is saturated (see the module documentation).
|
||||
pub fn check(&self, ip: IpAddr) -> bool {
|
||||
let now = Instant::now();
|
||||
let mut entry = self.buckets.entry(ip).or_insert_with(|| Bucket {
|
||||
let index = self.shard_index(ip);
|
||||
let mut shard = self.lock(index);
|
||||
if let Some(bucket) = shard.get_mut(&ip) {
|
||||
return refill_and_take(bucket, now, self.max_tokens, self.refill_rate);
|
||||
}
|
||||
if !self.reserve() {
|
||||
return false;
|
||||
}
|
||||
let mut bucket = Bucket {
|
||||
tokens: self.max_tokens,
|
||||
last_refill: now,
|
||||
last_access: now,
|
||||
});
|
||||
|
||||
let elapsed = now.duration_since(entry.last_refill);
|
||||
let refill = elapsed.as_secs_f64() * self.refill_rate;
|
||||
entry.tokens = (entry.tokens + refill).min(self.max_tokens);
|
||||
entry.last_refill = now;
|
||||
entry.last_access = now;
|
||||
|
||||
if entry.tokens >= 1.0 {
|
||||
entry.tokens -= 1.0;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
};
|
||||
let allowed = refill_and_take(&mut bucket, now, self.max_tokens, self.refill_rate);
|
||||
shard.insert(ip, bucket);
|
||||
allowed
|
||||
}
|
||||
|
||||
/// Number of live buckets.
|
||||
#[must_use]
|
||||
pub fn len(&self) -> usize {
|
||||
self.buckets.len()
|
||||
(0..self.shards.len()).map(|index| self.lock(index).len()).sum()
|
||||
}
|
||||
|
||||
/// Whether the table is empty.
|
||||
#[must_use]
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.buckets.is_empty()
|
||||
self.len() == 0
|
||||
}
|
||||
|
||||
/// Evicts idle buckets.
|
||||
///
|
||||
/// Runs either when the table grows past `MAX_BUCKETS` (cap enforcement)
|
||||
/// or once per `SWEEP_INTERVAL`. Buckets touched within `eviction_idle`
|
||||
/// survive, so active sources keep their token state.
|
||||
pub fn sweep(&self) {
|
||||
/// Evicts idle buckets from the next shard slice and returns how many were
|
||||
/// removed. See the eviction policy in the module documentation.
|
||||
pub fn sweep(&self) -> usize {
|
||||
let now = Instant::now();
|
||||
let elapsed_secs = now.duration_since(self.epoch).as_secs();
|
||||
let last = self.last_sweep_elapsed.load(Ordering::Relaxed);
|
||||
let due = last == 0 || elapsed_secs.saturating_sub(last) >= SWEEP_INTERVAL.as_secs();
|
||||
let over_cap = self.buckets.len() > MAX_BUCKETS;
|
||||
if !over_cap && !due {
|
||||
return;
|
||||
let shard_count = self.shards.len();
|
||||
let start = self.sweep_cursor.fetch_add(SWEEP_SHARDS_PER_TICK, Ordering::Relaxed);
|
||||
let mut removed = 0;
|
||||
for offset in 0..SWEEP_SHARDS_PER_TICK.min(shard_count) {
|
||||
let index = (start + offset) % shard_count;
|
||||
let mut shard = self.lock(index);
|
||||
let before = shard.len();
|
||||
shard.retain(|_, bucket| now.duration_since(bucket.last_access) < self.eviction_idle);
|
||||
removed += before - shard.len();
|
||||
}
|
||||
self.buckets
|
||||
.retain(|_, b| now.duration_since(b.last_access) < self.eviction_idle);
|
||||
self.last_sweep_elapsed.store(elapsed_secs, Ordering::Relaxed);
|
||||
if removed > 0 {
|
||||
self.live.fetch_sub(removed, Ordering::AcqRel);
|
||||
}
|
||||
removed
|
||||
}
|
||||
|
||||
/// Reserves one slot in the table, enforcing the hard bound.
|
||||
fn reserve(&self) -> bool {
|
||||
let live = self.live.fetch_add(1, Ordering::AcqRel);
|
||||
if live >= self.max_buckets {
|
||||
self.live.fetch_sub(1, Ordering::AcqRel);
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
fn shard_index(&self, ip: IpAddr) -> usize {
|
||||
(self.hasher.hash_one(ip) as usize) % self.shards.len()
|
||||
}
|
||||
|
||||
fn lock(&self, index: usize) -> MutexGuard<'_, HashMap<IpAddr, Bucket>> {
|
||||
// A poisoned shard still holds consistent token buckets; keep serving
|
||||
// instead of taking the whole limiter down.
|
||||
self.shards[index].lock().unwrap_or_else(|poison| poison.into_inner())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::net::{IpAddr, Ipv4Addr};
|
||||
|
||||
fn test_ip(octet: u8) -> IpAddr {
|
||||
IpAddr::V4(Ipv4Addr::new(10, 0, 0, octet))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rate_limit_under() {
|
||||
let limiter = RateLimiter::new(10.0, 10.0);
|
||||
assert!(limiter.check(test_ip(1)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rate_limit_over() {
|
||||
let limiter = RateLimiter::new(1.0, 1.0);
|
||||
assert!(limiter.check(test_ip(1)));
|
||||
assert!(!limiter.check(test_ip(1)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rate_limit_burst() {
|
||||
let limiter = RateLimiter::new(1.0, 5.0);
|
||||
for _ in 0..5 {
|
||||
assert!(limiter.check(test_ip(2)));
|
||||
}
|
||||
assert!(!limiter.check(test_ip(2)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rate_limit_refill() {
|
||||
let limiter = RateLimiter::new(100.0, 1.0);
|
||||
assert!(limiter.check(test_ip(3)));
|
||||
assert!(!limiter.check(test_ip(3)));
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
assert!(limiter.check(test_ip(3)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sweep_removes_idle_keeps_active() {
|
||||
let limiter = RateLimiter::with_eviction(1.0, 10.0, Duration::from_millis(20));
|
||||
limiter.check(test_ip(1));
|
||||
limiter.check(test_ip(2));
|
||||
std::thread::sleep(Duration::from_millis(50));
|
||||
limiter.check(test_ip(2));
|
||||
limiter.sweep();
|
||||
assert_eq!(limiter.len(), 1);
|
||||
assert!(!limiter.buckets.contains_key(&test_ip(1)));
|
||||
assert!(limiter.buckets.contains_key(&test_ip(2)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sweep_does_not_remove_active() {
|
||||
let limiter = RateLimiter::with_eviction(1.0, 10.0, Duration::from_millis(50));
|
||||
limiter.check(test_ip(1));
|
||||
limiter.sweep();
|
||||
assert_eq!(limiter.len(), 1);
|
||||
assert!(limiter.buckets.contains_key(&test_ip(1)));
|
||||
fn refill_and_take(bucket: &mut Bucket, now: Instant, max_tokens: f64, refill_rate: f64) -> bool {
|
||||
let elapsed = now.duration_since(bucket.last_refill);
|
||||
bucket.tokens = (bucket.tokens + elapsed.as_secs_f64() * refill_rate).min(max_tokens);
|
||||
bucket.last_refill = now;
|
||||
bucket.last_access = now;
|
||||
if bucket.tokens >= 1.0 {
|
||||
bucket.tokens -= 1.0;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
//! `POST /api/v1/auth/login`: password check with a per-IP attempt limiter.
|
||||
|
||||
use crate::manager::AppState;
|
||||
use axum::{
|
||||
Json,
|
||||
|
|
@ -11,7 +13,6 @@ use std::{
|
|||
sync::Arc,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use subtle::ConstantTimeEq;
|
||||
|
||||
const LOGIN_WINDOW: Duration = Duration::from_secs(60);
|
||||
const LOGIN_MAX_ATTEMPTS: u32 = 5;
|
||||
|
|
@ -33,7 +34,7 @@ pub async fn login(
|
|||
));
|
||||
}
|
||||
|
||||
if !verify_password(&req.password, &state.api_password) {
|
||||
if !crate::manager::security::verify_password(&req.password, &state.api_password) {
|
||||
return Err((
|
||||
StatusCode::UNAUTHORIZED,
|
||||
Json(serde_json::json!({"error": "invalid password"})),
|
||||
|
|
@ -49,16 +50,11 @@ pub async fn login(
|
|||
}
|
||||
}
|
||||
|
||||
fn verify_password(provided: &str, expected: &str) -> bool {
|
||||
let provided = provided.as_bytes();
|
||||
let expected = expected.as_bytes();
|
||||
let len_match = (provided.len() as u64).ct_eq(&(expected.len() as u64));
|
||||
let min_len = provided.len().min(expected.len());
|
||||
let bytes_match = provided[..min_len].ct_eq(&expected[..min_len]);
|
||||
bool::from(len_match & bytes_match)
|
||||
}
|
||||
|
||||
fn allow_login_attempt(limiter: &DashMap<IpAddr, (Instant, u32)>, ip: IpAddr) -> bool {
|
||||
/// Records a login attempt for `ip` and reports whether it is within the window.
|
||||
///
|
||||
/// The limiter allows `LOGIN_MAX_ATTEMPTS` attempts per `LOGIN_WINDOW` and
|
||||
/// resets the counter when the window elapses.
|
||||
pub fn allow_login_attempt(limiter: &DashMap<IpAddr, (Instant, u32)>, ip: IpAddr) -> bool {
|
||||
let now = Instant::now();
|
||||
let mut slot = limiter.entry(ip).or_insert((now, 0));
|
||||
let (last_reset, attempts) = &mut *slot;
|
||||
|
|
@ -72,33 +68,3 @@ fn allow_login_attempt(limiter: &DashMap<IpAddr, (Instant, u32)>, ip: IpAddr) ->
|
|||
}
|
||||
true
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn verify_password_matches() {
|
||||
assert!(verify_password("s3cret-pass", "s3cret-pass"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verify_password_rejects_wrong() {
|
||||
assert!(!verify_password("wrong-pass", "s3cret-pass"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verify_password_rejects_different_length() {
|
||||
assert!(!verify_password("short", "a-longer-password"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn allow_login_attempt_respects_limit() {
|
||||
let limiter = DashMap::new();
|
||||
let ip = IpAddr::from([127, 0, 0, 1]);
|
||||
for _ in 0..LOGIN_MAX_ATTEMPTS {
|
||||
assert!(allow_login_attempt(&limiter, ip));
|
||||
}
|
||||
assert!(!allow_login_attempt(&limiter, ip));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
//! Owns [`AppState`], the handle shared by every handler through [`SharedState`].
|
||||
pub mod api;
|
||||
pub mod auth;
|
||||
pub mod security;
|
||||
pub mod sync;
|
||||
|
||||
use dashmap::DashMap;
|
||||
|
|
|
|||
135
src/manager/security.rs
Normal file
135
src/manager/security.rs
Normal file
|
|
@ -0,0 +1,135 @@
|
|||
//! Startup security policy of the management plane.
|
||||
//!
|
||||
//! Weak credentials turn the control plane API into an authentication bypass:
|
||||
//! an empty `API_PASSWORD` matches an empty login, and a guessable
|
||||
//! `JWT_SECRET` lets anyone mint admin tokens. The manager validates both
|
||||
//! before it binds a socket, and derives its CORS policy from an explicit
|
||||
//! `CORS_ORIGIN` value instead of silently allowing every origin.
|
||||
|
||||
use anyhow::{Result, bail};
|
||||
use subtle::ConstantTimeEq;
|
||||
use tower_http::cors::{AllowOrigin, Any, CorsLayer};
|
||||
|
||||
/// Minimum length of `API_PASSWORD` in characters.
|
||||
const MIN_API_PASSWORD_CHARS: usize = 16;
|
||||
/// Minimum number of non-whitespace bytes in `JWT_SECRET`.
|
||||
const MIN_JWT_SECRET_BYTES: usize = 32;
|
||||
/// Placeholder password shipped in documentation and examples.
|
||||
const DEFAULT_API_PASSWORD: &str = "changeme";
|
||||
|
||||
/// Validates `API_PASSWORD` before the manager starts serving.
|
||||
///
|
||||
/// # Errors
|
||||
/// Rejects an empty or whitespace-only password, a password shorter than
|
||||
/// `MIN_API_PASSWORD_CHARS` characters, and the `changeme` placeholder.
|
||||
pub fn validate_api_password(password: &str) -> Result<()> {
|
||||
let trimmed = password.trim();
|
||||
if trimmed.is_empty() {
|
||||
bail!("API_PASSWORD must not be empty: an empty password would accept an empty login");
|
||||
}
|
||||
if trimmed.chars().count() < MIN_API_PASSWORD_CHARS {
|
||||
bail!("API_PASSWORD must be at least {MIN_API_PASSWORD_CHARS} characters long");
|
||||
}
|
||||
if trimmed.eq_ignore_ascii_case(DEFAULT_API_PASSWORD) {
|
||||
bail!("API_PASSWORD must not be the default placeholder '{DEFAULT_API_PASSWORD}'");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Validates `JWT_SECRET` before the manager starts serving.
|
||||
///
|
||||
/// # Errors
|
||||
/// Rejects a secret with fewer than `MIN_JWT_SECRET_BYTES` non-whitespace
|
||||
/// bytes, which includes empty and all-whitespace secrets.
|
||||
pub fn validate_jwt_secret(secret: &str) -> Result<()> {
|
||||
let significant = secret.bytes().filter(|byte| !byte.is_ascii_whitespace()).count();
|
||||
if significant < MIN_JWT_SECRET_BYTES {
|
||||
bail!("JWT_SECRET must contain at least {MIN_JWT_SECRET_BYTES} non-whitespace bytes, got {significant}");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Compares a login password with the configured one in constant time.
|
||||
///
|
||||
/// An empty stored password never verifies: that is the authentication bypass
|
||||
/// the startup policy ([`validate_api_password`]) rules out, and the handler
|
||||
/// keeps the same guarantee for any state built by other code paths.
|
||||
#[must_use]
|
||||
pub fn verify_password(provided: &str, expected: &str) -> bool {
|
||||
if expected.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let provided = provided.as_bytes();
|
||||
let expected = expected.as_bytes();
|
||||
let len_match = (provided.len() as u64).ct_eq(&(expected.len() as u64));
|
||||
let min_len = provided.len().min(expected.len());
|
||||
let bytes_match = provided[..min_len].ct_eq(&expected[..min_len]);
|
||||
bool::from(len_match & bytes_match)
|
||||
}
|
||||
|
||||
/// Cross-origin resource sharing policy derived from `CORS_ORIGIN`.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum CorsPolicy {
|
||||
/// No cross-origin layer at all: browsers on other origins are denied.
|
||||
Deny,
|
||||
/// Every origin is allowed; requires an explicit `CORS_ORIGIN=*`.
|
||||
Any,
|
||||
/// Only the listed origins are allowed.
|
||||
AllowList(Vec<String>),
|
||||
}
|
||||
|
||||
impl CorsPolicy {
|
||||
/// Parses the raw `CORS_ORIGIN` value.
|
||||
///
|
||||
/// An unset or empty variable denies cross-origin access, `*` must be set
|
||||
/// explicitly to allow any origin, and anything else is a comma-separated
|
||||
/// list of origins.
|
||||
///
|
||||
/// # Errors
|
||||
/// Fails on an empty entry, on a wildcard mixed with concrete origins, and
|
||||
/// on an entry that is not a valid HTTP header value.
|
||||
pub fn parse(raw: Option<&str>) -> Result<Self> {
|
||||
let Some(raw) = raw else {
|
||||
return Ok(Self::Deny);
|
||||
};
|
||||
let raw = raw.trim();
|
||||
if raw.is_empty() {
|
||||
return Ok(Self::Deny);
|
||||
}
|
||||
if raw == "*" {
|
||||
return Ok(Self::Any);
|
||||
}
|
||||
let mut origins = Vec::new();
|
||||
for entry in raw.split(',') {
|
||||
let origin = entry.trim();
|
||||
if origin.is_empty() {
|
||||
bail!("CORS_ORIGIN contains an empty origin entry");
|
||||
}
|
||||
if origin == "*" {
|
||||
bail!("CORS_ORIGIN wildcard '*' must be used alone, not inside an origin list");
|
||||
}
|
||||
axum::http::HeaderValue::from_str(origin)
|
||||
.map_err(|_| anyhow::anyhow!("CORS_ORIGIN entry '{origin}' is not a valid HTTP header value"))?;
|
||||
origins.push(origin.to_owned());
|
||||
}
|
||||
Ok(Self::AllowList(origins))
|
||||
}
|
||||
|
||||
/// Builds the axum CORS layer for this policy.
|
||||
///
|
||||
/// Returns `None` for [`CorsPolicy::Deny`] so the router is served without
|
||||
/// any cross-origin layer.
|
||||
#[must_use]
|
||||
pub fn cors_layer(&self) -> Option<CorsLayer> {
|
||||
match self {
|
||||
Self::Deny => None,
|
||||
Self::Any => Some(CorsLayer::new().allow_origin(Any)),
|
||||
Self::AllowList(origins) => {
|
||||
let allowed = origins
|
||||
.iter()
|
||||
.filter_map(|origin| axum::http::HeaderValue::from_str(origin).ok());
|
||||
Some(CorsLayer::new().allow_origin(AllowOrigin::list(allowed)))
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,13 +1,42 @@
|
|||
//! ClickHouse event sink.
|
||||
//!
|
||||
//! The data plane must never block on the sink: [`ClickHouseWriter::push`] only
|
||||
//! touches memory, while the background task drains a batch under the writer
|
||||
//! lock and performs network I/O outside of it. Every request is bounded by a
|
||||
//! request timeout and every flush by a global timeout, so an unreachable or
|
||||
//! hung ClickHouse degrades into dropped events instead of a stalled daemon.
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use prometheus::IntCounter;
|
||||
use serde::Serialize;
|
||||
use std::sync::Arc;
|
||||
use std::sync::{Arc, LazyLock};
|
||||
use std::time::Duration;
|
||||
use tokio::sync::Mutex;
|
||||
use tokio::sync::watch;
|
||||
|
||||
const BATCH_SIZE: usize = 1000;
|
||||
const FLUSH_INTERVAL: Duration = Duration::from_secs(1);
|
||||
/// Maximum number of events sent in one INSERT.
|
||||
pub const BATCH_SIZE: usize = 1000;
|
||||
/// Hard cap on buffered events; further pushes are dropped and counted.
|
||||
pub const MAX_BUFFER_EVENTS: usize = 10_000;
|
||||
|
||||
const FLUSH_INTERVAL: Duration = Duration::from_secs(1);
|
||||
const FLUSH_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
const REQUEST_TIMEOUT: Duration = Duration::from_secs(2);
|
||||
const CONNECT_TIMEOUT: Duration = Duration::from_secs(1);
|
||||
const ERROR_BODY_LIMIT: usize = 512;
|
||||
/// Batches drained per tick; keeps sustained event rates from backing up.
|
||||
const MAX_BATCHES_PER_FLUSH: usize = 4;
|
||||
|
||||
/// Events discarded because the buffer was full or a retry could not fit.
|
||||
pub static CLICKHOUSE_DROPPED: LazyLock<IntCounter> = LazyLock::new(|| {
|
||||
prometheus::register_int_counter!(
|
||||
"rampart_clickhouse_dropped_total",
|
||||
"ClickHouse events dropped (buffer overflow or failed flush)"
|
||||
)
|
||||
.expect("CLICKHOUSE_DROPPED")
|
||||
});
|
||||
|
||||
/// One row of the `rampart_events` table.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct ClickHouseEvent {
|
||||
pub timestamp: DateTime<Utc>,
|
||||
|
|
@ -18,72 +47,186 @@ pub struct ClickHouseEvent {
|
|||
pub data_string: String,
|
||||
}
|
||||
|
||||
/// Buffered ClickHouse writer shared by the data plane and the flush task.
|
||||
pub struct ClickHouseWriter {
|
||||
url: String,
|
||||
client: reqwest::Client,
|
||||
buffer: Vec<ClickHouseEvent>,
|
||||
dropped: u64,
|
||||
}
|
||||
|
||||
impl ClickHouseWriter {
|
||||
/// Creates a writer for `url` with bounded request and connect timeouts.
|
||||
pub fn new(url: &str) -> Self {
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(REQUEST_TIMEOUT)
|
||||
.connect_timeout(CONNECT_TIMEOUT)
|
||||
.build()
|
||||
.expect("clickhouse http client");
|
||||
Self {
|
||||
url: url.to_string(),
|
||||
client: reqwest::Client::new(),
|
||||
client,
|
||||
buffer: Vec::with_capacity(BATCH_SIZE),
|
||||
dropped: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Queues `event` for the next flush.
|
||||
///
|
||||
/// This never performs I/O and never fails; when the buffer is full the
|
||||
/// event is dropped and counted instead of blocking the data plane.
|
||||
///
|
||||
/// # Errors
|
||||
/// Currently infallible; the result is kept for API stability.
|
||||
pub async fn push(&mut self, event: ClickHouseEvent) -> anyhow::Result<()> {
|
||||
self.buffer.push(event);
|
||||
if self.buffer.len() >= BATCH_SIZE {
|
||||
self.flush().await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn flush(&mut self) -> anyhow::Result<()> {
|
||||
if self.buffer.is_empty() {
|
||||
if self.buffer.len() >= MAX_BUFFER_EVENTS {
|
||||
self.record_drop(1);
|
||||
return Ok(());
|
||||
}
|
||||
let events = std::mem::take(&mut self.buffer);
|
||||
let json = serde_json::to_string(&events)?;
|
||||
let response = self
|
||||
.client
|
||||
.post(&self.url)
|
||||
.query(&[("query", "INSERT INTO rampart_events FORMAT JSONEachRow")])
|
||||
.header("Content-Type", "application/json")
|
||||
.body(json)
|
||||
.send()
|
||||
.await?;
|
||||
let status = response.status();
|
||||
if !status.is_success() {
|
||||
let text = response.text().await?;
|
||||
anyhow::bail!("clickhouse insert failed ({}): {}", status, text);
|
||||
}
|
||||
tracing::debug!("flushed {} events to clickhouse", events.len());
|
||||
self.buffer.push(event);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Sends every buffered event, re-queueing the batch on failure.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error when the insert fails or exceeds `FLUSH_TIMEOUT`; the
|
||||
/// batch stays buffered for the next attempt.
|
||||
pub async fn flush(&mut self) -> anyhow::Result<()> {
|
||||
let batch = std::mem::take(&mut self.buffer);
|
||||
if batch.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
match tokio::time::timeout(FLUSH_TIMEOUT, send_batch(&self.client, &self.url, &batch)).await {
|
||||
Ok(Ok(())) => {
|
||||
tracing::debug!(events = batch.len(), "flushed events to clickhouse");
|
||||
Ok(())
|
||||
},
|
||||
Ok(Err(error)) => {
|
||||
self.requeue_batch(batch);
|
||||
Err(error)
|
||||
},
|
||||
Err(_) => {
|
||||
self.requeue_batch(batch);
|
||||
anyhow::bail!("clickhouse flush timed out after {FLUSH_TIMEOUT:?}")
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Number of events waiting in the buffer.
|
||||
pub fn pending(&self) -> usize {
|
||||
self.buffer.len()
|
||||
}
|
||||
|
||||
/// Number of events this writer discarded.
|
||||
pub fn dropped(&self) -> u64 {
|
||||
self.dropped
|
||||
}
|
||||
|
||||
fn take_batch(&mut self) -> Vec<ClickHouseEvent> {
|
||||
let take = self.buffer.len().min(BATCH_SIZE);
|
||||
self.buffer.drain(..take).collect()
|
||||
}
|
||||
|
||||
/// Puts a failed batch back in front of the buffer, keeping it bounded.
|
||||
fn requeue_batch(&mut self, mut batch: Vec<ClickHouseEvent>) {
|
||||
batch.append(&mut self.buffer);
|
||||
if batch.len() > MAX_BUFFER_EVENTS {
|
||||
let overflow = batch.len() - MAX_BUFFER_EVENTS;
|
||||
batch.truncate(MAX_BUFFER_EVENTS);
|
||||
self.record_drop(overflow);
|
||||
}
|
||||
self.buffer = batch;
|
||||
}
|
||||
|
||||
fn record_drop(&mut self, count: usize) {
|
||||
let count = u64::try_from(count).unwrap_or(u64::MAX);
|
||||
let first_drop = self.dropped == 0;
|
||||
self.dropped = self.dropped.saturating_add(count);
|
||||
CLICKHOUSE_DROPPED.inc_by(count);
|
||||
// Log once per writer: further drops are visible in the counter only,
|
||||
// so a sustained outage cannot flood the log.
|
||||
if first_drop {
|
||||
tracing::warn!(count, "clickhouse buffer full; dropping events");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Runs the periodic flush loop until shutdown.
|
||||
///
|
||||
/// A closed shutdown channel is treated as a shutdown request: the final
|
||||
/// flush runs and the task exits instead of busy-looping on `changed()`.
|
||||
pub fn start_flush_task(writer: Arc<Mutex<ClickHouseWriter>>, mut shutdown: watch::Receiver<bool>) {
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = tokio::time::sleep(FLUSH_INTERVAL) => {
|
||||
if let Err(e) = writer.lock().await.flush().await {
|
||||
tracing::error!("clickhouse flush error: {e}");
|
||||
}
|
||||
}
|
||||
_ = shutdown.changed() => {
|
||||
if *shutdown.borrow() {
|
||||
tracing::info!("flushing clickhouse on shutdown");
|
||||
if let Err(e) = writer.lock().await.flush().await {
|
||||
tracing::error!("clickhouse final flush error: {e}");
|
||||
for _ in 0..MAX_BATCHES_PER_FLUSH {
|
||||
if !flush_batch(&writer).await {
|
||||
break;
|
||||
}
|
||||
}
|
||||
},
|
||||
changed = shutdown.changed() => {
|
||||
if changed.is_err() || *shutdown.borrow_and_update() {
|
||||
tracing::info!("flushing clickhouse on shutdown");
|
||||
flush_batch(&writer).await;
|
||||
return;
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Drains one batch under the lock and sends it without holding the lock.
|
||||
///
|
||||
/// Returns `false` when there was nothing to send or the send failed.
|
||||
async fn flush_batch(writer: &Arc<Mutex<ClickHouseWriter>>) -> bool {
|
||||
let (client, url, batch) = {
|
||||
let mut writer = writer.lock().await;
|
||||
let batch = writer.take_batch();
|
||||
(writer.client.clone(), writer.url.clone(), batch)
|
||||
};
|
||||
if batch.is_empty() {
|
||||
return false;
|
||||
}
|
||||
match tokio::time::timeout(FLUSH_TIMEOUT, send_batch(&client, &url, &batch)).await {
|
||||
Ok(Ok(())) => {
|
||||
tracing::debug!(events = batch.len(), "flushed events to clickhouse");
|
||||
true
|
||||
},
|
||||
Ok(Err(error)) => {
|
||||
tracing::warn!(%error, events = batch.len(), "clickhouse flush failed; re-queueing");
|
||||
requeue_batch(writer, batch).await;
|
||||
false
|
||||
},
|
||||
Err(_) => {
|
||||
tracing::warn!(events = batch.len(), "clickhouse flush timed out; re-queueing");
|
||||
requeue_batch(writer, batch).await;
|
||||
false
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
async fn requeue_batch(writer: &Arc<Mutex<ClickHouseWriter>>, batch: Vec<ClickHouseEvent>) {
|
||||
writer.lock().await.requeue_batch(batch);
|
||||
}
|
||||
|
||||
async fn send_batch(client: &reqwest::Client, url: &str, batch: &[ClickHouseEvent]) -> anyhow::Result<()> {
|
||||
let json = serde_json::to_string(batch)?;
|
||||
let response = client
|
||||
.post(url)
|
||||
.query(&[("query", "INSERT INTO rampart_events FORMAT JSONEachRow")])
|
||||
.header("Content-Type", "application/json")
|
||||
.body(json)
|
||||
.send()
|
||||
.await?;
|
||||
let status = response.status();
|
||||
if !status.is_success() {
|
||||
let mut text = response.text().await.unwrap_or_default();
|
||||
text.truncate(ERROR_BODY_LIMIT);
|
||||
anyhow::bail!("clickhouse insert failed ({status}): {text}");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,41 +1,59 @@
|
|||
//! Redis-backed [`StateStore`] and blacklist synchronisation.
|
||||
//!
|
||||
//! Commands share one reconnecting [`ConnectionManager`] and are bounded by
|
||||
//! `connection::REDIS_TIMEOUT`; pub/sub connections are acquired and
|
||||
//! subscribed under the same timeout.
|
||||
|
||||
use crate::store::StateStore;
|
||||
use redis::AsyncCommands;
|
||||
use redis::aio::ConnectionManager;
|
||||
|
||||
mod backoff;
|
||||
mod connection;
|
||||
mod sync;
|
||||
|
||||
pub use backoff::{BACKOFF_MAX, BACKOFF_MIN, next_backoff};
|
||||
pub use sync::{start_blacklist_sync, start_local_ban_publisher};
|
||||
|
||||
/// Redis-backed state store.
|
||||
pub struct RedisStore {
|
||||
client: redis::Client,
|
||||
manager: ConnectionManager,
|
||||
}
|
||||
|
||||
impl RedisStore {
|
||||
pub fn new(url: &str) -> anyhow::Result<Self> {
|
||||
/// Connects to `url` and establishes a reconnecting manager.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error for an invalid URL, an unreachable server or an attempt
|
||||
/// that exceeds `connection::REDIS_TIMEOUT`.
|
||||
pub async fn new(url: &str) -> anyhow::Result<Self> {
|
||||
let client = redis::Client::open(url)?;
|
||||
Ok(Self { client })
|
||||
let manager = connection::connection_manager(&client).await?;
|
||||
Ok(Self { manager })
|
||||
}
|
||||
}
|
||||
|
||||
impl StateStore for RedisStore {
|
||||
async fn get(&self, key: &str) -> anyhow::Result<Option<String>> {
|
||||
let mut conn = self.client.get_multiplexed_async_connection().await?;
|
||||
Ok(conn.get(key).await?)
|
||||
let mut conn = self.manager.clone();
|
||||
connection::command(conn.get(key)).await
|
||||
}
|
||||
|
||||
async fn set(&self, key: &str, value: &str) -> anyhow::Result<()> {
|
||||
let mut conn = self.client.get_multiplexed_async_connection().await?;
|
||||
let _: () = conn.set(key, value).await?;
|
||||
let mut conn = self.manager.clone();
|
||||
let _: () = connection::command(conn.set(key, value)).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn del(&self, key: &str) -> anyhow::Result<()> {
|
||||
let mut conn = self.client.get_multiplexed_async_connection().await?;
|
||||
let _: () = conn.del(key).await?;
|
||||
let mut conn = self.manager.clone();
|
||||
let _: () = connection::command(conn.del(key)).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn publish(&self, channel: &str, message: &str) -> anyhow::Result<()> {
|
||||
let mut conn = self.client.get_multiplexed_async_connection().await?;
|
||||
let _: () = conn.publish(channel, message).await?;
|
||||
let mut conn = self.manager.clone();
|
||||
let _: i64 = connection::command(conn.publish(channel, message)).await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
|
|
|||
14
src/store/redis/backoff.rs
Normal file
14
src/store/redis/backoff.rs
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
//! Exponential reconnect backoff shared by the Redis sync tasks.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
/// Initial reconnect delay.
|
||||
pub const BACKOFF_MIN: Duration = Duration::from_secs(1);
|
||||
/// Upper bound for the reconnect delay.
|
||||
pub const BACKOFF_MAX: Duration = Duration::from_secs(30);
|
||||
|
||||
/// Doubles `previous`, capped at `max`; never overflows.
|
||||
#[must_use]
|
||||
pub fn next_backoff(previous: Duration, max: Duration) -> Duration {
|
||||
previous.saturating_mul(2).min(max)
|
||||
}
|
||||
36
src/store/redis/connection.rs
Normal file
36
src/store/redis/connection.rs
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
//! Timeout-bounded Redis connection helpers.
|
||||
//!
|
||||
//! Every connect and command in the store layer goes through this module so a
|
||||
//! stalled server can never block a data-plane or shutdown path.
|
||||
|
||||
use redis::aio::ConnectionManager;
|
||||
use std::future::Future;
|
||||
use std::time::Duration;
|
||||
|
||||
/// Maximum time spent on one Redis connect or command.
|
||||
pub const REDIS_TIMEOUT: Duration = Duration::from_secs(2);
|
||||
|
||||
/// Opens a reconnecting connection manager, bounded by `REDIS_TIMEOUT`.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error for an unreachable server or a timed-out attempt.
|
||||
pub async fn connection_manager(client: &redis::Client) -> anyhow::Result<ConnectionManager> {
|
||||
match tokio::time::timeout(REDIS_TIMEOUT, client.get_connection_manager()).await {
|
||||
Ok(result) => Ok(result?),
|
||||
Err(_) => anyhow::bail!("redis connection timed out after {REDIS_TIMEOUT:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Awaits a Redis command future, bounded by `REDIS_TIMEOUT`.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error when the command fails or exceeds the timeout.
|
||||
pub async fn command<F, T>(future: F) -> anyhow::Result<T>
|
||||
where
|
||||
F: Future<Output = redis::RedisResult<T>>,
|
||||
{
|
||||
match tokio::time::timeout(REDIS_TIMEOUT, future).await {
|
||||
Ok(result) => Ok(result?),
|
||||
Err(_) => anyhow::bail!("redis command timed out after {REDIS_TIMEOUT:?}"),
|
||||
}
|
||||
}
|
||||
|
|
@ -1,9 +1,18 @@
|
|||
//! Blacklist synchronisation over a Redis pub/sub channel.
|
||||
//!
|
||||
//! The subscriber reconciles the durable snapshot on every (re)connect, then
|
||||
//! applies events. All connects, subscribes and commands are timeout-bounded;
|
||||
//! a failed attempt backs off exponentially and a panicking handler is logged
|
||||
//! instead of killing the task.
|
||||
|
||||
mod event;
|
||||
mod publisher;
|
||||
mod snapshot;
|
||||
|
||||
pub use publisher::start_local_ban_publisher;
|
||||
|
||||
use super::backoff::{BACKOFF_MAX, BACKOFF_MIN, next_backoff};
|
||||
use super::connection::{REDIS_TIMEOUT, connection_manager};
|
||||
use crate::filter::blacklist::Blacklist;
|
||||
use futures::StreamExt;
|
||||
use std::sync::Arc;
|
||||
|
|
@ -11,19 +20,17 @@ use std::time::Duration;
|
|||
use tokio::sync::watch;
|
||||
|
||||
const BLACKLIST_CHANNEL: &str = "rampart:blacklist:events";
|
||||
const BACKOFF_MIN: Duration = Duration::from_secs(1);
|
||||
const BACKOFF_MAX: Duration = Duration::from_secs(30);
|
||||
|
||||
fn next_backoff(prev: Duration, max: Duration) -> Duration {
|
||||
prev.saturating_mul(2).min(max)
|
||||
}
|
||||
|
||||
/// Why a subscription attempt ended.
|
||||
enum Flow {
|
||||
/// Shutdown was requested.
|
||||
Stop,
|
||||
/// The attempt failed; reconnect after a backoff delay.
|
||||
Retry,
|
||||
}
|
||||
|
||||
async fn wait_for_shutdown(shutdown: &mut watch::Receiver<bool>) {
|
||||
/// Resolves once shutdown is requested or the channel is closed.
|
||||
pub(super) async fn wait_for_shutdown(shutdown: &mut watch::Receiver<bool>) {
|
||||
loop {
|
||||
if shutdown.changed().await.is_err() || *shutdown.borrow_and_update() {
|
||||
return;
|
||||
|
|
@ -31,6 +38,7 @@ async fn wait_for_shutdown(shutdown: &mut watch::Receiver<bool>) {
|
|||
}
|
||||
}
|
||||
|
||||
/// Runs the blacklist subscriber until shutdown.
|
||||
pub async fn start_blacklist_sync(
|
||||
client: &redis::Client,
|
||||
blacklist: Arc<Blacklist>,
|
||||
|
|
@ -63,30 +71,44 @@ async fn subscribe_once(
|
|||
backoff: &mut Duration,
|
||||
) -> Flow {
|
||||
let mut pubsub = tokio::select! {
|
||||
connected = client.get_async_pubsub() => match connected {
|
||||
Ok(pubsub) => pubsub,
|
||||
Err(e) => {
|
||||
connected = tokio::time::timeout(REDIS_TIMEOUT, client.get_async_pubsub()) => match connected {
|
||||
Ok(Ok(pubsub)) => pubsub,
|
||||
Ok(Err(e)) => {
|
||||
tracing::error!("failed to connect to Redis for blacklist sync: {e}");
|
||||
return Flow::Retry;
|
||||
},
|
||||
Err(_) => {
|
||||
tracing::error!("timed out connecting to Redis for blacklist sync");
|
||||
return Flow::Retry;
|
||||
},
|
||||
},
|
||||
() = wait_for_shutdown(shutdown) => return Flow::Stop,
|
||||
};
|
||||
|
||||
tokio::select! {
|
||||
subscribed = pubsub.subscribe(BLACKLIST_CHANNEL) => match subscribed {
|
||||
Ok(()) => tracing::info!("subscribed to {BLACKLIST_CHANNEL}"),
|
||||
Err(e) => {
|
||||
subscribed = tokio::time::timeout(REDIS_TIMEOUT, pubsub.subscribe(BLACKLIST_CHANNEL)) => match subscribed {
|
||||
Ok(Ok(())) => {
|
||||
tracing::info!("subscribed to {BLACKLIST_CHANNEL}");
|
||||
*backoff = BACKOFF_MIN;
|
||||
},
|
||||
Ok(Err(e)) => {
|
||||
tracing::error!("failed to subscribe to blacklist events: {e}");
|
||||
return Flow::Retry;
|
||||
},
|
||||
Err(_) => {
|
||||
tracing::error!("timed out subscribing to blacklist events");
|
||||
return Flow::Retry;
|
||||
},
|
||||
},
|
||||
() = wait_for_shutdown(shutdown) => return Flow::Stop,
|
||||
}
|
||||
|
||||
tokio::select! {
|
||||
loaded = snapshot::reconcile(client, blacklist) => match loaded {
|
||||
Ok(()) => tracing::info!("blacklist snapshot reconciled from Redis"),
|
||||
loaded = reconcile(blacklist, client) => match loaded {
|
||||
Ok(()) => {
|
||||
tracing::info!("blacklist snapshot reconciled from Redis");
|
||||
*backoff = BACKOFF_MIN;
|
||||
},
|
||||
Err(e) => {
|
||||
tracing::error!("failed to reconcile blacklist snapshot: {e}");
|
||||
return Flow::Retry;
|
||||
|
|
@ -99,13 +121,7 @@ async fn subscribe_once(
|
|||
loop {
|
||||
tokio::select! {
|
||||
maybe_msg = stream.next() => match maybe_msg {
|
||||
Some(msg) => {
|
||||
if let Err(e) = event::handle(&msg, blacklist) {
|
||||
tracing::error!("blacklist event error: {e}");
|
||||
} else {
|
||||
*backoff = BACKOFF_MIN;
|
||||
}
|
||||
},
|
||||
Some(msg) => handle_message(&msg, blacklist, backoff),
|
||||
None => {
|
||||
tracing::error!("blacklist pubsub stream ended; resubscribing");
|
||||
return Flow::Retry;
|
||||
|
|
@ -116,22 +132,18 @@ async fn subscribe_once(
|
|||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn backoff_doubles_until_cap() {
|
||||
let mut b = BACKOFF_MIN;
|
||||
for expected_secs in [2, 4, 8, 16, 30, 30] {
|
||||
b = next_backoff(b, BACKOFF_MAX);
|
||||
assert_eq!(b, Duration::from_secs(expected_secs));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn backoff_never_exceeds_max_even_with_huge_prev() {
|
||||
let b = next_backoff(Duration::from_secs(u64::MAX / 2), BACKOFF_MAX);
|
||||
assert!(b <= BACKOFF_MAX);
|
||||
/// Handles one event without letting a rejected or panicking payload kill the
|
||||
/// subscriber task.
|
||||
fn handle_message(msg: &redis::Msg, blacklist: &Blacklist, backoff: &mut Duration) {
|
||||
let handled = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| event::handle(msg, blacklist)));
|
||||
match handled {
|
||||
Ok(Ok(())) => *backoff = BACKOFF_MIN,
|
||||
Ok(Err(error)) => tracing::warn!(%error, "rejected blacklist event"),
|
||||
Err(_) => tracing::error!("blacklist event handler panicked; continuing"),
|
||||
}
|
||||
}
|
||||
|
||||
async fn reconcile(blacklist: &Blacklist, client: &redis::Client) -> anyhow::Result<()> {
|
||||
let manager = connection_manager(client).await?;
|
||||
snapshot::reconcile(&manager, blacklist).await
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,14 @@
|
|||
use crate::filter::blacklist::Blacklist;
|
||||
//! Decoding and validation of blacklist events received from Redis.
|
||||
//!
|
||||
//! A remote or stale manager must not be able to ban an address for an
|
||||
//! arbitrary duration, so every ban is range-checked against
|
||||
//! [`MAX_BAN_DURATION_SECS`] before it reaches the in-memory table.
|
||||
|
||||
use crate::filter::blacklist::{Blacklist, MAX_BAN_DURATION_SECS};
|
||||
use redis::Msg;
|
||||
use serde::Deserialize;
|
||||
use std::time::Duration;
|
||||
|
||||
const MAX_BAN_DURATION_SECS: u64 = 31_536_000;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct BlacklistEvent {
|
||||
ip: String,
|
||||
|
|
@ -27,31 +31,54 @@ pub(super) fn handle(msg: &Msg, blacklist: &Blacklist) -> anyhow::Result<()> {
|
|||
match event.action.as_str() {
|
||||
"ban" => {
|
||||
let duration = match event.expires_at_unix {
|
||||
Some(expires_at) => {
|
||||
if chrono::DateTime::from_timestamp(expires_at, 0).is_none() {
|
||||
anyhow::bail!("invalid blacklist event expiry timestamp");
|
||||
}
|
||||
let remaining = expires_at.saturating_sub(chrono::Utc::now().timestamp());
|
||||
if remaining <= 0 {
|
||||
blacklist.remove_redis(ip);
|
||||
return Ok(());
|
||||
}
|
||||
let remaining = u64::try_from(remaining)?;
|
||||
if remaining > MAX_BAN_DURATION_SECS {
|
||||
anyhow::bail!("blacklist event expiry exceeds maximum duration");
|
||||
}
|
||||
Duration::from_secs(remaining)
|
||||
},
|
||||
None => Duration::from_secs(event.duration_secs),
|
||||
Some(expires_at) => absolute_duration(expires_at)?,
|
||||
None => checked_duration(event.duration_secs)?,
|
||||
};
|
||||
blacklist.add_redis(ip, duration, "manager_event");
|
||||
tracing::info!("blacklist add via Redis: {}", event.ip);
|
||||
match duration {
|
||||
Some(duration) => {
|
||||
blacklist.add_redis(ip, duration, "manager_event");
|
||||
tracing::info!(ip = %event.ip, "blacklist add via Redis");
|
||||
},
|
||||
None => {
|
||||
blacklist.remove_redis(ip);
|
||||
tracing::info!(ip = %event.ip, "blacklist expiry already passed");
|
||||
},
|
||||
}
|
||||
},
|
||||
"unban" => {
|
||||
blacklist.remove_redis(ip);
|
||||
tracing::info!("blacklist remove via Redis: {}", event.ip);
|
||||
tracing::info!(ip = %event.ip, "blacklist remove via Redis");
|
||||
},
|
||||
action => anyhow::bail!("unknown action: {action}"),
|
||||
action => anyhow::bail!("unknown blacklist action: {action}"),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Converts an absolute expiry into the remaining ban duration.
|
||||
///
|
||||
/// Returns `Ok(None)` when the expiry is already in the past.
|
||||
fn absolute_duration(expires_at: i64) -> anyhow::Result<Option<Duration>> {
|
||||
if chrono::DateTime::from_timestamp(expires_at, 0).is_none() {
|
||||
anyhow::bail!("invalid blacklist event expiry timestamp");
|
||||
}
|
||||
let remaining = expires_at.saturating_sub(chrono::Utc::now().timestamp());
|
||||
if remaining <= 0 {
|
||||
return Ok(None);
|
||||
}
|
||||
let remaining = u64::try_from(remaining)?;
|
||||
if remaining > MAX_BAN_DURATION_SECS {
|
||||
anyhow::bail!("blacklist event expiry exceeds {MAX_BAN_DURATION_SECS}s");
|
||||
}
|
||||
Ok(Some(Duration::from_secs(remaining)))
|
||||
}
|
||||
|
||||
/// Validates a relative ban duration.
|
||||
fn checked_duration(secs: u64) -> anyhow::Result<Option<Duration>> {
|
||||
if secs > MAX_BAN_DURATION_SECS {
|
||||
anyhow::bail!("blacklist event duration {secs}s exceeds {MAX_BAN_DURATION_SECS}s");
|
||||
}
|
||||
if secs == 0 {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(Duration::from_secs(secs)))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,16 @@
|
|||
//! Publishes locally detected bans to Redis.
|
||||
//!
|
||||
//! Each ban is stored atomically (set member, metadata hash, event publish) via
|
||||
//! a Lua script on a shared reconnecting connection. Failed publishes retry
|
||||
//! with exponential backoff and abandon the event when shutdown is requested.
|
||||
|
||||
use super::wait_for_shutdown;
|
||||
use crate::filter::blacklist::{Blacklist, LocalBanEvent};
|
||||
use crate::store::redis::backoff::{BACKOFF_MAX, BACKOFF_MIN, next_backoff};
|
||||
use crate::store::redis::connection::{command, connection_manager};
|
||||
use redis::aio::ConnectionManager;
|
||||
use serde::Serialize;
|
||||
use std::time::Duration;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::{broadcast, watch};
|
||||
|
||||
const BLACKLIST_KEY: &str = "rampart:blacklist";
|
||||
|
|
@ -24,14 +34,16 @@ struct BanEvent<'a> {
|
|||
reason: &'a str,
|
||||
}
|
||||
|
||||
/// Publishes local bans and the current local snapshot until shutdown.
|
||||
pub async fn start_local_ban_publisher(
|
||||
client: redis::Client,
|
||||
blacklist: std::sync::Arc<Blacklist>,
|
||||
blacklist: Arc<Blacklist>,
|
||||
mut events: broadcast::Receiver<LocalBanEvent>,
|
||||
mut shutdown: watch::Receiver<bool>,
|
||||
) {
|
||||
let mut manager: Option<ConnectionManager> = None;
|
||||
for event in blacklist.local_snapshot() {
|
||||
if !publish_with_retry(&client, event, &mut shutdown).await {
|
||||
if !publish_with_retry(&client, &mut manager, event, &mut shutdown).await {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
|
@ -40,14 +52,14 @@ pub async fn start_local_ban_publisher(
|
|||
tokio::select! {
|
||||
received = events.recv() => match received {
|
||||
Ok(event) => {
|
||||
if !publish_with_retry(&client, event, &mut shutdown).await {
|
||||
if !publish_with_retry(&client, &mut manager, event, &mut shutdown).await {
|
||||
return;
|
||||
}
|
||||
},
|
||||
Err(broadcast::error::RecvError::Lagged(skipped)) => {
|
||||
tracing::warn!(skipped, "local ban publisher lagged; reconciling active local bans");
|
||||
for event in blacklist.local_snapshot() {
|
||||
if !publish_with_retry(&client, event, &mut shutdown).await {
|
||||
if !publish_with_retry(&client, &mut manager, event, &mut shutdown).await {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
|
@ -63,19 +75,29 @@ pub async fn start_local_ban_publisher(
|
|||
}
|
||||
}
|
||||
|
||||
/// Publishes one event, retrying until it succeeds or shutdown is requested.
|
||||
async fn publish_with_retry(
|
||||
client: &redis::Client,
|
||||
manager: &mut Option<ConnectionManager>,
|
||||
event: LocalBanEvent,
|
||||
shutdown: &mut watch::Receiver<bool>,
|
||||
) -> bool {
|
||||
let mut delay = Duration::from_secs(1);
|
||||
let mut delay = BACKOFF_MIN;
|
||||
loop {
|
||||
match publish_ban(client, &event).await {
|
||||
let result = tokio::select! {
|
||||
result = publish_ban(client, manager, &event) => result,
|
||||
() = wait_for_shutdown(shutdown) => return false,
|
||||
};
|
||||
match result {
|
||||
Ok(()) => return true,
|
||||
Err(error) => tracing::warn!(%error, ip = %event.ip, "failed to persist local ban; retrying"),
|
||||
Err(error) => {
|
||||
// Force a reconnect on the next attempt.
|
||||
*manager = None;
|
||||
tracing::warn!(%error, ip = %event.ip, "failed to persist local ban; retrying");
|
||||
},
|
||||
}
|
||||
tokio::select! {
|
||||
_ = tokio::time::sleep(delay) => delay = (delay * 2).min(Duration::from_secs(30)),
|
||||
_ = tokio::time::sleep(delay) => delay = next_backoff(delay, BACKOFF_MAX),
|
||||
changed = shutdown.changed() => {
|
||||
if changed.is_err() || *shutdown.borrow_and_update() {
|
||||
return false;
|
||||
|
|
@ -85,7 +107,11 @@ async fn publish_with_retry(
|
|||
}
|
||||
}
|
||||
|
||||
async fn publish_ban(client: &redis::Client, event: &LocalBanEvent) -> anyhow::Result<()> {
|
||||
async fn publish_ban(
|
||||
client: &redis::Client,
|
||||
manager: &mut Option<ConnectionManager>,
|
||||
event: &LocalBanEvent,
|
||||
) -> anyhow::Result<()> {
|
||||
let now = chrono::Utc::now().timestamp();
|
||||
let remaining = event.expires_at_unix.saturating_sub(now);
|
||||
if remaining <= 0 {
|
||||
|
|
@ -103,17 +129,26 @@ async fn publish_ban(client: &redis::Client, event: &LocalBanEvent) -> anyhow::R
|
|||
expires_at_unix: event.expires_at_unix,
|
||||
reason: &event.reason,
|
||||
})?;
|
||||
let mut conn = client.get_multiplexed_async_connection().await?;
|
||||
let _: i64 = redis::cmd("EVAL")
|
||||
.arg(STORE_BAN_SCRIPT)
|
||||
.arg(3)
|
||||
.arg(BLACKLIST_KEY)
|
||||
.arg(BLACKLIST_METADATA_KEY)
|
||||
.arg(BLACKLIST_CHANNEL)
|
||||
.arg(event.ip.to_string())
|
||||
.arg(metadata)
|
||||
.arg(payload)
|
||||
.query_async(&mut conn)
|
||||
.await?;
|
||||
let mut conn = match manager {
|
||||
Some(conn) => conn.clone(),
|
||||
None => {
|
||||
let conn = connection_manager(client).await?;
|
||||
*manager = Some(conn.clone());
|
||||
conn
|
||||
},
|
||||
};
|
||||
let _: i64 = command(
|
||||
redis::cmd("EVAL")
|
||||
.arg(STORE_BAN_SCRIPT)
|
||||
.arg(3)
|
||||
.arg(BLACKLIST_KEY)
|
||||
.arg(BLACKLIST_METADATA_KEY)
|
||||
.arg(BLACKLIST_CHANNEL)
|
||||
.arg(event.ip.to_string())
|
||||
.arg(metadata)
|
||||
.arg(payload)
|
||||
.query_async(&mut conn),
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,13 @@
|
|||
use crate::filter::blacklist::Blacklist;
|
||||
//! Durable blacklist snapshot reconciliation.
|
||||
//!
|
||||
//! The snapshot is rebuilt from `rampart:blacklist` (set) and
|
||||
//! `rampart:blacklist:metadata` (hash) after every (re)subscribe and then
|
||||
//! swapped into the mirrored in-memory table.
|
||||
|
||||
use crate::filter::blacklist::{Blacklist, MAX_BAN_DURATION_SECS};
|
||||
use crate::store::redis::connection::command;
|
||||
use redis::AsyncCommands;
|
||||
use redis::aio::ConnectionManager;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::time::Duration;
|
||||
|
|
@ -7,7 +15,6 @@ use std::time::Duration;
|
|||
const BLACKLIST_KEY: &str = "rampart:blacklist";
|
||||
const BLACKLIST_METADATA_KEY: &str = "rampart:blacklist:metadata";
|
||||
const LEGACY_BAN_DURATION_SECS: i64 = 300;
|
||||
const MAX_BAN_DURATION_SECS: i64 = 31_536_000;
|
||||
|
||||
#[derive(Deserialize, Serialize)]
|
||||
struct BlacklistMetadata {
|
||||
|
|
@ -16,11 +23,15 @@ struct BlacklistMetadata {
|
|||
reason: String,
|
||||
}
|
||||
|
||||
/// Restore current durable bans after subscribing to the event channel.
|
||||
pub(super) async fn reconcile(client: &redis::Client, blacklist: &Blacklist) -> anyhow::Result<()> {
|
||||
let mut conn = client.get_multiplexed_async_connection().await?;
|
||||
let members: Vec<String> = redis::cmd("SMEMBERS").arg(BLACKLIST_KEY).query_async(&mut conn).await?;
|
||||
let metadata: HashMap<String, String> = conn.hgetall(BLACKLIST_METADATA_KEY).await?;
|
||||
/// Restores current durable bans after subscribing to the event channel.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error when Redis is unreachable, a command times out or a
|
||||
/// recorded timestamp is outside the representable range.
|
||||
pub(super) async fn reconcile(manager: &ConnectionManager, blacklist: &Blacklist) -> anyhow::Result<()> {
|
||||
let mut conn = manager.clone();
|
||||
let members: Vec<String> = command(redis::cmd("SMEMBERS").arg(BLACKLIST_KEY).query_async(&mut conn)).await?;
|
||||
let metadata: HashMap<String, String> = command(conn.hgetall(BLACKLIST_METADATA_KEY)).await?;
|
||||
let now = chrono::Utc::now().timestamp();
|
||||
let mut active = Vec::with_capacity(members.len());
|
||||
|
||||
|
|
@ -39,22 +50,28 @@ pub(super) async fn reconcile(client: &redis::Client, blacklist: &Blacklist) ->
|
|||
None => continue,
|
||||
},
|
||||
};
|
||||
if record.expires_at_unix <= now
|
||||
|| chrono::DateTime::from_timestamp(record.created_at_unix, 0).is_none()
|
||||
if chrono::DateTime::from_timestamp(record.created_at_unix, 0).is_none()
|
||||
|| chrono::DateTime::from_timestamp(record.expires_at_unix, 0).is_none()
|
||||
|| record.expires_at_unix.saturating_sub(now) > MAX_BAN_DURATION_SECS
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let remaining = u64::try_from(record.expires_at_unix - now)?;
|
||||
let remaining = record.expires_at_unix.saturating_sub(now);
|
||||
if remaining <= 0 {
|
||||
continue;
|
||||
}
|
||||
let remaining = u64::try_from(remaining)?;
|
||||
if remaining > MAX_BAN_DURATION_SECS {
|
||||
continue;
|
||||
}
|
||||
active.push((ip, Duration::from_secs(remaining), record.reason));
|
||||
}
|
||||
blacklist.replace_redis(active);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Backfills metadata for a ban written by an older manager.
|
||||
async fn legacy_record(
|
||||
conn: &mut redis::aio::MultiplexedConnection,
|
||||
conn: &mut ConnectionManager,
|
||||
target: &str,
|
||||
now: i64,
|
||||
) -> anyhow::Result<Option<BlacklistMetadata>> {
|
||||
|
|
@ -63,15 +80,17 @@ async fn legacy_record(
|
|||
expires_at_unix: now.saturating_add(LEGACY_BAN_DURATION_SECS),
|
||||
reason: "legacy_unverified".to_string(),
|
||||
};
|
||||
let inserted: i64 = redis::cmd("HSETNX")
|
||||
.arg(BLACKLIST_METADATA_KEY)
|
||||
.arg(target)
|
||||
.arg(serde_json::to_string(&legacy)?)
|
||||
.query_async(conn)
|
||||
.await?;
|
||||
let inserted: i64 = command(
|
||||
redis::cmd("HSETNX")
|
||||
.arg(BLACKLIST_METADATA_KEY)
|
||||
.arg(target)
|
||||
.arg(serde_json::to_string(&legacy)?)
|
||||
.query_async(conn),
|
||||
)
|
||||
.await?;
|
||||
if inserted == 1 {
|
||||
return Ok(Some(legacy));
|
||||
}
|
||||
let current: Option<String> = conn.hget(BLACKLIST_METADATA_KEY, target).await?;
|
||||
let current: Option<String> = command(conn.hget(BLACKLIST_METADATA_KEY, target)).await?;
|
||||
Ok(current.and_then(|value| serde_json::from_str(&value).ok()))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,16 +1,93 @@
|
|||
//! Attack state machine over the hour-of-week traffic profile.
|
||||
//!
|
||||
//! [`AttackDetector`] scores every one-second window against the baselines
|
||||
//! *before* the window is folded into them. Scoring first is what keeps a
|
||||
//! sustained flood visible: a detector that learns the attack as normal stops
|
||||
//! escalating after a few ticks, and anything that depends on
|
||||
//! [`AttackStatus::UnderAttack`] (for example offender bans) stops working for
|
||||
//! the rest of the attack.
|
||||
//!
|
||||
//! Two baselines are combined:
|
||||
//!
|
||||
//! - [`TrafficProfiler`]: the hour-of-week shape, updated from clean windows.
|
||||
//! - A slow seeded EWMA (1% per window): the global traffic level. It covers
|
||||
//! the cold start and the first tick of every hour slot, where the
|
||||
//! hour-of-week baseline is still zero or close to it.
|
||||
//!
|
||||
//! A window is scored against the higher of the two, and attack-level windows
|
||||
//! are not recorded at all. A flood therefore never becomes the baseline it is
|
||||
//! compared to, while ordinary drift (below [`DetectorThresholds::attack_score`])
|
||||
//! keeps both baselines current.
|
||||
//!
|
||||
//! Leaving [`AttackStatus::UnderAttack`] requires
|
||||
//! [`DetectorThresholds::recover_after`] consecutive windows below
|
||||
//! [`DetectorThresholds::attack_score`]. The exit hysteresis keeps a single
|
||||
//! calm window in the middle of an attack from restarting the escalation
|
||||
//! count.
|
||||
|
||||
use crate::traffic::ewma::Ewma;
|
||||
use crate::traffic::profiler::TrafficProfiler;
|
||||
|
||||
/// Smoothing factor of the slow global baseline: one percent per window, so a
|
||||
/// legitimate traffic shift is learned in about a hundred seconds.
|
||||
const SLOW_ALPHA: f64 = 0.01;
|
||||
/// Upper bound of a reported score; `10` means "at least ten times the
|
||||
/// baseline" and keeps the attack state machine from chasing huge ratios.
|
||||
const MAX_ANOMALY_RATIO: f64 = 10.0;
|
||||
|
||||
/// Classification of the current traffic window.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum AttackStatus {
|
||||
/// Traffic is within the learned baseline.
|
||||
Normal,
|
||||
/// A ratio above [`DetectorThresholds::suspicious_score`] was observed,
|
||||
/// but the escalation count is not reached yet.
|
||||
Suspicious,
|
||||
/// Consecutive anomalous windows were observed; the edge escalates.
|
||||
UnderAttack,
|
||||
}
|
||||
|
||||
/// Tunables of the attack state machine.
|
||||
///
|
||||
/// The scores are ratios to the baseline (a value of `3.0` means "three times
|
||||
/// the usual rate"). The defaults match the v0.3 policy; a caller that needs a
|
||||
/// different sensitivity passes its own values to
|
||||
/// [`AttackDetector::with_thresholds`].
|
||||
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||
pub struct DetectorThresholds {
|
||||
/// Ratio that marks a window as suspicious.
|
||||
pub suspicious_score: f64,
|
||||
/// Ratio that counts a window as anomalous for escalation.
|
||||
pub attack_score: f64,
|
||||
/// Consecutive anomalous windows required to enter `UnderAttack`.
|
||||
pub escalate_after: u32,
|
||||
/// Consecutive sub-attack windows required to leave `UnderAttack`.
|
||||
pub recover_after: u32,
|
||||
}
|
||||
|
||||
impl Default for DetectorThresholds {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
suspicious_score: 2.0,
|
||||
attack_score: 3.0,
|
||||
escalate_after: 3,
|
||||
recover_after: 5,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Detects floods from the packets-per-second and connections-per-second
|
||||
/// profiles.
|
||||
pub struct AttackDetector {
|
||||
pps_profiler: TrafficProfiler,
|
||||
cps_profiler: TrafficProfiler,
|
||||
pps_slow: Ewma,
|
||||
cps_slow: Ewma,
|
||||
primed: bool,
|
||||
thresholds: DetectorThresholds,
|
||||
consecutive_anomalies: u32,
|
||||
recovery_windows: u32,
|
||||
under_attack: bool,
|
||||
}
|
||||
|
||||
impl Default for AttackDetector {
|
||||
|
|
@ -20,95 +97,99 @@ impl Default for AttackDetector {
|
|||
}
|
||||
|
||||
impl AttackDetector {
|
||||
/// Builds a detector with the default [`DetectorThresholds`].
|
||||
#[must_use]
|
||||
pub fn new() -> Self {
|
||||
Self::with_thresholds(DetectorThresholds::default())
|
||||
}
|
||||
|
||||
/// Builds a detector with explicit thresholds.
|
||||
#[must_use]
|
||||
pub fn with_thresholds(thresholds: DetectorThresholds) -> Self {
|
||||
Self {
|
||||
pps_profiler: TrafficProfiler::new(),
|
||||
cps_profiler: TrafficProfiler::new(),
|
||||
pps_slow: Ewma::new(SLOW_ALPHA),
|
||||
cps_slow: Ewma::new(SLOW_ALPHA),
|
||||
primed: false,
|
||||
thresholds,
|
||||
consecutive_anomalies: 0,
|
||||
recovery_windows: 0,
|
||||
under_attack: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Analyses one metrics window (packets/s and new connections/s) against
|
||||
/// the accumulated profiles.
|
||||
/// Analyses one metrics window (packets/s and new connections/s) and
|
||||
/// returns the attack classification.
|
||||
///
|
||||
/// A spike in either metric counts as an anomaly. Three consecutive
|
||||
/// anomalous windows escalate to [`AttackStatus::UnderAttack`]; a single
|
||||
/// one above the suspicious score yields [`AttackStatus::Suspicious`].
|
||||
/// A window is anomalous when its ratio to the baseline reaches
|
||||
/// [`DetectorThresholds::attack_score`].
|
||||
/// [`DetectorThresholds::escalate_after`] consecutive anomalous windows
|
||||
/// enter `UnderAttack`; the status is left only after
|
||||
/// [`DetectorThresholds::recover_after`] consecutive windows below the
|
||||
/// attack score. The first window only primes the baselines.
|
||||
pub fn analyze(&mut self, pps: f64, cps: f64) -> AttackStatus {
|
||||
self.pps_profiler.record(pps);
|
||||
self.cps_profiler.record(cps);
|
||||
let score = self
|
||||
.pps_profiler
|
||||
.anomaly_score(pps)
|
||||
.max(self.cps_profiler.anomaly_score(cps));
|
||||
|
||||
if score > 3.0 {
|
||||
self.consecutive_anomalies += 1;
|
||||
} else {
|
||||
self.consecutive_anomalies = 0;
|
||||
if !self.primed {
|
||||
self.primed = true;
|
||||
self.pps_slow.seed(pps);
|
||||
self.cps_slow.seed(cps);
|
||||
self.pps_profiler.record(pps);
|
||||
self.cps_profiler.record(cps);
|
||||
return AttackStatus::Normal;
|
||||
}
|
||||
|
||||
if self.consecutive_anomalies >= 3 {
|
||||
// Score against the baselines first: recording first would fold the
|
||||
// spike into the baseline it is scored against.
|
||||
let pps_baseline = self.pps_profiler.baseline().max(self.pps_slow.value());
|
||||
let cps_baseline = self.cps_profiler.baseline().max(self.cps_slow.value());
|
||||
let score = anomaly_ratio(pps, pps_baseline).max(anomaly_ratio(cps, cps_baseline));
|
||||
|
||||
self.update_state(score);
|
||||
|
||||
// Attack-level windows never enter the baselines (see the module
|
||||
// documentation); everything else keeps both of them current.
|
||||
if score < self.thresholds.attack_score {
|
||||
self.pps_profiler.record(pps);
|
||||
self.cps_profiler.record(cps);
|
||||
self.pps_slow.update(pps);
|
||||
self.cps_slow.update(cps);
|
||||
}
|
||||
|
||||
if self.under_attack {
|
||||
AttackStatus::UnderAttack
|
||||
} else if score > 2.0 {
|
||||
} else if score >= self.thresholds.suspicious_score {
|
||||
AttackStatus::Suspicious
|
||||
} else {
|
||||
AttackStatus::Normal
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_attack_status_normal() {
|
||||
let mut d = AttackDetector::new();
|
||||
for _ in 0..10 {
|
||||
d.analyze(100.0, 10.0);
|
||||
fn update_state(&mut self, score: f64) {
|
||||
if score >= self.thresholds.attack_score {
|
||||
self.consecutive_anomalies = self.consecutive_anomalies.saturating_add(1);
|
||||
self.recovery_windows = 0;
|
||||
} else {
|
||||
self.consecutive_anomalies = 0;
|
||||
self.recovery_windows = self.recovery_windows.saturating_add(1);
|
||||
}
|
||||
assert_eq!(d.analyze(100.0, 10.0), AttackStatus::Normal);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_attack_status_suspicious_pps_spike() {
|
||||
let mut d = AttackDetector::new();
|
||||
for _ in 0..10 {
|
||||
d.analyze(1.0, 1.0);
|
||||
if self.under_attack {
|
||||
if self.recovery_windows >= self.thresholds.recover_after {
|
||||
self.under_attack = false;
|
||||
}
|
||||
} else if self.consecutive_anomalies >= self.thresholds.escalate_after {
|
||||
self.under_attack = true;
|
||||
}
|
||||
assert_eq!(d.analyze(5.0, 1.0), AttackStatus::Suspicious);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_attack_status_suspicious_cps_spike() {
|
||||
let mut d = AttackDetector::new();
|
||||
for _ in 0..10 {
|
||||
d.analyze(1.0, 1.0);
|
||||
}
|
||||
assert_eq!(d.analyze(1.0, 5.0), AttackStatus::Suspicious);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_attack_status_under_attack() {
|
||||
let mut d = AttackDetector::new();
|
||||
for _ in 0..10 {
|
||||
d.analyze(1.0, 1.0);
|
||||
}
|
||||
d.analyze(10_000.0, 100.0);
|
||||
d.analyze(10_000.0, 100.0);
|
||||
assert_eq!(d.analyze(10_000.0, 100.0), AttackStatus::UnderAttack);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_consecutive_resets_on_normal() {
|
||||
let mut d = AttackDetector::new();
|
||||
for _ in 0..10 {
|
||||
d.analyze(1.0, 1.0);
|
||||
}
|
||||
d.analyze(100.0, 10.0);
|
||||
d.analyze(100.0, 10.0);
|
||||
d.analyze(1.0, 1.0);
|
||||
assert_eq!(d.analyze(1.0, 1.0), AttackStatus::Normal);
|
||||
}
|
||||
}
|
||||
|
||||
/// Ratio of a window to its baseline, capped at [`MAX_ANOMALY_RATIO`].
|
||||
///
|
||||
/// Returns zero for an unset baseline and for non-finite windows, so a broken
|
||||
/// counter cannot pin the detector in the attack state.
|
||||
fn anomaly_ratio(sample: f64, baseline: f64) -> f64 {
|
||||
if !sample.is_finite() || baseline <= 0.0 {
|
||||
return 0.0;
|
||||
}
|
||||
(sample.max(0.0) / baseline).min(MAX_ANOMALY_RATIO)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,29 @@
|
|||
//! Time-aware exponential moving average used by the traffic profilers.
|
||||
//!
|
||||
//! The average is expressed in samples per unit time and is fed once per
|
||||
//! metrics window by [`crate::traffic::profiler::TrafficProfiler`].
|
||||
|
||||
use std::time::Instant;
|
||||
|
||||
/// Smoothing factor used for `alpha` values outside the valid `(0, 1]` range
|
||||
/// and for `NaN`, so a malformed configuration cannot freeze the baseline.
|
||||
const DEFAULT_ALPHA: f64 = 0.125;
|
||||
|
||||
/// Exponential moving average with a wall-clock aware step count.
|
||||
///
|
||||
/// # Formula
|
||||
///
|
||||
/// For a sample `x` that arrives `dt` seconds after the previous update,
|
||||
/// `steps = max(dt, 1)` and `weight = (1 - alpha)^steps`. The average is then
|
||||
///
|
||||
/// ```text
|
||||
/// value = value * weight + x * (1 - weight)
|
||||
/// ```
|
||||
///
|
||||
/// A larger `alpha` gives more weight to the newest sample; `alpha = 1` makes
|
||||
/// the average equal to the last finite sample. The step count keeps the
|
||||
/// smoothing independent of the tick rate: one update after a two-second gap
|
||||
/// is worth two one-second updates.
|
||||
pub struct Ewma {
|
||||
value: f64,
|
||||
last: Instant,
|
||||
|
|
@ -7,51 +31,59 @@ pub struct Ewma {
|
|||
}
|
||||
|
||||
impl Ewma {
|
||||
/// Builds a smoothed average that starts at zero.
|
||||
///
|
||||
/// `alpha` is clamped to `(0, 1]`; `NaN`, infinity and non-positive values
|
||||
/// fall back to the default 0.125 instead of silently disabling the
|
||||
/// smoothing or producing a non-finite baseline.
|
||||
#[must_use]
|
||||
pub fn new(alpha: f64) -> Self {
|
||||
Self {
|
||||
value: 0.0,
|
||||
last: Instant::now(),
|
||||
alpha,
|
||||
alpha: sanitize_alpha(alpha),
|
||||
}
|
||||
}
|
||||
|
||||
/// Folds `sample` into the average, following the formula in the type
|
||||
/// documentation.
|
||||
///
|
||||
/// Non-finite samples are ignored (the update timestamp is not advanced
|
||||
/// either), so one broken counter reading cannot poison the baseline.
|
||||
pub fn update(&mut self, sample: f64) {
|
||||
if !sample.is_finite() {
|
||||
return;
|
||||
}
|
||||
let now = Instant::now();
|
||||
let elapsed = now.duration_since(self.last).as_secs_f64();
|
||||
let steps = elapsed.max(1.0);
|
||||
let steps = now.duration_since(self.last).as_secs_f64().max(1.0);
|
||||
let weight = (1.0 - self.alpha).powf(steps);
|
||||
self.value = self.value * weight + sample * (1.0 - weight);
|
||||
self.last = now;
|
||||
}
|
||||
|
||||
/// Replaces the average with `value` and restarts the step timer.
|
||||
///
|
||||
/// Used to prime a baseline from the first observed window instead of
|
||||
/// climbing from zero, which would otherwise read as an anomaly.
|
||||
pub fn seed(&mut self, value: f64) {
|
||||
if !value.is_finite() {
|
||||
return;
|
||||
}
|
||||
self.value = value;
|
||||
self.last = Instant::now();
|
||||
}
|
||||
|
||||
/// Current smoothed value.
|
||||
#[must_use]
|
||||
pub fn value(&self) -> f64 {
|
||||
self.value
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_ewma_initial() {
|
||||
let ewma = Ewma::new(0.125);
|
||||
assert_eq!(ewma.value(), 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ewma_update() {
|
||||
let mut ewma = Ewma::new(1.0);
|
||||
ewma.update(100.0);
|
||||
assert!((ewma.value() - 100.0).abs() < 1.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ewma_convergence() {
|
||||
let mut ewma = Ewma::new(0.5);
|
||||
for _ in 0..10 {
|
||||
ewma.update(50.0);
|
||||
}
|
||||
assert!((ewma.value() - 50.0).abs() < 1.0);
|
||||
fn sanitize_alpha(alpha: f64) -> f64 {
|
||||
if alpha.is_finite() && alpha > 0.0 {
|
||||
alpha.min(1.0)
|
||||
} else {
|
||||
DEFAULT_ALPHA
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,9 +1,79 @@
|
|||
use dashmap::DashMap;
|
||||
use std::net::IpAddr;
|
||||
use std::sync::Arc;
|
||||
//! Per-source reputation with time decay and idle-TTL eviction.
|
||||
//!
|
||||
//! A source starts neutral (`0`); [`IpReputation::record_bad`] subtracts ten
|
||||
//! points and [`IpReputation::record_good`] adds one, clamped to `[-100, 100]`.
|
||||
//! Scores decay toward neutral on read (half of the magnitude per thirty
|
||||
//! minutes of inactivity), so one old success cannot keep a source trusted
|
||||
//! forever and one old failure cannot keep it toxic.
|
||||
//!
|
||||
//! State lives in sixty-four locked shards. [`IpReputation::sweep`] visits
|
||||
//! eight of them per call and drops entries idle for longer than ten minutes;
|
||||
//! recording advances one step per minute, so the table is reclaimed without
|
||||
//! an external scheduler and no full-map `retain`.
|
||||
//!
|
||||
//! [`IpReputation::offenders_below`] returns at most 64 of the worst sources,
|
||||
//! so one escalation cannot flood the ban list or the log.
|
||||
|
||||
use std::collections::hash_map::RandomState;
|
||||
use std::collections::{BinaryHeap, HashMap};
|
||||
use std::hash::BuildHasher;
|
||||
use std::net::IpAddr;
|
||||
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
|
||||
use std::sync::{Mutex, MutexGuard};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Idle age after which a source is forgotten.
|
||||
const IDLE_TTL: Duration = Duration::from_secs(600);
|
||||
/// Time after which a dead source's score is halved.
|
||||
const DECAY_HALF_LIFE: Duration = Duration::from_secs(1800);
|
||||
/// Minimum interval between two automatic sweep steps.
|
||||
const SWEEP_INTERVAL: Duration = Duration::from_secs(60);
|
||||
/// Number of independently locked score shards.
|
||||
const SHARDS: usize = 64;
|
||||
/// Shards examined by one [`IpReputation::sweep`] call.
|
||||
const SWEEP_SHARDS_PER_TICK: usize = 8;
|
||||
/// Upper bound of a single offender batch.
|
||||
const MAX_OFFENDERS_PER_CYCLE: usize = 64;
|
||||
const GOOD_STEP: i32 = 1;
|
||||
const BAD_STEP: i32 = -10;
|
||||
const MAX_SCORE: i32 = 100;
|
||||
const MIN_SCORE: i32 = -100;
|
||||
|
||||
/// Tunables of [`IpReputation`].
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct ReputationPolicy {
|
||||
/// Idle age after which a source is forgotten.
|
||||
pub idle_ttl: Duration,
|
||||
/// Time after which a dead source's score is halved.
|
||||
pub decay_half_life: Duration,
|
||||
/// Upper bound of a single [`IpReputation::offenders_below`] batch.
|
||||
pub max_offenders: usize,
|
||||
}
|
||||
|
||||
impl Default for ReputationPolicy {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
idle_ttl: IDLE_TTL,
|
||||
decay_half_life: DECAY_HALF_LIFE,
|
||||
max_offenders: MAX_OFFENDERS_PER_CYCLE,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Reputation entry: the raw score and when it was last touched.
|
||||
struct Entry {
|
||||
score: i32,
|
||||
last_update: Instant,
|
||||
}
|
||||
|
||||
/// Reputation scores of the sources seen by the edge node.
|
||||
pub struct IpReputation {
|
||||
scores: Arc<DashMap<IpAddr, i32>>,
|
||||
shards: Box<[Mutex<HashMap<IpAddr, Entry>>]>,
|
||||
hasher: RandomState,
|
||||
policy: ReputationPolicy,
|
||||
epoch: Instant,
|
||||
last_sweep: AtomicU64,
|
||||
sweep_cursor: AtomicUsize,
|
||||
}
|
||||
|
||||
impl Default for IpReputation {
|
||||
|
|
@ -13,129 +83,166 @@ impl Default for IpReputation {
|
|||
}
|
||||
|
||||
impl IpReputation {
|
||||
/// Builds a reputation table with the production policy.
|
||||
#[must_use]
|
||||
pub fn new() -> Self {
|
||||
Self::with_policy(ReputationPolicy::default())
|
||||
}
|
||||
|
||||
/// Builds a reputation table with an explicit policy.
|
||||
pub fn with_policy(policy: ReputationPolicy) -> Self {
|
||||
Self {
|
||||
scores: Arc::new(DashMap::new()),
|
||||
shards: (0..SHARDS).map(|_| Mutex::new(HashMap::new())).collect(),
|
||||
hasher: RandomState::new(),
|
||||
policy,
|
||||
epoch: Instant::now(),
|
||||
last_sweep: AtomicU64::new(0),
|
||||
sweep_cursor: AtomicUsize::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
/// Records a successful connection.
|
||||
pub fn record_good(&self, ip: IpAddr) {
|
||||
let mut entry = self.scores.entry(ip).or_insert(0);
|
||||
*entry = (*entry + 1).min(100);
|
||||
self.sweep_if_due();
|
||||
self.adjust(ip, GOOD_STEP);
|
||||
}
|
||||
|
||||
/// Records a failed or abusive connection.
|
||||
pub fn record_bad(&self, ip: IpAddr) {
|
||||
let mut entry = self.scores.entry(ip).or_insert(0);
|
||||
*entry = (*entry - 10).max(-100);
|
||||
self.sweep_if_due();
|
||||
self.adjust(ip, BAD_STEP);
|
||||
}
|
||||
|
||||
/// Current decayed score of `ip`; `0` for unknown or idle sources.
|
||||
pub fn score(&self, ip: IpAddr) -> i32 {
|
||||
self.scores.get(&ip).map(|v| *v).unwrap_or(0)
|
||||
let now = Instant::now();
|
||||
let index = self.shard_index(ip);
|
||||
let shard = self.lock(index);
|
||||
let Some(entry) = shard.get(&ip) else { return 0 };
|
||||
let idle = now.duration_since(entry.last_update);
|
||||
if idle > self.policy.idle_ttl {
|
||||
return 0;
|
||||
}
|
||||
decayed_score(entry.score, idle, self.policy.decay_half_life)
|
||||
}
|
||||
|
||||
/// Returns the sources scored at or below `threshold` (the offenders).
|
||||
/// Returns up to [`ReputationPolicy::max_offenders`] of the worst sources
|
||||
/// scored at or below `threshold`.
|
||||
///
|
||||
/// The result is a snapshot copy, so it is safe to ban from it while other
|
||||
/// threads keep recording events.
|
||||
/// The snapshot is safe to ban from while other threads keep recording
|
||||
/// events. When more offenders exist than the batch cap, exactly one
|
||||
/// aggregated warning carries the total count.
|
||||
pub fn offenders_below(&self, threshold: i32) -> Vec<IpAddr> {
|
||||
self.scores
|
||||
.iter()
|
||||
.filter(|entry| *entry.value() <= threshold)
|
||||
.map(|entry| *entry.key())
|
||||
.collect()
|
||||
let now = Instant::now();
|
||||
let limit = self.policy.max_offenders;
|
||||
let mut worst: BinaryHeap<(i32, IpAddr)> = BinaryHeap::new();
|
||||
let mut found = 0usize;
|
||||
for index in 0..self.shards.len() {
|
||||
let shard = self.lock(index);
|
||||
for (ip, entry) in shard.iter() {
|
||||
let idle = now.duration_since(entry.last_update);
|
||||
if idle > self.policy.idle_ttl {
|
||||
continue;
|
||||
}
|
||||
let score = decayed_score(entry.score, idle, self.policy.decay_half_life);
|
||||
if score > threshold {
|
||||
continue;
|
||||
}
|
||||
found += 1;
|
||||
worst.push((score, *ip));
|
||||
if worst.len() > limit {
|
||||
worst.pop();
|
||||
}
|
||||
}
|
||||
}
|
||||
let offenders: Vec<IpAddr> = worst.into_iter().map(|(_, ip)| ip).collect();
|
||||
if found > offenders.len() {
|
||||
tracing::warn!(
|
||||
found,
|
||||
returned = offenders.len(),
|
||||
"reputation offender batch truncated for this cycle"
|
||||
);
|
||||
}
|
||||
offenders
|
||||
}
|
||||
|
||||
/// Whether the source is above the trusted score.
|
||||
pub fn is_trusted(&self, ip: IpAddr) -> bool {
|
||||
self.score(ip) > 50
|
||||
}
|
||||
|
||||
/// Number of live entries.
|
||||
pub fn len(&self) -> usize {
|
||||
(0..self.shards.len()).map(|index| self.lock(index).len()).sum()
|
||||
}
|
||||
|
||||
/// Whether the table is empty.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.len() == 0
|
||||
}
|
||||
|
||||
/// Evicts idle entries from the next shard slice; returns the count.
|
||||
pub fn sweep(&self) -> usize {
|
||||
let now = Instant::now();
|
||||
let shard_count = self.shards.len();
|
||||
let start = self.sweep_cursor.fetch_add(SWEEP_SHARDS_PER_TICK, Ordering::Relaxed);
|
||||
let mut removed = 0;
|
||||
for offset in 0..SWEEP_SHARDS_PER_TICK.min(shard_count) {
|
||||
let index = (start + offset) % shard_count;
|
||||
let mut shard = self.lock(index);
|
||||
let before = shard.len();
|
||||
shard.retain(|_, entry| now.duration_since(entry.last_update) <= self.policy.idle_ttl);
|
||||
removed += before - shard.len();
|
||||
}
|
||||
removed
|
||||
}
|
||||
|
||||
fn adjust(&self, ip: IpAddr, delta: i32) {
|
||||
let now = Instant::now();
|
||||
let index = self.shard_index(ip);
|
||||
let mut shard = self.lock(index);
|
||||
let entry = shard.entry(ip).or_insert(Entry {
|
||||
score: 0,
|
||||
last_update: now,
|
||||
});
|
||||
let idle = now.duration_since(entry.last_update);
|
||||
let decayed = decayed_score(entry.score, idle, self.policy.decay_half_life);
|
||||
entry.score = (decayed + delta).clamp(MIN_SCORE, MAX_SCORE);
|
||||
entry.last_update = now;
|
||||
}
|
||||
|
||||
/// Advances one sweep step when [`SWEEP_INTERVAL`] has elapsed, so eviction
|
||||
/// does not depend on an external timer.
|
||||
fn sweep_if_due(&self) {
|
||||
let elapsed = self.epoch.elapsed().as_secs();
|
||||
let last = self.last_sweep.load(Ordering::Relaxed);
|
||||
if elapsed.saturating_sub(last) < SWEEP_INTERVAL.as_secs() {
|
||||
return;
|
||||
}
|
||||
let advanced = self
|
||||
.last_sweep
|
||||
.compare_exchange(last, elapsed, Ordering::AcqRel, Ordering::Relaxed);
|
||||
if advanced.is_ok() {
|
||||
self.sweep();
|
||||
}
|
||||
}
|
||||
|
||||
fn shard_index(&self, ip: IpAddr) -> usize {
|
||||
(self.hasher.hash_one(ip) as usize) % self.shards.len()
|
||||
}
|
||||
|
||||
fn lock(&self, index: usize) -> MutexGuard<'_, HashMap<IpAddr, Entry>> {
|
||||
// A poisoned shard still holds consistent scores; keep serving.
|
||||
self.shards[index].lock().unwrap_or_else(|poison| poison.into_inner())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
|
||||
|
||||
fn v4(octets: [u8; 4]) -> IpAddr {
|
||||
IpAddr::V4(Ipv4Addr::from(octets))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reputation_initial_score() {
|
||||
let rep = IpReputation::new();
|
||||
assert_eq!(rep.score(v4([192, 168, 1, 1])), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reputation_good() {
|
||||
let rep = IpReputation::new();
|
||||
let ip = v4([10, 0, 0, 1]);
|
||||
rep.record_good(ip);
|
||||
assert_eq!(rep.score(ip), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reputation_bad() {
|
||||
let rep = IpReputation::new();
|
||||
let ip = v4([10, 0, 0, 2]);
|
||||
rep.record_bad(ip);
|
||||
assert_eq!(rep.score(ip), -10);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reputation_cap_positive() {
|
||||
let rep = IpReputation::new();
|
||||
let ip = v4([10, 0, 0, 3]);
|
||||
for _ in 0..200 {
|
||||
rep.record_good(ip);
|
||||
}
|
||||
assert_eq!(rep.score(ip), 100);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reputation_cap_negative() {
|
||||
let rep = IpReputation::new();
|
||||
let ip = v4([10, 0, 0, 4]);
|
||||
for _ in 0..20 {
|
||||
rep.record_bad(ip);
|
||||
}
|
||||
assert_eq!(rep.score(ip), -100);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_trusted() {
|
||||
let rep = IpReputation::new();
|
||||
let ip = v4([10, 0, 0, 5]);
|
||||
assert!(!rep.is_trusted(ip));
|
||||
for _ in 0..51 {
|
||||
rep.record_good(ip);
|
||||
}
|
||||
assert!(rep.is_trusted(ip));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reputation_v6() {
|
||||
let rep = IpReputation::new();
|
||||
let ip = IpAddr::V6(Ipv6Addr::LOCALHOST);
|
||||
rep.record_bad(ip);
|
||||
assert_eq!(rep.score(ip), -10);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_offenders_below_threshold() {
|
||||
let rep = IpReputation::new();
|
||||
let bad = v4([10, 0, 0, 1]);
|
||||
let good = v4([10, 0, 0, 2]);
|
||||
for _ in 0..6 {
|
||||
rep.record_bad(bad);
|
||||
}
|
||||
rep.record_good(good);
|
||||
let offenders = rep.offenders_below(-50);
|
||||
assert_eq!(offenders, vec![bad]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_offenders_empty_when_none_below() {
|
||||
let rep = IpReputation::new();
|
||||
assert!(rep.offenders_below(-50).is_empty());
|
||||
fn decayed_score(score: i32, idle: Duration, half_life: Duration) -> i32 {
|
||||
if score == 0 || half_life.is_zero() {
|
||||
return 0;
|
||||
}
|
||||
let factor = 0.5f64.powf(idle.as_secs_f64() / half_life.as_secs_f64());
|
||||
// Rounding (instead of truncation) keeps an immediate second update from
|
||||
// decaying a score of `1` back to zero on the microsecond scale.
|
||||
(f64::from(score) * factor).round() as i32
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,17 +17,22 @@ pub struct PrefixKey {
|
|||
impl PrefixKey {
|
||||
/// Builds the prefix key for an address (v4 maps to /24 in the first four
|
||||
/// bytes, v6 maps to /64).
|
||||
///
|
||||
/// The address is normalised through [`prefix_of`] first, so an
|
||||
/// IPv4-mapped IPv6 source (`::ffff:a.b.c.d`) produces exactly the same key
|
||||
/// as the plain IPv4 address. XDP `prefix_stats` entries and the user-space
|
||||
/// tracker therefore agree on the bucket of a dual-stack client.
|
||||
#[must_use]
|
||||
pub fn new(ip: IpAddr) -> Self {
|
||||
let mut key = Self::default();
|
||||
match ip {
|
||||
match prefix_of(ip) {
|
||||
IpAddr::V4(v4) => {
|
||||
key.family = PREFIX_FAMILY_V4;
|
||||
key.addr[..4].copy_from_slice(&masked_v4(v4).octets());
|
||||
key.addr[..4].copy_from_slice(&v4.octets());
|
||||
},
|
||||
IpAddr::V6(v6) => {
|
||||
key.family = PREFIX_FAMILY_V6;
|
||||
key.addr.copy_from_slice(&masked_v6_segments(v6));
|
||||
key.addr.copy_from_slice(&v6.octets());
|
||||
},
|
||||
}
|
||||
key
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue