From 8cbd86c98a00e3796c5374d33e33f87dd10aafe6 Mon Sep 17 00:00:00 2001 From: loki5512344 Date: Fri, 16 Oct 2026 23:09:26 +0200 Subject: [PATCH] feat: sync edge autobans across nodes --- docs/deployment.md | 6 ++ docs/disaster_recovery.md | 4 +- src/app/runtime.rs | 13 ++++ src/filter/blacklist.rs | 44 +++++++++++ src/store/mod.rs | 2 +- src/store/redis.rs | 2 +- src/store/redis/sync.rs | 3 + src/store/redis/sync/publisher.rs | 119 ++++++++++++++++++++++++++++++ 8 files changed, 189 insertions(+), 4 deletions(-) create mode 100644 src/store/redis/sync/publisher.rs diff --git a/docs/deployment.md b/docs/deployment.md index ca23459..a8c544a 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -85,6 +85,12 @@ cargo run --release --bin rampart-manager docker compose up -d redis clickhouse grafana ``` +Для нескольких edge и Manager укажи один и тот же защищённый Redis endpoint в +`store.redis_url` каждого edge и `REDIS_URL` Manager. Edge передаёт туда +локальные автобаны и восстанавливает общие bans после переподключения. Открой +Redis только через private network/VPN; при пустом `redis_url` edge остаётся +автономным, и автобаны между узлами не распространяются. + ## 5. Docker ```bash diff --git a/docs/disaster_recovery.md b/docs/disaster_recovery.md index fef48da..f49c69d 100644 --- a/docs/disaster_recovery.md +++ b/docs/disaster_recovery.md @@ -7,7 +7,7 @@ ``` Redis упал: Edge: продолжает автономно на локальном кэше блэклиста - Manager: API отдаёт пустые списки / ошибки → рестарт Redis + Manager: Redis-зависимые API отвечают 503 → восстановить Redis Manager упал: Edge: продолжает фильтрацию (фильтрация не зависит от Manager) @@ -20,7 +20,7 @@ Edge нода упала: Полный сбой дата-центра: Edge в других ДЦ продолжают работу - Баны/ноды не синхронизируются до восстановления Redis + Общие баны/ноды не синхронизируются до восстановления Redis ``` --- diff --git a/src/app/runtime.rs b/src/app/runtime.rs index 1318c72..238f8c9 100644 --- a/src/app/runtime.rs +++ b/src/app/runtime.rs @@ -93,6 +93,19 @@ pub async fn run() -> anyhow::Result { return; }, }; + let local_events = bl.subscribe_local_bans(); + let publisher_client = client.clone(); + let publisher_blacklist = bl.clone(); + let publisher_shutdown = sd.clone(); + tokio::spawn(async move { + crate::store::start_local_ban_publisher( + publisher_client, + publisher_blacklist, + local_events, + publisher_shutdown, + ) + .await; + }); crate::store::start_blacklist_sync(&client, bl, sd).await; }); } diff --git a/src/filter/blacklist.rs b/src/filter/blacklist.rs index 2ac1f14..bc68192 100644 --- a/src/filter/blacklist.rs +++ b/src/filter/blacklist.rs @@ -2,6 +2,16 @@ use dashmap::DashMap; use std::net::IpAddr; use std::sync::Arc; use std::time::{Duration, Instant}; +use tokio::sync::broadcast; + +const LOCAL_BAN_EVENT_CAPACITY: usize = 4096; + +#[derive(Clone, Debug)] +pub struct LocalBanEvent { + pub ip: IpAddr, + pub expires_at_unix: i64, + pub reason: String, +} struct BanEntry { expires: Instant, @@ -11,6 +21,7 @@ struct BanEntry { pub struct Blacklist { entries: Arc>, redis_entries: Arc>, + local_ban_events: broadcast::Sender, } impl Default for Blacklist { @@ -21,9 +32,11 @@ impl Default for Blacklist { impl Blacklist { pub fn new() -> Self { + let (local_ban_events, _) = broadcast::channel(LOCAL_BAN_EVENT_CAPACITY); Self { entries: Arc::new(DashMap::new()), redis_entries: Arc::new(DashMap::new()), + local_ban_events, } } @@ -32,6 +45,11 @@ impl Blacklist { } pub fn add(&self, ip: IpAddr, duration: Duration, reason: &str) { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + let expires_at_unix = now.saturating_add(duration.as_secs()).min(i64::MAX as u64) as i64; self.entries.insert( ip, BanEntry { @@ -39,6 +57,32 @@ impl Blacklist { _reason: reason.to_string(), }, ); + let _ = self.local_ban_events.send(LocalBanEvent { + ip, + expires_at_unix, + reason: reason.to_string(), + }); + } + + pub fn subscribe_local_bans(&self) -> broadcast::Receiver { + self.local_ban_events.subscribe() + } + + pub fn local_snapshot(&self) -> Vec { + self.entries + .iter() + .filter(|entry| entry.expires > Instant::now()) + .map(|entry| LocalBanEvent { + ip: *entry.key(), + expires_at_unix: std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs() + .saturating_add(entry.expires.saturating_duration_since(Instant::now()).as_secs()) + .min(i64::MAX as u64) as i64, + reason: entry._reason.clone(), + }) + .collect() } pub fn remove(&self, ip: IpAddr) { diff --git a/src/store/mod.rs b/src/store/mod.rs index 77dc4be..f731027 100644 --- a/src/store/mod.rs +++ b/src/store/mod.rs @@ -1,7 +1,7 @@ #[cfg(feature = "store-redis")] pub mod redis; #[cfg(feature = "store-redis")] -pub use redis::start_blacklist_sync; +pub use redis::{start_blacklist_sync, start_local_ban_publisher}; pub mod clickhouse; diff --git a/src/store/redis.rs b/src/store/redis.rs index 18a3af7..19f2b3e 100644 --- a/src/store/redis.rs +++ b/src/store/redis.rs @@ -2,7 +2,7 @@ use crate::store::StateStore; use redis::AsyncCommands; mod sync; -pub use sync::start_blacklist_sync; +pub use sync::{start_blacklist_sync, start_local_ban_publisher}; pub struct RedisStore { client: redis::Client, diff --git a/src/store/redis/sync.rs b/src/store/redis/sync.rs index 63eb73f..2728d40 100644 --- a/src/store/redis/sync.rs +++ b/src/store/redis/sync.rs @@ -1,6 +1,9 @@ mod event; +mod publisher; mod snapshot; +pub use publisher::start_local_ban_publisher; + use crate::filter::blacklist::Blacklist; use futures::StreamExt; use std::sync::Arc; diff --git a/src/store/redis/sync/publisher.rs b/src/store/redis/sync/publisher.rs new file mode 100644 index 0000000..8508455 --- /dev/null +++ b/src/store/redis/sync/publisher.rs @@ -0,0 +1,119 @@ +use crate::filter::blacklist::{Blacklist, LocalBanEvent}; +use serde::Serialize; +use std::time::Duration; +use tokio::sync::{broadcast, watch}; + +const BLACKLIST_KEY: &str = "rampart:blacklist"; +const BLACKLIST_METADATA_KEY: &str = "rampart:blacklist:metadata"; +const BLACKLIST_CHANNEL: &str = "rampart:blacklist:events"; +const STORE_BAN_SCRIPT: &str = "redis.call('HSET', KEYS[2], ARGV[1], ARGV[2]); redis.call('SADD', KEYS[1], ARGV[1]); redis.call('PUBLISH', KEYS[3], ARGV[3]); return 1"; + +#[derive(Serialize)] +struct BanMetadata<'a> { + created_at_unix: i64, + expires_at_unix: i64, + reason: &'a str, +} + +#[derive(Serialize)] +struct BanEvent<'a> { + ip: String, + action: &'static str, + duration_secs: u64, + expires_at_unix: i64, + reason: &'a str, +} + +pub async fn start_local_ban_publisher( + client: redis::Client, + blacklist: std::sync::Arc, + mut events: broadcast::Receiver, + mut shutdown: watch::Receiver, +) { + for event in blacklist.local_snapshot() { + if !publish_with_retry(&client, event, &mut shutdown).await { + return; + } + } + + loop { + tokio::select! { + received = events.recv() => match received { + Ok(event) => { + if !publish_with_retry(&client, event, &mut shutdown).await { + return; + } + }, + Err(broadcast::error::RecvError::Lagged(skipped)) => { + tracing::warn!(skipped, "local ban publisher lagged; reconciling active local bans"); + for event in blacklist.local_snapshot() { + if !publish_with_retry(&client, event, &mut shutdown).await { + return; + } + } + }, + Err(broadcast::error::RecvError::Closed) => return, + }, + changed = shutdown.changed() => { + if changed.is_err() || *shutdown.borrow_and_update() { + return; + } + }, + } + } +} + +async fn publish_with_retry( + client: &redis::Client, + event: LocalBanEvent, + shutdown: &mut watch::Receiver, +) -> bool { + let mut delay = Duration::from_secs(1); + loop { + match publish_ban(client, &event).await { + Ok(()) => return true, + Err(error) => tracing::warn!(%error, ip = %event.ip, "failed to persist local ban; retrying"), + } + tokio::select! { + _ = tokio::time::sleep(delay) => delay = (delay * 2).min(Duration::from_secs(30)), + changed = shutdown.changed() => { + if changed.is_err() || *shutdown.borrow_and_update() { + return false; + } + }, + } + } +} + +async fn publish_ban(client: &redis::Client, event: &LocalBanEvent) -> anyhow::Result<()> { + let now = chrono::Utc::now().timestamp(); + let remaining = event.expires_at_unix.saturating_sub(now); + if remaining <= 0 { + return Ok(()); + } + let metadata = serde_json::to_string(&BanMetadata { + created_at_unix: now, + expires_at_unix: event.expires_at_unix, + reason: &event.reason, + })?; + let payload = serde_json::to_string(&BanEvent { + ip: event.ip.to_string(), + action: "ban", + duration_secs: u64::try_from(remaining)?, + expires_at_unix: event.expires_at_unix, + reason: &event.reason, + })?; + let mut conn = client.get_multiplexed_async_connection().await?; + let _: i64 = redis::cmd("EVAL") + .arg(STORE_BAN_SCRIPT) + .arg(3) + .arg(BLACKLIST_KEY) + .arg(BLACKLIST_METADATA_KEY) + .arg(BLACKLIST_CHANNEL) + .arg(event.ip.to_string()) + .arg(metadata) + .arg(payload) + .query_async(&mut conn) + .await?; + Ok(()) +}