feat: sync edge autobans across nodes

This commit is contained in:
Loki 2026-10-16 23:09:26 +02:00
parent d37cb0926f
commit 8cbd86c98a
Signed by: boba
GPG key ID: 253067914055423B
8 changed files with 189 additions and 4 deletions

View file

@ -85,6 +85,12 @@ cargo run --release --bin rampart-manager
docker compose up -d redis clickhouse grafana
```
Для нескольких edge и Manager укажи один и тот же защищённый Redis endpoint в
`store.redis_url` каждого edge и `REDIS_URL` Manager. Edge передаёт туда
локальные автобаны и восстанавливает общие bans после переподключения. Открой
Redis только через private network/VPN; при пустом `redis_url` edge остаётся
автономным, и автобаны между узлами не распространяются.
## 5. Docker
```bash

View file

@ -7,7 +7,7 @@
```
Redis упал:
Edge: продолжает автономно на локальном кэше блэклиста
Manager: API отдаёт пустые списки / ошибки → рестарт Redis
Manager: Redis-зависимые API отвечают 503 → восстановить Redis
Manager упал:
Edge: продолжает фильтрацию (фильтрация не зависит от Manager)
@ -20,7 +20,7 @@ Edge нода упала:
Полный сбой дата-центра:
Edge в других ДЦ продолжают работу
Баны/ноды не синхронизируются до восстановления Redis
Общие баны/ноды не синхронизируются до восстановления Redis
```
---

View file

@ -93,6 +93,19 @@ pub async fn run() -> anyhow::Result<RunExit> {
return;
},
};
let local_events = bl.subscribe_local_bans();
let publisher_client = client.clone();
let publisher_blacklist = bl.clone();
let publisher_shutdown = sd.clone();
tokio::spawn(async move {
crate::store::start_local_ban_publisher(
publisher_client,
publisher_blacklist,
local_events,
publisher_shutdown,
)
.await;
});
crate::store::start_blacklist_sync(&client, bl, sd).await;
});
}

View file

@ -2,6 +2,16 @@ use dashmap::DashMap;
use std::net::IpAddr;
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::sync::broadcast;
const LOCAL_BAN_EVENT_CAPACITY: usize = 4096;
#[derive(Clone, Debug)]
pub struct LocalBanEvent {
pub ip: IpAddr,
pub expires_at_unix: i64,
pub reason: String,
}
struct BanEntry {
expires: Instant,
@ -11,6 +21,7 @@ struct BanEntry {
pub struct Blacklist {
entries: Arc<DashMap<IpAddr, BanEntry>>,
redis_entries: Arc<DashMap<IpAddr, BanEntry>>,
local_ban_events: broadcast::Sender<LocalBanEvent>,
}
impl Default for Blacklist {
@ -21,9 +32,11 @@ impl Default for Blacklist {
impl Blacklist {
pub fn new() -> Self {
let (local_ban_events, _) = broadcast::channel(LOCAL_BAN_EVENT_CAPACITY);
Self {
entries: Arc::new(DashMap::new()),
redis_entries: Arc::new(DashMap::new()),
local_ban_events,
}
}
@ -32,6 +45,11 @@ impl Blacklist {
}
pub fn add(&self, ip: IpAddr, duration: Duration, reason: &str) {
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_secs();
let expires_at_unix = now.saturating_add(duration.as_secs()).min(i64::MAX as u64) as i64;
self.entries.insert(
ip,
BanEntry {
@ -39,6 +57,32 @@ impl Blacklist {
_reason: reason.to_string(),
},
);
let _ = self.local_ban_events.send(LocalBanEvent {
ip,
expires_at_unix,
reason: reason.to_string(),
});
}
pub fn subscribe_local_bans(&self) -> broadcast::Receiver<LocalBanEvent> {
self.local_ban_events.subscribe()
}
pub fn local_snapshot(&self) -> Vec<LocalBanEvent> {
self.entries
.iter()
.filter(|entry| entry.expires > Instant::now())
.map(|entry| LocalBanEvent {
ip: *entry.key(),
expires_at_unix: std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
.saturating_add(entry.expires.saturating_duration_since(Instant::now()).as_secs())
.min(i64::MAX as u64) as i64,
reason: entry._reason.clone(),
})
.collect()
}
pub fn remove(&self, ip: IpAddr) {

View file

@ -1,7 +1,7 @@
#[cfg(feature = "store-redis")]
pub mod redis;
#[cfg(feature = "store-redis")]
pub use redis::start_blacklist_sync;
pub use redis::{start_blacklist_sync, start_local_ban_publisher};
pub mod clickhouse;

View file

@ -2,7 +2,7 @@ use crate::store::StateStore;
use redis::AsyncCommands;
mod sync;
pub use sync::start_blacklist_sync;
pub use sync::{start_blacklist_sync, start_local_ban_publisher};
pub struct RedisStore {
client: redis::Client,

View file

@ -1,6 +1,9 @@
mod event;
mod publisher;
mod snapshot;
pub use publisher::start_local_ban_publisher;
use crate::filter::blacklist::Blacklist;
use futures::StreamExt;
use std::sync::Arc;

View file

@ -0,0 +1,119 @@
use crate::filter::blacklist::{Blacklist, LocalBanEvent};
use serde::Serialize;
use std::time::Duration;
use tokio::sync::{broadcast, watch};
const BLACKLIST_KEY: &str = "rampart:blacklist";
const BLACKLIST_METADATA_KEY: &str = "rampart:blacklist:metadata";
const BLACKLIST_CHANNEL: &str = "rampart:blacklist:events";
const STORE_BAN_SCRIPT: &str = "redis.call('HSET', KEYS[2], ARGV[1], ARGV[2]); redis.call('SADD', KEYS[1], ARGV[1]); redis.call('PUBLISH', KEYS[3], ARGV[3]); return 1";
#[derive(Serialize)]
struct BanMetadata<'a> {
created_at_unix: i64,
expires_at_unix: i64,
reason: &'a str,
}
#[derive(Serialize)]
struct BanEvent<'a> {
ip: String,
action: &'static str,
duration_secs: u64,
expires_at_unix: i64,
reason: &'a str,
}
pub async fn start_local_ban_publisher(
client: redis::Client,
blacklist: std::sync::Arc<Blacklist>,
mut events: broadcast::Receiver<LocalBanEvent>,
mut shutdown: watch::Receiver<bool>,
) {
for event in blacklist.local_snapshot() {
if !publish_with_retry(&client, event, &mut shutdown).await {
return;
}
}
loop {
tokio::select! {
received = events.recv() => match received {
Ok(event) => {
if !publish_with_retry(&client, event, &mut shutdown).await {
return;
}
},
Err(broadcast::error::RecvError::Lagged(skipped)) => {
tracing::warn!(skipped, "local ban publisher lagged; reconciling active local bans");
for event in blacklist.local_snapshot() {
if !publish_with_retry(&client, event, &mut shutdown).await {
return;
}
}
},
Err(broadcast::error::RecvError::Closed) => return,
},
changed = shutdown.changed() => {
if changed.is_err() || *shutdown.borrow_and_update() {
return;
}
},
}
}
}
async fn publish_with_retry(
client: &redis::Client,
event: LocalBanEvent,
shutdown: &mut watch::Receiver<bool>,
) -> bool {
let mut delay = Duration::from_secs(1);
loop {
match publish_ban(client, &event).await {
Ok(()) => return true,
Err(error) => tracing::warn!(%error, ip = %event.ip, "failed to persist local ban; retrying"),
}
tokio::select! {
_ = tokio::time::sleep(delay) => delay = (delay * 2).min(Duration::from_secs(30)),
changed = shutdown.changed() => {
if changed.is_err() || *shutdown.borrow_and_update() {
return false;
}
},
}
}
}
async fn publish_ban(client: &redis::Client, event: &LocalBanEvent) -> anyhow::Result<()> {
let now = chrono::Utc::now().timestamp();
let remaining = event.expires_at_unix.saturating_sub(now);
if remaining <= 0 {
return Ok(());
}
let metadata = serde_json::to_string(&BanMetadata {
created_at_unix: now,
expires_at_unix: event.expires_at_unix,
reason: &event.reason,
})?;
let payload = serde_json::to_string(&BanEvent {
ip: event.ip.to_string(),
action: "ban",
duration_secs: u64::try_from(remaining)?,
expires_at_unix: event.expires_at_unix,
reason: &event.reason,
})?;
let mut conn = client.get_multiplexed_async_connection().await?;
let _: i64 = redis::cmd("EVAL")
.arg(STORE_BAN_SCRIPT)
.arg(3)
.arg(BLACKLIST_KEY)
.arg(BLACKLIST_METADATA_KEY)
.arg(BLACKLIST_CHANNEL)
.arg(event.ip.to_string())
.arg(metadata)
.arg(payload)
.query_async(&mut conn)
.await?;
Ok(())
}