feat: sync edge autobans across nodes
This commit is contained in:
parent
d37cb0926f
commit
8cbd86c98a
8 changed files with 189 additions and 4 deletions
|
|
@ -85,6 +85,12 @@ cargo run --release --bin rampart-manager
|
|||
docker compose up -d redis clickhouse grafana
|
||||
```
|
||||
|
||||
Для нескольких edge и Manager укажи один и тот же защищённый Redis endpoint в
|
||||
`store.redis_url` каждого edge и `REDIS_URL` Manager. Edge передаёт туда
|
||||
локальные автобаны и восстанавливает общие bans после переподключения. Открой
|
||||
Redis только через private network/VPN; при пустом `redis_url` edge остаётся
|
||||
автономным, и автобаны между узлами не распространяются.
|
||||
|
||||
## 5. Docker
|
||||
|
||||
```bash
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
```
|
||||
Redis упал:
|
||||
Edge: продолжает автономно на локальном кэше блэклиста
|
||||
Manager: API отдаёт пустые списки / ошибки → рестарт Redis
|
||||
Manager: Redis-зависимые API отвечают 503 → восстановить Redis
|
||||
|
||||
Manager упал:
|
||||
Edge: продолжает фильтрацию (фильтрация не зависит от Manager)
|
||||
|
|
@ -20,7 +20,7 @@ Edge нода упала:
|
|||
|
||||
Полный сбой дата-центра:
|
||||
Edge в других ДЦ продолжают работу
|
||||
Баны/ноды не синхронизируются до восстановления Redis
|
||||
Общие баны/ноды не синхронизируются до восстановления Redis
|
||||
```
|
||||
|
||||
---
|
||||
|
|
|
|||
|
|
@ -93,6 +93,19 @@ pub async fn run() -> anyhow::Result<RunExit> {
|
|||
return;
|
||||
},
|
||||
};
|
||||
let local_events = bl.subscribe_local_bans();
|
||||
let publisher_client = client.clone();
|
||||
let publisher_blacklist = bl.clone();
|
||||
let publisher_shutdown = sd.clone();
|
||||
tokio::spawn(async move {
|
||||
crate::store::start_local_ban_publisher(
|
||||
publisher_client,
|
||||
publisher_blacklist,
|
||||
local_events,
|
||||
publisher_shutdown,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
crate::store::start_blacklist_sync(&client, bl, sd).await;
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,6 +2,16 @@ use dashmap::DashMap;
|
|||
use std::net::IpAddr;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::sync::broadcast;
|
||||
|
||||
const LOCAL_BAN_EVENT_CAPACITY: usize = 4096;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct LocalBanEvent {
|
||||
pub ip: IpAddr,
|
||||
pub expires_at_unix: i64,
|
||||
pub reason: String,
|
||||
}
|
||||
|
||||
struct BanEntry {
|
||||
expires: Instant,
|
||||
|
|
@ -11,6 +21,7 @@ struct BanEntry {
|
|||
pub struct Blacklist {
|
||||
entries: Arc<DashMap<IpAddr, BanEntry>>,
|
||||
redis_entries: Arc<DashMap<IpAddr, BanEntry>>,
|
||||
local_ban_events: broadcast::Sender<LocalBanEvent>,
|
||||
}
|
||||
|
||||
impl Default for Blacklist {
|
||||
|
|
@ -21,9 +32,11 @@ impl Default for Blacklist {
|
|||
|
||||
impl Blacklist {
|
||||
pub fn new() -> Self {
|
||||
let (local_ban_events, _) = broadcast::channel(LOCAL_BAN_EVENT_CAPACITY);
|
||||
Self {
|
||||
entries: Arc::new(DashMap::new()),
|
||||
redis_entries: Arc::new(DashMap::new()),
|
||||
local_ban_events,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -32,6 +45,11 @@ impl Blacklist {
|
|||
}
|
||||
|
||||
pub fn add(&self, ip: IpAddr, duration: Duration, reason: &str) {
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs();
|
||||
let expires_at_unix = now.saturating_add(duration.as_secs()).min(i64::MAX as u64) as i64;
|
||||
self.entries.insert(
|
||||
ip,
|
||||
BanEntry {
|
||||
|
|
@ -39,6 +57,32 @@ impl Blacklist {
|
|||
_reason: reason.to_string(),
|
||||
},
|
||||
);
|
||||
let _ = self.local_ban_events.send(LocalBanEvent {
|
||||
ip,
|
||||
expires_at_unix,
|
||||
reason: reason.to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
pub fn subscribe_local_bans(&self) -> broadcast::Receiver<LocalBanEvent> {
|
||||
self.local_ban_events.subscribe()
|
||||
}
|
||||
|
||||
pub fn local_snapshot(&self) -> Vec<LocalBanEvent> {
|
||||
self.entries
|
||||
.iter()
|
||||
.filter(|entry| entry.expires > Instant::now())
|
||||
.map(|entry| LocalBanEvent {
|
||||
ip: *entry.key(),
|
||||
expires_at_unix: std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs()
|
||||
.saturating_add(entry.expires.saturating_duration_since(Instant::now()).as_secs())
|
||||
.min(i64::MAX as u64) as i64,
|
||||
reason: entry._reason.clone(),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn remove(&self, ip: IpAddr) {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
#[cfg(feature = "store-redis")]
|
||||
pub mod redis;
|
||||
#[cfg(feature = "store-redis")]
|
||||
pub use redis::start_blacklist_sync;
|
||||
pub use redis::{start_blacklist_sync, start_local_ban_publisher};
|
||||
|
||||
pub mod clickhouse;
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ use crate::store::StateStore;
|
|||
use redis::AsyncCommands;
|
||||
|
||||
mod sync;
|
||||
pub use sync::start_blacklist_sync;
|
||||
pub use sync::{start_blacklist_sync, start_local_ban_publisher};
|
||||
|
||||
pub struct RedisStore {
|
||||
client: redis::Client,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
mod event;
|
||||
mod publisher;
|
||||
mod snapshot;
|
||||
|
||||
pub use publisher::start_local_ban_publisher;
|
||||
|
||||
use crate::filter::blacklist::Blacklist;
|
||||
use futures::StreamExt;
|
||||
use std::sync::Arc;
|
||||
|
|
|
|||
119
src/store/redis/sync/publisher.rs
Normal file
119
src/store/redis/sync/publisher.rs
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
use crate::filter::blacklist::{Blacklist, LocalBanEvent};
|
||||
use serde::Serialize;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::{broadcast, watch};
|
||||
|
||||
const BLACKLIST_KEY: &str = "rampart:blacklist";
|
||||
const BLACKLIST_METADATA_KEY: &str = "rampart:blacklist:metadata";
|
||||
const BLACKLIST_CHANNEL: &str = "rampart:blacklist:events";
|
||||
const STORE_BAN_SCRIPT: &str = "redis.call('HSET', KEYS[2], ARGV[1], ARGV[2]); redis.call('SADD', KEYS[1], ARGV[1]); redis.call('PUBLISH', KEYS[3], ARGV[3]); return 1";
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct BanMetadata<'a> {
|
||||
created_at_unix: i64,
|
||||
expires_at_unix: i64,
|
||||
reason: &'a str,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct BanEvent<'a> {
|
||||
ip: String,
|
||||
action: &'static str,
|
||||
duration_secs: u64,
|
||||
expires_at_unix: i64,
|
||||
reason: &'a str,
|
||||
}
|
||||
|
||||
pub async fn start_local_ban_publisher(
|
||||
client: redis::Client,
|
||||
blacklist: std::sync::Arc<Blacklist>,
|
||||
mut events: broadcast::Receiver<LocalBanEvent>,
|
||||
mut shutdown: watch::Receiver<bool>,
|
||||
) {
|
||||
for event in blacklist.local_snapshot() {
|
||||
if !publish_with_retry(&client, event, &mut shutdown).await {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
received = events.recv() => match received {
|
||||
Ok(event) => {
|
||||
if !publish_with_retry(&client, event, &mut shutdown).await {
|
||||
return;
|
||||
}
|
||||
},
|
||||
Err(broadcast::error::RecvError::Lagged(skipped)) => {
|
||||
tracing::warn!(skipped, "local ban publisher lagged; reconciling active local bans");
|
||||
for event in blacklist.local_snapshot() {
|
||||
if !publish_with_retry(&client, event, &mut shutdown).await {
|
||||
return;
|
||||
}
|
||||
}
|
||||
},
|
||||
Err(broadcast::error::RecvError::Closed) => return,
|
||||
},
|
||||
changed = shutdown.changed() => {
|
||||
if changed.is_err() || *shutdown.borrow_and_update() {
|
||||
return;
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn publish_with_retry(
|
||||
client: &redis::Client,
|
||||
event: LocalBanEvent,
|
||||
shutdown: &mut watch::Receiver<bool>,
|
||||
) -> bool {
|
||||
let mut delay = Duration::from_secs(1);
|
||||
loop {
|
||||
match publish_ban(client, &event).await {
|
||||
Ok(()) => return true,
|
||||
Err(error) => tracing::warn!(%error, ip = %event.ip, "failed to persist local ban; retrying"),
|
||||
}
|
||||
tokio::select! {
|
||||
_ = tokio::time::sleep(delay) => delay = (delay * 2).min(Duration::from_secs(30)),
|
||||
changed = shutdown.changed() => {
|
||||
if changed.is_err() || *shutdown.borrow_and_update() {
|
||||
return false;
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn publish_ban(client: &redis::Client, event: &LocalBanEvent) -> anyhow::Result<()> {
|
||||
let now = chrono::Utc::now().timestamp();
|
||||
let remaining = event.expires_at_unix.saturating_sub(now);
|
||||
if remaining <= 0 {
|
||||
return Ok(());
|
||||
}
|
||||
let metadata = serde_json::to_string(&BanMetadata {
|
||||
created_at_unix: now,
|
||||
expires_at_unix: event.expires_at_unix,
|
||||
reason: &event.reason,
|
||||
})?;
|
||||
let payload = serde_json::to_string(&BanEvent {
|
||||
ip: event.ip.to_string(),
|
||||
action: "ban",
|
||||
duration_secs: u64::try_from(remaining)?,
|
||||
expires_at_unix: event.expires_at_unix,
|
||||
reason: &event.reason,
|
||||
})?;
|
||||
let mut conn = client.get_multiplexed_async_connection().await?;
|
||||
let _: i64 = redis::cmd("EVAL")
|
||||
.arg(STORE_BAN_SCRIPT)
|
||||
.arg(3)
|
||||
.arg(BLACKLIST_KEY)
|
||||
.arg(BLACKLIST_METADATA_KEY)
|
||||
.arg(BLACKLIST_CHANNEL)
|
||||
.arg(event.ip.to_string())
|
||||
.arg(metadata)
|
||||
.arg(payload)
|
||||
.query_async(&mut conn)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue