- Inventory nodes/servers use cursor SCAN plus MGET batches instead of KEYS and per-key GET; a Redis error returns 503 instead of an empty fleet. - GET /blacklist no longer writes: the legacy metadata backfill moved to a one-shot startup task. The list endpoint paginates (limit 100, max 1000, offset max 10000) instead of dumping SMEMBERS. - Router-wide DefaultBodyLimit 8 KiB and TimeoutLayer 10 s; every mutating endpoint is rate limited (30/min per client IP) with idle eviction, and the login limiter resets on success. reason > 256 bytes and malformed targets are rejected with 400. - SIGTERM/Ctrl-C trigger graceful shutdown; the heartbeat and eviction tasks are cancelled and joined so in-flight ban edits reach Redis. - MANAGER_TRUSTED_PROXIES enables X-Forwarded-For (right-to-left walk) only for configured proxy networks; otherwise the TCP peer is used. - tests/manager_api.rs covers 401/400/413/429/503 and CORS preflight through the real router; manager_security gains limiter and proxy-IP tests.
82 lines
2.7 KiB
Rust
82 lines
2.7 KiB
Rust
//! One-shot startup backfill of legacy blacklist metadata.
|
|
//!
|
|
//! Older managers wrote `rampart:blacklist` members without a record in
|
|
//! `rampart:blacklist:metadata`. The read path is side-effect free now, so the
|
|
//! repair happens once at startup instead of on every `GET`: set members are
|
|
//! iterated with SSCAN, read in HMGET batches and filled with HSETNX, which
|
|
//! never overwrites metadata written by a concurrent add.
|
|
|
|
use super::common::{
|
|
BLACKLIST_KEY, BLACKLIST_METADATA_KEY, PAGE_SCAN_BATCH, legacy_metadata, redis_error, redis_unavailable,
|
|
};
|
|
use crate::manager::{AppState, SharedState};
|
|
use axum::{Json, http::StatusCode};
|
|
use std::net::IpAddr;
|
|
|
|
type ApiResult<T> = Result<T, (StatusCode, Json<serde_json::Value>)>;
|
|
|
|
/// Repairs legacy members in the background; failures are logged, not fatal.
|
|
pub async fn backfill_legacy_metadata(state: SharedState) {
|
|
match backfill(&state).await {
|
|
Ok(0) => {},
|
|
Ok(filled) => tracing::info!("blacklist metadata backfill repaired {filled} legacy entries"),
|
|
Err((_, Json(body))) => tracing::warn!("blacklist metadata backfill skipped: {body}"),
|
|
}
|
|
}
|
|
|
|
/// Walks the ban set and persists fallback metadata for members without any.
|
|
///
|
|
/// Returns the number of repaired members. A Redis failure aborts the run: the
|
|
/// next manager start retries, and the read path keeps serving meanwhile.
|
|
async fn backfill(state: &AppState) -> ApiResult<usize> {
|
|
let mut conn = state
|
|
.redis_client
|
|
.get_multiplexed_async_connection()
|
|
.await
|
|
.map_err(redis_unavailable)?;
|
|
let now = chrono::Utc::now().timestamp();
|
|
let mut cursor = 0_u64;
|
|
let mut filled = 0_usize;
|
|
loop {
|
|
let (next, members): (u64, Vec<String>) = redis::cmd("SSCAN")
|
|
.arg(BLACKLIST_KEY)
|
|
.arg(cursor)
|
|
.arg("COUNT")
|
|
.arg(PAGE_SCAN_BATCH)
|
|
.query_async(&mut conn)
|
|
.await
|
|
.map_err(redis_error)?;
|
|
let targets: Vec<String> = members
|
|
.into_iter()
|
|
.filter(|target| target.parse::<IpAddr>().is_ok())
|
|
.collect();
|
|
if !targets.is_empty() {
|
|
let existing: Vec<Option<String>> = redis::cmd("HMGET")
|
|
.arg(BLACKLIST_METADATA_KEY)
|
|
.arg(&targets)
|
|
.query_async(&mut conn)
|
|
.await
|
|
.map_err(redis_error)?;
|
|
for (target, encoded) in targets.iter().zip(existing) {
|
|
if encoded.is_some() {
|
|
continue;
|
|
}
|
|
let encoded = serde_json::to_string(&legacy_metadata(now)).map_err(|_| {
|
|
super::common::api_error(StatusCode::INTERNAL_SERVER_ERROR, "failed to encode blacklist metadata")
|
|
})?;
|
|
let _: i64 = redis::cmd("HSETNX")
|
|
.arg(BLACKLIST_METADATA_KEY)
|
|
.arg(target)
|
|
.arg(encoded)
|
|
.query_async(&mut conn)
|
|
.await
|
|
.map_err(redis_error)?;
|
|
filled += 1;
|
|
}
|
|
}
|
|
cursor = next;
|
|
if cursor == 0 {
|
|
return Ok(filled);
|
|
}
|
|
}
|
|
}
|