feat(xdp): estimate distinct sources in prefix stats, fix the detection window
The subnet detector's spoof gate needs the number of distinct sources in a prefix, but the XDP path reported 0, so a /24 flooded from a botnet could never reach the Block verdict and the whole subnet detector was inert with XDP on. - prefix_stats carries a 64-bit bitmap of hashed source addresses (interface ID for IPv6, address for IPv4); userspace counts the set bits. - read_prefix_stats now resets each entry after reading: the counters are documented as per-window and compared against fixed thresholds, but were cumulative, so with enough traffic every active prefix eventually looked like an attack. - The monitor no longer suppresses Block on the XDP path and no longer carries the now-dead "no source diversity" branch. ABI: the prefix_stats value grows 24 -> 32 bytes; the xdp_object_maps test pins the new size.
This commit is contained in:
parent
c1a076a4a3
commit
9baae25acb
8 changed files with 88 additions and 56 deletions
|
|
@ -31,15 +31,6 @@ impl Source {
|
||||||
Source::Tracker(tracker) => tracker.snapshot().await,
|
Source::Tracker(tracker) => tracker.snapshot().await,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Whether the source reports distinct source addresses per prefix.
|
|
||||||
fn counts_unique_sources(&self) -> bool {
|
|
||||||
match self {
|
|
||||||
#[cfg(feature = "xdp")]
|
|
||||||
Source::Xdp(_) => false,
|
|
||||||
Source::Tracker(_) => true,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Verdict enforcement settings shared by every snapshot tick.
|
/// Verdict enforcement settings shared by every snapshot tick.
|
||||||
|
|
@ -82,7 +73,6 @@ pub fn spawn(
|
||||||
ttl: ban_ttl,
|
ttl: ban_ttl,
|
||||||
trusted: trusted_peer,
|
trusted: trusted_peer,
|
||||||
};
|
};
|
||||||
let extreme_threshold = detect.syn_threshold.saturating_mul(4);
|
|
||||||
|
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
let mut tick = tokio::time::interval(window);
|
let mut tick = tokio::time::interval(window);
|
||||||
|
|
@ -99,7 +89,7 @@ pub fn spawn(
|
||||||
match source.snapshot().await {
|
match source.snapshot().await {
|
||||||
Ok(snaps) => {
|
Ok(snaps) => {
|
||||||
let verdicts = detector.evaluate(&snaps);
|
let verdicts = detector.evaluate(&snaps);
|
||||||
apply_verdicts(&verdicts, &source, &policy, extreme_threshold);
|
apply_verdicts(&verdicts, &source, &policy);
|
||||||
},
|
},
|
||||||
Err(e) => tracing::debug!("prefix stats unavailable: {e}"),
|
Err(e) => tracing::debug!("prefix stats unavailable: {e}"),
|
||||||
}
|
}
|
||||||
|
|
@ -119,7 +109,7 @@ fn build_source(tracker: Option<Arc<SubnetTracker>>, xdp: Option<Arc<Mutex<XdpFi
|
||||||
tracker.map(|t| Source::Tracker(TrackerSource { tracker: t }))
|
tracker.map(|t| Source::Tracker(TrackerSource { tracker: t }))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn apply_verdicts(verdicts: &[PrefixVerdict], source: &Source, policy: &BanPolicy, extreme_threshold: u64) {
|
fn apply_verdicts(verdicts: &[PrefixVerdict], source: &Source, policy: &BanPolicy) {
|
||||||
for pv in verdicts {
|
for pv in verdicts {
|
||||||
metrics::SUBNET_VERDICTS
|
metrics::SUBNET_VERDICTS
|
||||||
.with_label_values(&[verdict_name(pv.verdict)])
|
.with_label_values(&[verdict_name(pv.verdict)])
|
||||||
|
|
@ -131,18 +121,6 @@ fn apply_verdicts(verdicts: &[PrefixVerdict], source: &Source, policy: &BanPolic
|
||||||
verdict = verdict_name(pv.verdict),
|
verdict = verdict_name(pv.verdict),
|
||||||
"subnet verdict"
|
"subnet verdict"
|
||||||
);
|
);
|
||||||
if pv.verdict == SubnetVerdict::Challenge
|
|
||||||
&& !source.counts_unique_sources()
|
|
||||||
&& pv.snapshot.syn_count >= extreme_threshold
|
|
||||||
{
|
|
||||||
// XDP does not count distinct sources, so the spoof gate can never
|
|
||||||
// confirm `Block` on that path; make the cap visible instead of silent.
|
|
||||||
tracing::info!(
|
|
||||||
prefix = %pv.prefix,
|
|
||||||
syn = pv.snapshot.syn_count,
|
|
||||||
"spoof gate suppressed block: XDP has no source diversity data; escalating at L7 only"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
match source {
|
match source {
|
||||||
#[cfg(feature = "xdp")]
|
#[cfg(feature = "xdp")]
|
||||||
|
|
|
||||||
|
|
@ -9,30 +9,41 @@ pub struct PrefixStatsVal {
|
||||||
pub syn_count: u64,
|
pub syn_count: u64,
|
||||||
pub pkt_count: u64,
|
pub pkt_count: u64,
|
||||||
pub last_seen_ns: u64,
|
pub last_seen_ns: u64,
|
||||||
|
/// 64-bit bitmap of hashed source addresses; popcount estimates the number
|
||||||
|
/// of distinct sources in the window.
|
||||||
|
pub source_bits: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl PrefixStatsVal {
|
impl PrefixStatsVal {
|
||||||
/// Parses the raw 24-byte map value.
|
/// Parses the raw 32-byte map value.
|
||||||
#[must_use]
|
#[must_use]
|
||||||
pub fn from_bytes(bytes: [u8; 24]) -> Self {
|
pub fn from_bytes(bytes: [u8; 32]) -> Self {
|
||||||
let read_u64 =
|
let read_u64 =
|
||||||
|range: std::ops::Range<usize>| u64::from_le_bytes(bytes[range].try_into().expect("8-byte field"));
|
|range: std::ops::Range<usize>| u64::from_le_bytes(bytes[range].try_into().expect("8-byte field"));
|
||||||
Self {
|
Self {
|
||||||
syn_count: read_u64(0..8),
|
syn_count: read_u64(0..8),
|
||||||
pkt_count: read_u64(8..16),
|
pkt_count: read_u64(8..16),
|
||||||
last_seen_ns: read_u64(16..24),
|
last_seen_ns: read_u64(16..24),
|
||||||
|
source_bits: read_u64(24..32),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Serialises the value into the raw 24-byte map layout.
|
/// Serialises the value into the raw 32-byte map layout.
|
||||||
#[must_use]
|
#[must_use]
|
||||||
pub fn to_bytes(self) -> [u8; 24] {
|
pub fn to_bytes(self) -> [u8; 32] {
|
||||||
let mut out = [0u8; 24];
|
let mut out = [0u8; 32];
|
||||||
out[0..8].copy_from_slice(&self.syn_count.to_le_bytes());
|
out[0..8].copy_from_slice(&self.syn_count.to_le_bytes());
|
||||||
out[8..16].copy_from_slice(&self.pkt_count.to_le_bytes());
|
out[8..16].copy_from_slice(&self.pkt_count.to_le_bytes());
|
||||||
out[16..24].copy_from_slice(&self.last_seen_ns.to_le_bytes());
|
out[16..24].copy_from_slice(&self.last_seen_ns.to_le_bytes());
|
||||||
|
out[24..32].copy_from_slice(&self.source_bits.to_le_bytes());
|
||||||
out
|
out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Estimated number of distinct sources seen in this window.
|
||||||
|
#[must_use]
|
||||||
|
pub fn unique_sources(&self) -> u64 {
|
||||||
|
u64::from(self.source_bits.count_ones())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Aggregated counters for one prefix over a detection window.
|
/// Aggregated counters for one prefix over a detection window.
|
||||||
|
|
@ -178,26 +189,10 @@ impl PrefixStatsSource for XdpPrefixStats {
|
||||||
syn_count: val.syn_count,
|
syn_count: val.syn_count,
|
||||||
pkt_count: val.pkt_count,
|
pkt_count: val.pkt_count,
|
||||||
last_seen_ns: val.last_seen_ns,
|
last_seen_ns: val.last_seen_ns,
|
||||||
unique_sources: 0,
|
unique_sources: val.unique_sources(),
|
||||||
},
|
},
|
||||||
))
|
))
|
||||||
})
|
})
|
||||||
.collect())
|
.collect())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn stats_val_roundtrip() {
|
|
||||||
let val = PrefixStatsVal {
|
|
||||||
syn_count: 42,
|
|
||||||
pkt_count: 100,
|
|
||||||
last_seen_ns: 1_234_567_890,
|
|
||||||
};
|
|
||||||
assert_eq!(PrefixStatsVal::from_bytes(val.to_bytes()), val);
|
|
||||||
assert_eq!(size_of::<PrefixStatsVal>(), 24);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -96,7 +96,12 @@ impl XdpFilter {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Reads the `prefix_stats` map (aggregates per /24 and /64 prefix).
|
/// Reads and resets the `prefix_stats` map (aggregates per /24 and /64).
|
||||||
|
///
|
||||||
|
/// The counters are per-window by contract: the detector compares them
|
||||||
|
/// against fixed thresholds, so leaving them cumulative would eventually
|
||||||
|
/// flag every prefix. Packets that arrive between the read and the reset
|
||||||
|
/// are lost, which is acceptable for statistics.
|
||||||
///
|
///
|
||||||
/// # Errors
|
/// # Errors
|
||||||
/// XDP is not loaded, the map is missing from the object, or a BPF read fails.
|
/// XDP is not loaded, the map is missing from the object, or a BPF read fails.
|
||||||
|
|
@ -110,10 +115,13 @@ impl XdpFilter {
|
||||||
let Some(val) = map.lookup(&kb, MapFlags::ANY).context("prefix_stats lookup failed")? else {
|
let Some(val) = map.lookup(&kb, MapFlags::ANY).context("prefix_stats lookup failed")? else {
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
let Ok(vb) = <[u8; 24]>::try_from(val.as_slice()) else {
|
let Ok(vb) = <[u8; 32]>::try_from(val.as_slice()) else {
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
out.push((PrefixKey::from_bytes(kb), PrefixStatsVal::from_bytes(vb)));
|
out.push((PrefixKey::from_bytes(kb), PrefixStatsVal::from_bytes(vb)));
|
||||||
|
if let Err(e) = map.update(&kb, &[0u8; 32], MapFlags::ANY) {
|
||||||
|
tracing::debug!(prefix = ?PrefixKey::from_bytes(kb), error = %e, "prefix_stats reset failed");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
Ok(out)
|
Ok(out)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -152,7 +152,7 @@ fn mapped_ipv6_treated_as_ipv4() {
|
||||||
#[test]
|
#[test]
|
||||||
fn prefix_key_roundtrip_and_layout() {
|
fn prefix_key_roundtrip_and_layout() {
|
||||||
assert_eq!(size_of::<PrefixKey>(), 24);
|
assert_eq!(size_of::<PrefixKey>(), 24);
|
||||||
assert_eq!(size_of::<PrefixStatsVal>(), 24);
|
assert_eq!(size_of::<PrefixStatsVal>(), 32);
|
||||||
let key = PrefixKey::new(v4([10, 1, 2, 3]));
|
let key = PrefixKey::new(v4([10, 1, 2, 3]));
|
||||||
assert_eq!(key.family, rampart::traffic::prefix::PREFIX_FAMILY_V4);
|
assert_eq!(key.family, rampart::traffic::prefix::PREFIX_FAMILY_V4);
|
||||||
assert_eq!(key.to_prefix(), Some(v4([10, 1, 2, 0])));
|
assert_eq!(key.to_prefix(), Some(v4([10, 1, 2, 0])));
|
||||||
|
|
|
||||||
25
tests/traffic_prefix_stats.rs
Normal file
25
tests/traffic_prefix_stats.rs
Normal file
|
|
@ -0,0 +1,25 @@
|
||||||
|
//! Layout of the `prefix_stats` BPF map value: the 32-byte little-endian
|
||||||
|
//! encoding round-trips and the struct size matches the kernel side.
|
||||||
|
|
||||||
|
use rampart::traffic::prefix::PrefixStatsVal;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn stats_val_roundtrip() {
|
||||||
|
let val = PrefixStatsVal {
|
||||||
|
syn_count: 42,
|
||||||
|
pkt_count: 100,
|
||||||
|
last_seen_ns: 1_234_567_890,
|
||||||
|
source_bits: 0b1011,
|
||||||
|
};
|
||||||
|
assert_eq!(PrefixStatsVal::from_bytes(val.to_bytes()), val);
|
||||||
|
assert_eq!(size_of::<PrefixStatsVal>(), 32);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unique_sources_counts_set_bits() {
|
||||||
|
let val = PrefixStatsVal {
|
||||||
|
source_bits: 0b1011,
|
||||||
|
..PrefixStatsVal::default()
|
||||||
|
};
|
||||||
|
assert_eq!(val.unique_sources(), 3);
|
||||||
|
}
|
||||||
|
|
@ -69,7 +69,7 @@ fn map_key_and_value_sizes_match_the_rust_encoding() {
|
||||||
("blacklist6_map", 20, 8),
|
("blacklist6_map", 20, 8),
|
||||||
("whitelist_map", 8, 1),
|
("whitelist_map", 8, 1),
|
||||||
("whitelist6_map", 20, 1),
|
("whitelist6_map", 20, 1),
|
||||||
("prefix_stats", 24, 24),
|
("prefix_stats", 24, 32),
|
||||||
("stats_map", 4, 8),
|
("stats_map", 4, 8),
|
||||||
];
|
];
|
||||||
for (name, key_size, value_size) in expected {
|
for (name, key_size, value_size) in expected {
|
||||||
|
|
|
||||||
|
|
@ -34,6 +34,7 @@ struct prefix_stats_val {
|
||||||
__u64 syn_count;
|
__u64 syn_count;
|
||||||
__u64 pkt_count;
|
__u64 pkt_count;
|
||||||
__u64 last_seen_ns; // bpf_ktime_get_ns()
|
__u64 last_seen_ns; // bpf_ktime_get_ns()
|
||||||
|
__u64 source_bits; // 64-bit bitmap of hashed source addresses
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -87,6 +88,24 @@ static __always_inline void prefix_key_from_v6(struct prefix_key *key,
|
||||||
__builtin_memcpy(key->addr, src_addr, 8);
|
__builtin_memcpy(key->addr, src_addr, 8);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Maps a 32-bit source identifier to one bit of a 64-bit bitmap.
|
||||||
|
*
|
||||||
|
* The subnet detector needs an estimate of distinct sources ("spoof gate":
|
||||||
|
* a /24 flooded from one IP is not a botnet). Exact counting in XDP would
|
||||||
|
* need an unbounded map, so we set a bit of a multiplicative hash. Collisions
|
||||||
|
* can only over-count, and with min_unique_sources=16 the 64-bit space makes
|
||||||
|
* that unlikely for a real botnet while a single source still sets one bit.
|
||||||
|
*
|
||||||
|
* @param src_id identifier that differs per source address
|
||||||
|
* @return a single-bit mask
|
||||||
|
*/
|
||||||
|
static __always_inline __u64 source_bit(__u32 src_id)
|
||||||
|
{
|
||||||
|
__u32 hash = src_id * 2654435761u;
|
||||||
|
return 1ULL << (hash >> 26);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Increment the counters for one prefix.
|
* Increment the counters for one prefix.
|
||||||
*
|
*
|
||||||
|
|
@ -97,17 +116,21 @@ static __always_inline void prefix_key_from_v6(struct prefix_key *key,
|
||||||
* @param key prefix key, already built
|
* @param key prefix key, already built
|
||||||
* @param is_syn 1 counts a SYN, 0 counts any other packet
|
* @param is_syn 1 counts a SYN, 0 counts any other packet
|
||||||
* @param now current bpf_ktime_get_ns() timestamp
|
* @param now current bpf_ktime_get_ns() timestamp
|
||||||
|
* @param src_id 32-bit identifier of the full source address
|
||||||
*/
|
*/
|
||||||
static __always_inline void update_prefix_stats(struct prefix_key *key,
|
static __always_inline void update_prefix_stats(struct prefix_key *key,
|
||||||
__u8 is_syn, __u64 now)
|
__u8 is_syn, __u64 now,
|
||||||
|
__u32 src_id)
|
||||||
{
|
{
|
||||||
|
__u64 bit = source_bit(src_id);
|
||||||
struct prefix_stats_val *val = bpf_map_lookup_elem(&prefix_stats, key);
|
struct prefix_stats_val *val = bpf_map_lookup_elem(&prefix_stats, key);
|
||||||
|
|
||||||
if (!val) {
|
if (!val) {
|
||||||
struct prefix_stats_val init = {
|
struct prefix_stats_val init = {
|
||||||
.syn_count = is_syn ? 1 : 0,
|
.syn_count = is_syn ? 1 : 0,
|
||||||
.pkt_count = is_syn ? 0 : 1,
|
.pkt_count = is_syn ? 0 : 1,
|
||||||
.last_seen_ns = now,
|
.last_seen_ns = now,
|
||||||
|
.source_bits = bit,
|
||||||
};
|
};
|
||||||
bpf_map_update_elem(&prefix_stats, key, &init, BPF_ANY);
|
bpf_map_update_elem(&prefix_stats, key, &init, BPF_ANY);
|
||||||
return;
|
return;
|
||||||
|
|
@ -117,6 +140,7 @@ static __always_inline void update_prefix_stats(struct prefix_key *key,
|
||||||
__sync_fetch_and_add(&val->syn_count, 1);
|
__sync_fetch_and_add(&val->syn_count, 1);
|
||||||
else
|
else
|
||||||
__sync_fetch_and_add(&val->pkt_count, 1);
|
__sync_fetch_and_add(&val->pkt_count, 1);
|
||||||
|
__sync_fetch_and_or(&val->source_bits, bit);
|
||||||
val->last_seen_ns = now;
|
val->last_seen_ns = now;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -425,7 +425,9 @@ int rampart_universal_filter(struct xdp_md *ctx)
|
||||||
struct prefix_key pkey6;
|
struct prefix_key pkey6;
|
||||||
prefix_key_from_v6(&pkey6, src6);
|
prefix_key_from_v6(&pkey6, src6);
|
||||||
struct tcphdr *tcp6 = (struct tcphdr *)l4;
|
struct tcphdr *tcp6 = (struct tcphdr *)l4;
|
||||||
update_prefix_stats(&pkey6, (tcp6->syn && !tcp6->ack) ? 1 : 0, now);
|
__u32 v6_id;
|
||||||
|
__builtin_memcpy(&v6_id, src6 + 12, 4); // interface ID differs per host
|
||||||
|
update_prefix_stats(&pkey6, (tcp6->syn && !tcp6->ack) ? 1 : 0, now, v6_id);
|
||||||
inc_pass();
|
inc_pass();
|
||||||
return XDP_PASS;
|
return XDP_PASS;
|
||||||
}
|
}
|
||||||
|
|
@ -560,7 +562,7 @@ int rampart_universal_filter(struct xdp_md *ctx)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Count SYN only after throttle passed (don't count throttled SYNs)
|
// Count SYN only after throttle passed (don't count throttled SYNs)
|
||||||
update_prefix_stats(&pkey, 1, now);
|
update_prefix_stats(&pkey, 1, now, src_ip);
|
||||||
|
|
||||||
// Create conntrack entry for new connection. A SYN may carry data
|
// Create conntrack entry for new connection. A SYN may carry data
|
||||||
// (TCP Fast Open), which is acknowledged as part of the handshake;
|
// (TCP Fast Open), which is acknowledged as part of the handshake;
|
||||||
|
|
@ -584,7 +586,7 @@ int rampart_universal_filter(struct xdp_md *ctx)
|
||||||
|
|
||||||
// Connection tracking lookup.
|
// Connection tracking lookup.
|
||||||
// pkt_count: all non-SYN TCP that survived blacklist/bypass/throttle
|
// pkt_count: all non-SYN TCP that survived blacklist/bypass/throttle
|
||||||
update_prefix_stats(&pkey, 0, now);
|
update_prefix_stats(&pkey, 0, now, src_ip);
|
||||||
|
|
||||||
// RST challenge proof: match a pending challenge before conntrack,
|
// RST challenge proof: match a pending challenge before conntrack,
|
||||||
// because the RST answers a flow that was never tracked.
|
// because the RST answers a flow that was never tracked.
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue