feat(xdp): estimate distinct sources in prefix stats, fix the detection window

The subnet detector's spoof gate needs the number of distinct sources in a
prefix, but the XDP path reported 0, so a /24 flooded from a botnet could never
reach the Block verdict and the whole subnet detector was inert with XDP on.

- prefix_stats carries a 64-bit bitmap of hashed source addresses (interface
  ID for IPv6, address for IPv4); userspace counts the set bits.
- read_prefix_stats now resets each entry after reading: the counters are
  documented as per-window and compared against fixed thresholds, but were
  cumulative, so with enough traffic every active prefix eventually looked
  like an attack.
- The monitor no longer suppresses Block on the XDP path and no longer carries
  the now-dead "no source diversity" branch.

ABI: the prefix_stats value grows 24 -> 32 bytes; the xdp_object_maps test
pins the new size.
This commit is contained in:
Loki 2026-11-16 12:36:19 +01:00
parent c1a076a4a3
commit 9baae25acb
Signed by: boba
GPG key ID: 6CDE87BC9A778D97
8 changed files with 88 additions and 56 deletions

View file

@ -31,15 +31,6 @@ impl Source {
Source::Tracker(tracker) => tracker.snapshot().await, Source::Tracker(tracker) => tracker.snapshot().await,
} }
} }
/// Whether the source reports distinct source addresses per prefix.
fn counts_unique_sources(&self) -> bool {
match self {
#[cfg(feature = "xdp")]
Source::Xdp(_) => false,
Source::Tracker(_) => true,
}
}
} }
/// Verdict enforcement settings shared by every snapshot tick. /// Verdict enforcement settings shared by every snapshot tick.
@ -82,7 +73,6 @@ pub fn spawn(
ttl: ban_ttl, ttl: ban_ttl,
trusted: trusted_peer, trusted: trusted_peer,
}; };
let extreme_threshold = detect.syn_threshold.saturating_mul(4);
tokio::spawn(async move { tokio::spawn(async move {
let mut tick = tokio::time::interval(window); let mut tick = tokio::time::interval(window);
@ -99,7 +89,7 @@ pub fn spawn(
match source.snapshot().await { match source.snapshot().await {
Ok(snaps) => { Ok(snaps) => {
let verdicts = detector.evaluate(&snaps); let verdicts = detector.evaluate(&snaps);
apply_verdicts(&verdicts, &source, &policy, extreme_threshold); apply_verdicts(&verdicts, &source, &policy);
}, },
Err(e) => tracing::debug!("prefix stats unavailable: {e}"), Err(e) => tracing::debug!("prefix stats unavailable: {e}"),
} }
@ -119,7 +109,7 @@ fn build_source(tracker: Option<Arc<SubnetTracker>>, xdp: Option<Arc<Mutex<XdpFi
tracker.map(|t| Source::Tracker(TrackerSource { tracker: t })) tracker.map(|t| Source::Tracker(TrackerSource { tracker: t }))
} }
fn apply_verdicts(verdicts: &[PrefixVerdict], source: &Source, policy: &BanPolicy, extreme_threshold: u64) { fn apply_verdicts(verdicts: &[PrefixVerdict], source: &Source, policy: &BanPolicy) {
for pv in verdicts { for pv in verdicts {
metrics::SUBNET_VERDICTS metrics::SUBNET_VERDICTS
.with_label_values(&[verdict_name(pv.verdict)]) .with_label_values(&[verdict_name(pv.verdict)])
@ -131,18 +121,6 @@ fn apply_verdicts(verdicts: &[PrefixVerdict], source: &Source, policy: &BanPolic
verdict = verdict_name(pv.verdict), verdict = verdict_name(pv.verdict),
"subnet verdict" "subnet verdict"
); );
if pv.verdict == SubnetVerdict::Challenge
&& !source.counts_unique_sources()
&& pv.snapshot.syn_count >= extreme_threshold
{
// XDP does not count distinct sources, so the spoof gate can never
// confirm `Block` on that path; make the cap visible instead of silent.
tracing::info!(
prefix = %pv.prefix,
syn = pv.snapshot.syn_count,
"spoof gate suppressed block: XDP has no source diversity data; escalating at L7 only"
);
}
} }
match source { match source {
#[cfg(feature = "xdp")] #[cfg(feature = "xdp")]

View file

@ -9,30 +9,41 @@ pub struct PrefixStatsVal {
pub syn_count: u64, pub syn_count: u64,
pub pkt_count: u64, pub pkt_count: u64,
pub last_seen_ns: u64, pub last_seen_ns: u64,
/// 64-bit bitmap of hashed source addresses; popcount estimates the number
/// of distinct sources in the window.
pub source_bits: u64,
} }
impl PrefixStatsVal { impl PrefixStatsVal {
/// Parses the raw 24-byte map value. /// Parses the raw 32-byte map value.
#[must_use] #[must_use]
pub fn from_bytes(bytes: [u8; 24]) -> Self { pub fn from_bytes(bytes: [u8; 32]) -> Self {
let read_u64 = let read_u64 =
|range: std::ops::Range<usize>| u64::from_le_bytes(bytes[range].try_into().expect("8-byte field")); |range: std::ops::Range<usize>| u64::from_le_bytes(bytes[range].try_into().expect("8-byte field"));
Self { Self {
syn_count: read_u64(0..8), syn_count: read_u64(0..8),
pkt_count: read_u64(8..16), pkt_count: read_u64(8..16),
last_seen_ns: read_u64(16..24), last_seen_ns: read_u64(16..24),
source_bits: read_u64(24..32),
} }
} }
/// Serialises the value into the raw 24-byte map layout. /// Serialises the value into the raw 32-byte map layout.
#[must_use] #[must_use]
pub fn to_bytes(self) -> [u8; 24] { pub fn to_bytes(self) -> [u8; 32] {
let mut out = [0u8; 24]; let mut out = [0u8; 32];
out[0..8].copy_from_slice(&self.syn_count.to_le_bytes()); out[0..8].copy_from_slice(&self.syn_count.to_le_bytes());
out[8..16].copy_from_slice(&self.pkt_count.to_le_bytes()); out[8..16].copy_from_slice(&self.pkt_count.to_le_bytes());
out[16..24].copy_from_slice(&self.last_seen_ns.to_le_bytes()); out[16..24].copy_from_slice(&self.last_seen_ns.to_le_bytes());
out[24..32].copy_from_slice(&self.source_bits.to_le_bytes());
out out
} }
/// Estimated number of distinct sources seen in this window.
#[must_use]
pub fn unique_sources(&self) -> u64 {
u64::from(self.source_bits.count_ones())
}
} }
/// Aggregated counters for one prefix over a detection window. /// Aggregated counters for one prefix over a detection window.
@ -178,26 +189,10 @@ impl PrefixStatsSource for XdpPrefixStats {
syn_count: val.syn_count, syn_count: val.syn_count,
pkt_count: val.pkt_count, pkt_count: val.pkt_count,
last_seen_ns: val.last_seen_ns, last_seen_ns: val.last_seen_ns,
unique_sources: 0, unique_sources: val.unique_sources(),
}, },
)) ))
}) })
.collect()) .collect())
} }
} }
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn stats_val_roundtrip() {
let val = PrefixStatsVal {
syn_count: 42,
pkt_count: 100,
last_seen_ns: 1_234_567_890,
};
assert_eq!(PrefixStatsVal::from_bytes(val.to_bytes()), val);
assert_eq!(size_of::<PrefixStatsVal>(), 24);
}
}

View file

@ -96,7 +96,12 @@ impl XdpFilter {
}) })
} }
/// Reads the `prefix_stats` map (aggregates per /24 and /64 prefix). /// Reads and resets the `prefix_stats` map (aggregates per /24 and /64).
///
/// The counters are per-window by contract: the detector compares them
/// against fixed thresholds, so leaving them cumulative would eventually
/// flag every prefix. Packets that arrive between the read and the reset
/// are lost, which is acceptable for statistics.
/// ///
/// # Errors /// # Errors
/// XDP is not loaded, the map is missing from the object, or a BPF read fails. /// XDP is not loaded, the map is missing from the object, or a BPF read fails.
@ -110,10 +115,13 @@ impl XdpFilter {
let Some(val) = map.lookup(&kb, MapFlags::ANY).context("prefix_stats lookup failed")? else { let Some(val) = map.lookup(&kb, MapFlags::ANY).context("prefix_stats lookup failed")? else {
continue; continue;
}; };
let Ok(vb) = <[u8; 24]>::try_from(val.as_slice()) else { let Ok(vb) = <[u8; 32]>::try_from(val.as_slice()) else {
continue; continue;
}; };
out.push((PrefixKey::from_bytes(kb), PrefixStatsVal::from_bytes(vb))); out.push((PrefixKey::from_bytes(kb), PrefixStatsVal::from_bytes(vb)));
if let Err(e) = map.update(&kb, &[0u8; 32], MapFlags::ANY) {
tracing::debug!(prefix = ?PrefixKey::from_bytes(kb), error = %e, "prefix_stats reset failed");
}
} }
Ok(out) Ok(out)
} }

View file

@ -152,7 +152,7 @@ fn mapped_ipv6_treated_as_ipv4() {
#[test] #[test]
fn prefix_key_roundtrip_and_layout() { fn prefix_key_roundtrip_and_layout() {
assert_eq!(size_of::<PrefixKey>(), 24); assert_eq!(size_of::<PrefixKey>(), 24);
assert_eq!(size_of::<PrefixStatsVal>(), 24); assert_eq!(size_of::<PrefixStatsVal>(), 32);
let key = PrefixKey::new(v4([10, 1, 2, 3])); let key = PrefixKey::new(v4([10, 1, 2, 3]));
assert_eq!(key.family, rampart::traffic::prefix::PREFIX_FAMILY_V4); assert_eq!(key.family, rampart::traffic::prefix::PREFIX_FAMILY_V4);
assert_eq!(key.to_prefix(), Some(v4([10, 1, 2, 0]))); assert_eq!(key.to_prefix(), Some(v4([10, 1, 2, 0])));

View file

@ -0,0 +1,25 @@
//! Layout of the `prefix_stats` BPF map value: the 32-byte little-endian
//! encoding round-trips and the struct size matches the kernel side.
use rampart::traffic::prefix::PrefixStatsVal;
#[test]
fn stats_val_roundtrip() {
let val = PrefixStatsVal {
syn_count: 42,
pkt_count: 100,
last_seen_ns: 1_234_567_890,
source_bits: 0b1011,
};
assert_eq!(PrefixStatsVal::from_bytes(val.to_bytes()), val);
assert_eq!(size_of::<PrefixStatsVal>(), 32);
}
#[test]
fn unique_sources_counts_set_bits() {
let val = PrefixStatsVal {
source_bits: 0b1011,
..PrefixStatsVal::default()
};
assert_eq!(val.unique_sources(), 3);
}

View file

@ -69,7 +69,7 @@ fn map_key_and_value_sizes_match_the_rust_encoding() {
("blacklist6_map", 20, 8), ("blacklist6_map", 20, 8),
("whitelist_map", 8, 1), ("whitelist_map", 8, 1),
("whitelist6_map", 20, 1), ("whitelist6_map", 20, 1),
("prefix_stats", 24, 24), ("prefix_stats", 24, 32),
("stats_map", 4, 8), ("stats_map", 4, 8),
]; ];
for (name, key_size, value_size) in expected { for (name, key_size, value_size) in expected {

View file

@ -34,6 +34,7 @@ struct prefix_stats_val {
__u64 syn_count; __u64 syn_count;
__u64 pkt_count; __u64 pkt_count;
__u64 last_seen_ns; // bpf_ktime_get_ns() __u64 last_seen_ns; // bpf_ktime_get_ns()
__u64 source_bits; // 64-bit bitmap of hashed source addresses
}; };
/** /**
@ -87,6 +88,24 @@ static __always_inline void prefix_key_from_v6(struct prefix_key *key,
__builtin_memcpy(key->addr, src_addr, 8); __builtin_memcpy(key->addr, src_addr, 8);
} }
/**
* Maps a 32-bit source identifier to one bit of a 64-bit bitmap.
*
* The subnet detector needs an estimate of distinct sources ("spoof gate":
* a /24 flooded from one IP is not a botnet). Exact counting in XDP would
* need an unbounded map, so we set a bit of a multiplicative hash. Collisions
* can only over-count, and with min_unique_sources=16 the 64-bit space makes
* that unlikely for a real botnet while a single source still sets one bit.
*
* @param src_id identifier that differs per source address
* @return a single-bit mask
*/
static __always_inline __u64 source_bit(__u32 src_id)
{
__u32 hash = src_id * 2654435761u;
return 1ULL << (hash >> 26);
}
/** /**
* Increment the counters for one prefix. * Increment the counters for one prefix.
* *
@ -97,17 +116,21 @@ static __always_inline void prefix_key_from_v6(struct prefix_key *key,
* @param key prefix key, already built * @param key prefix key, already built
* @param is_syn 1 counts a SYN, 0 counts any other packet * @param is_syn 1 counts a SYN, 0 counts any other packet
* @param now current bpf_ktime_get_ns() timestamp * @param now current bpf_ktime_get_ns() timestamp
* @param src_id 32-bit identifier of the full source address
*/ */
static __always_inline void update_prefix_stats(struct prefix_key *key, static __always_inline void update_prefix_stats(struct prefix_key *key,
__u8 is_syn, __u64 now) __u8 is_syn, __u64 now,
__u32 src_id)
{ {
__u64 bit = source_bit(src_id);
struct prefix_stats_val *val = bpf_map_lookup_elem(&prefix_stats, key); struct prefix_stats_val *val = bpf_map_lookup_elem(&prefix_stats, key);
if (!val) { if (!val) {
struct prefix_stats_val init = { struct prefix_stats_val init = {
.syn_count = is_syn ? 1 : 0, .syn_count = is_syn ? 1 : 0,
.pkt_count = is_syn ? 0 : 1, .pkt_count = is_syn ? 0 : 1,
.last_seen_ns = now, .last_seen_ns = now,
.source_bits = bit,
}; };
bpf_map_update_elem(&prefix_stats, key, &init, BPF_ANY); bpf_map_update_elem(&prefix_stats, key, &init, BPF_ANY);
return; return;
@ -117,6 +140,7 @@ static __always_inline void update_prefix_stats(struct prefix_key *key,
__sync_fetch_and_add(&val->syn_count, 1); __sync_fetch_and_add(&val->syn_count, 1);
else else
__sync_fetch_and_add(&val->pkt_count, 1); __sync_fetch_and_add(&val->pkt_count, 1);
__sync_fetch_and_or(&val->source_bits, bit);
val->last_seen_ns = now; val->last_seen_ns = now;
} }

View file

@ -425,7 +425,9 @@ int rampart_universal_filter(struct xdp_md *ctx)
struct prefix_key pkey6; struct prefix_key pkey6;
prefix_key_from_v6(&pkey6, src6); prefix_key_from_v6(&pkey6, src6);
struct tcphdr *tcp6 = (struct tcphdr *)l4; struct tcphdr *tcp6 = (struct tcphdr *)l4;
update_prefix_stats(&pkey6, (tcp6->syn && !tcp6->ack) ? 1 : 0, now); __u32 v6_id;
__builtin_memcpy(&v6_id, src6 + 12, 4); // interface ID differs per host
update_prefix_stats(&pkey6, (tcp6->syn && !tcp6->ack) ? 1 : 0, now, v6_id);
inc_pass(); inc_pass();
return XDP_PASS; return XDP_PASS;
} }
@ -560,7 +562,7 @@ int rampart_universal_filter(struct xdp_md *ctx)
} }
// Count SYN only after throttle passed (don't count throttled SYNs) // Count SYN only after throttle passed (don't count throttled SYNs)
update_prefix_stats(&pkey, 1, now); update_prefix_stats(&pkey, 1, now, src_ip);
// Create conntrack entry for new connection. A SYN may carry data // Create conntrack entry for new connection. A SYN may carry data
// (TCP Fast Open), which is acknowledged as part of the handshake; // (TCP Fast Open), which is acknowledged as part of the handshake;
@ -584,7 +586,7 @@ int rampart_universal_filter(struct xdp_md *ctx)
// Connection tracking lookup. // Connection tracking lookup.
// pkt_count: all non-SYN TCP that survived blacklist/bypass/throttle // pkt_count: all non-SYN TCP that survived blacklist/bypass/throttle
update_prefix_stats(&pkey, 0, now); update_prefix_stats(&pkey, 0, now, src_ip);
// RST challenge proof: match a pending challenge before conntrack, // RST challenge proof: match a pending challenge before conntrack,
// because the RST answers a flow that was never tracked. // because the RST answers a flow that was never tracked.