From c1a076a4a36cac1271a84150b4b7cf23ab8c6c92 Mon Sep 17 00:00:00 2001 From: loki5512344 Date: Mon, 16 Nov 2026 12:06:23 +0100 Subject: [PATCH] fix(core): close auth bypass, hangs and memory leaks found by review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Security: - An empty API_PASSWORD passed validation (only the literal "changeme" was rejected) and verify_password("", "") returned true, so a login with an empty password minted an admin token. Require >= 16 characters and >= 32 non-whitespace bytes for JWT_SECRET; verify rejects an empty stored secret. - syn_challenge_enabled required a real challenge_secret_hex: the public placeholder from config.h is now rejected in every spelling. - docker-compose: Redis/ClickHouse/Grafana are loopback-only and every password uses ${VAR:?} instead of a default. An exposed Redis was a way to lift bans and publish unbans to the whole fleet with one redis-cli. - Empty CORS_ORIGIN no longer means "allow any origin". - ban_cidr / userspace blacklist validate and mask prefixes; CIDR bans are stored as ranges, so a /24 or /64 ban actually covers the subnet (before, only the network address was blocked). Availability: - ClickHouse client has connect/request timeouts and no mutex held across await: a hung analytics backend can no longer freeze the ban path. Events go through a bounded buffer with a drop counter. - Redis uses a ConnectionManager with per-command timeouts; pubsub and reconcile reset the backoff after success; publish cannot block shutdown. - Ban durations from pubsub are clamped and use checked_add; the sync task survives panics instead of silently dying. - Blacklist TOCTOU (remove_if) and replace_redis no longer empties the table during a Redis reconnect. - RateLimiter and reputation have hard caps with incremental eviction instead of an unbounded DashMap and a stop-the-world retain. - The attack detector scores before updating its baseline and needs several quiet windows to leave UnderAttack, so a sustained flood keeps banning new offenders instead of ending after one tick. - accept() backs off on EMFILE/ENFILE, shutdown Err is treated as shutdown, a global semaphore bounds in-flight connections, rate limiting runs before PoW, whitelisted IPs are never banned, and connect/handshake/relay have timeouts with an idle budget. - PoW uses the configured difficulty as a floor, the clamp is enforced, and nonce framing survives split segments and coalesced handshake bytes. - Subnet tracker memory is capped (prefixes and per-prefix sources) and the monitor bans with the configured TTL. CI and docs: - CI runs the test suite with --no-default-features --features protocol-http; make ebpf fails on errors; the secrets gate catches weak compose defaults. - doctor skips protected endpoints without a token and exits non-zero on failure; a drain timeout returns exit code 1. - TODO §2b records the review findings, what is fixed and what is deferred; configuration/testing docs describe the new XDP and secret semantics. --- .github/workflows/ci.yml | 12 + Makefile | 6 +- TODO.md | 55 ++++ deploy/config/edge-docker.toml | 2 + deploy/docker-compose.yml | 19 +- deploy/test/README.md | 2 +- docker-compose.yml | 20 +- docs/configuration.md | 20 +- docs/deployment.md | 37 ++- scripts/check_default_secrets.sh | 93 +++--- src/app/runtime.rs | 7 +- src/app/shutdown.rs | 17 +- src/bin/rampart-manager.rs | 31 +- src/cli/commands/config.rs | 2 +- src/cli/commands/doctor.rs | 98 ++++-- src/config/mod.rs | 155 ++++------ src/engine/challenge/difficulty.rs | 66 +++-- src/engine/challenge/mod.rs | 2 +- src/engine/challenge/pow.rs | 122 ++++---- src/engine/listener.rs | 134 ++++++++- src/engine/subnet/monitor.rs | 138 +++++++-- src/engine/subnet/tracker.rs | 196 +++++++----- src/engine/tunnel/gateway.rs | 156 +--------- src/engine/tunnel/gateway/admission.rs | 67 +++++ src/engine/tunnel/gateway/challenge.rs | 4 + src/engine/tunnel/gateway/pipeline.rs | 208 +++++++++++++ src/engine/tunnel/gateway/relay.rs | 75 +++++ src/engine/tunnel/proxy.rs | 30 +- src/filter/blacklist/mod.rs | 184 +++++++----- src/filter/blacklist/prefix.rs | 119 ++++++++ src/filter/rate_limit.rs | 232 ++++++++------- src/manager/api/auth.rs | 50 +--- src/manager/mod.rs | 1 + src/manager/security.rs | 135 +++++++++ src/store/clickhouse.rs | 221 +++++++++++--- src/store/redis.rs | 40 ++- src/store/redis/backoff.rs | 14 + src/store/redis/connection.rs | 36 +++ src/store/redis/sync.rs | 90 +++--- src/store/redis/sync/event.rs | 73 +++-- src/store/redis/sync/publisher.rs | 79 +++-- src/store/redis/sync/snapshot.rs | 57 ++-- src/traffic/intel/detector.rs | 217 +++++++++----- src/traffic/intel/ewma.rs | 86 ++++-- src/traffic/intel/reputation.rs | 317 +++++++++++++------- src/traffic/prefix/key.rs | 11 +- tests/app_shutdown.rs | 15 + tests/clickhouse_writer.rs | 81 +++++ tests/config_parse.rs | 124 ++++++++ tests/connection_limits.rs | 313 ++++++++++++++++++++ tests/filter_logic.rs | 198 +++++++++++++ tests/manager_security.rs | 124 ++++++++ tests/pow_difficulty.rs | 80 +++++ tests/pow_roundtrip.rs | 111 ++++++- tests/redis_backoff.rs | 25 ++ tests/subnet_detection.rs | 164 ++++++++++- tests/traffic_intel.rs | 393 +++++++++++++++++++++++-- 57 files changed, 4193 insertions(+), 1171 deletions(-) create mode 100644 src/engine/tunnel/gateway/admission.rs create mode 100644 src/engine/tunnel/gateway/pipeline.rs create mode 100644 src/engine/tunnel/gateway/relay.rs create mode 100644 src/filter/blacklist/prefix.rs create mode 100644 src/manager/security.rs create mode 100644 src/store/redis/backoff.rs create mode 100644 src/store/redis/connection.rs create mode 100644 tests/app_shutdown.rs create mode 100644 tests/clickhouse_writer.rs create mode 100644 tests/connection_limits.rs create mode 100644 tests/manager_security.rs create mode 100644 tests/pow_difficulty.rs create mode 100644 tests/redis_backoff.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ee9f084..04f1f90 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,6 +49,18 @@ jobs: - run: sudo apt-get update && sudo apt-get install -y libelf-dev zlib1g-dev libbpf-dev clang linux-libc-dev - run: cargo test --all-features + rust-test-no-default-features: + name: Rust — test (no default features, protocol-http) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - run: sudo apt-get update && sudo apt-get install -y libelf-dev zlib1g-dev libbpf-dev clang linux-libc-dev + # Feature-gated code must actually run: without this job the + # `--no-default-features` combination is only type-checked, never tested. + - run: cargo test --no-default-features --features protocol-http + repo-gates: name: Repo — module size & default secrets runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index 3796aeb..af78429 100644 --- a/Makefile +++ b/Makefile @@ -52,9 +52,9 @@ audit: ebpf: @echo "Building XDP/eBPF filter..." - mkdir -p $(TARGET_DIR)/xdp && \ - clang -O2 -g -target bpf -c xdp/core/universal_filter.c -o $(TARGET_DIR)/xdp/universal_filter.o 2>/dev/null || \ - echo "WARNING: clang+bpf not installed, skipping eBPF build" + mkdir -p $(TARGET_DIR)/xdp + clang -O2 -g -target bpf -I"/usr/include/$$(gcc -print-multiarch)" \ + -c xdp/core/universal_filter.c -o $(TARGET_DIR)/xdp/universal_filter.o ebpf-clean: rm -f $(TARGET_DIR)/xdp/universal_filter.o diff --git a/TODO.md b/TODO.md index bd0d12f..f38e655 100644 --- a/TODO.md +++ b/TODO.md @@ -241,6 +241,61 @@ guard/ `value_size` `.rodata` == `RODATA_SIZE`, без root) + перезапуск реального бинаря в netns (attach подтверждён), плюс весь `scripts/xdp_ipv6_smoke.sh`. +## 2b. Ревью 2026-11-16 — качество и надёжность + +Четыре независимых ревью (XDP/загрузчик, data-path, фильтры/хранилища, API/конфиг/CI) +и один раунд фиксов. Гейты после раунда: fmt, clippy `-D warnings`, `cargo test +--all-features` (258 тестов), `--no-default-features` + тесты с `protocol-http`, +module-size, secrets, `cargo deny`, rustdoc без warning'ов, clang+verifier, +`xdp_ipv6_smoke.sh`, `xdp_pkt_test.sh`, живой attach в netns (native и generic). + +### Исправлено + +- XDP: длина TCP-payload по `ip->tot_len`/`payload_len` — на реальном NIC паддинг больше не + ломает `expected_seq` (было: каждое соединение умирало на 4-м сегменте). Регрессия — + `scripts/xdp_pkt_test.sh`. +- XDP: чтение map-значения до `bpf_map_delete_elem` (UAF), сброс `fails`, TFO в `expected_seq`, + исправленный v6 SYN-ACK, `inc_drop()` на всех DROP-путях, per-CPU инкремент без atomic. +- XDP: VLAN/QinQ до L3, фрагменты проверяются по blacklist, неизвестный non-SYN поток → PASS + («stateless-фильтр, не firewall»), `whitelist6_map`, whitelist до state machine. +- XDP: протухшие префиксные баны удаляются userspace-reaper'ом раз в 60 с (пакетный путь + больше не пытается удалить `/24` ключом `/32`). +- Загрузчик: запоминается фактический attach-флаг (native/skb) и проверяется detach; + `DetachGuard` откатывает attach при ошибке; whitelist из конфига грузится в карты; + `ban_cidr` валидирует и маскирует префикс; статистика знает все 13 счётчиков. +- Security: `API_PASSWORD` не пустой и ≥16 символов, `JWT_SECRET` ≥32 непробельных, + `syn_challenge_enabled` требует свой `challenge_secret_hex`, compose без публичных + Redis/ClickHouse и без дефолтных паролей, пустой `CORS_ORIGIN` → deny. +- Storage: ClickHouse с таймаутами и без lock через `.await` (bounded-буфер, метрика дропов), + Redis на `ConnectionManager` с таймаутами и реконнектом, баны из pubsub клампятся, + TOCTOU в blacklist, атомарный `replace_redis`. +- Data path: backoff на `accept()`, корректный `shutdown.changed() == Err`, глобальный бюджет + соединений, rate-limit **до** PoW, whitelist не банится, таймауты connect/handshake/relay, + idle-таймаут relay. +- Детекция: score считается до обновления baseline + гистерезис выхода из `UnderAttack` + (атака больше не «затухает» через тик); реальный cap и инкрементальная эвикция в + rate-limiter и репутации; IPv4-mapped ключи совпадают. +- Blacklist: CIDR-баны хранятся диапазонами — `/24`/`/64` покрывают всю подсеть; + нормализация IPv4-mapped на входе и на lookup. +- CI: джоба `cargo test --no-default-features --features protocol-http`, `make ebpf` больше + не глотает ошибки, secrets-гейт ловит слабые compose-дефолты. +- `doctor` не дёргает защищённые ручки без токена и возвращает ненулевой код при провале. +- Инлайн-тесты: 25 → 10 файлов (остальные — в следующих итерациях). + +### Осталось (следующий раунд) + +- [ ] API manager'а: `KEYS` → `SCAN`, batch `MGET` вместо N+1, ошибки Redis → 503 (а не + `200 []`), `DefaultBodyLimit`/`TimeoutLayer`, лимит длины `reason`, graceful shutdown, + эвикция login-лимитера, ключ по `X-Forwarded-For` за доверенным прокси. +- [ ] `trusted_cidrs` + PROXY-header IP как ключ всех политик: банить/лимитировать по + transport-IP, header-IP — только после валидации (ban-amplification через LB). +- [ ] XDP spoof-gate: `unique_sources = 0` на XDP-пути делает `Block` недостижимым — + либо counting в ядре, либо явная документация (сейчас логируется). +- [ ] IPv6 bind в listener (`Domain::IPV4` + строковый парсинг адреса). +- [ ] `relay_idle_timeout_secs` в `[limits]` (сейчас константа 300 с) и семафор бюджета + полем `Gateway` вместо `OnceLock`. +- [ ] Оставшиеся инлайн-тесты (10 файлов) мигрировать в `tests/`. + ## 3. Backlog - [ ] Мигрировать существующие инлайн-тесты (`#[cfg(test)]` в src/) в `tests/` — diff --git a/deploy/config/edge-docker.toml b/deploy/config/edge-docker.toml index c0e06b6..d712aed 100644 --- a/deploy/config/edge-docker.toml +++ b/deploy/config/edge-docker.toml @@ -28,6 +28,8 @@ rate_limit_burst = 10.0 ban_duration_secs = 3600 [store] +# deploy/docker-compose.yml requires REDIS_PASSWORD; add it to the URL when the +# edge uses that stack: redis://:PASSWORD@redis:6379/0 redis_url = "redis://redis:6379/0" blacklist_cache_ttl_secs = 300 clickhouse_url = "" diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 1ad6867..ec66f20 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -1,19 +1,26 @@ services: redis: image: redis:7-alpine - restart: unless-stopped + # No published host port: only containers on this network reach Redis, and + # the password is mandatory so an unset variable aborts startup. + environment: + REDIS_PASSWORD: "${REDIS_PASSWORD:?REDIS_PASSWORD must be set (see docs/deployment.md)}" + command: redis-server --requirepass "${REDIS_PASSWORD:?REDIS_PASSWORD must be set (see docs/deployment.md)}" healthcheck: - test: ["CMD", "redis-cli", "ping"] + test: ["CMD-SHELL", "redis-cli -a \"$$REDIS_PASSWORD\" --no-auth-warning ping | grep -q PONG"] interval: 3s timeout: 1s retries: 5 + restart: unless-stopped clickhouse: image: clickhouse/clickhouse-server:latest container_name: rampart-clickhouse + # ClickHouse has no authentication here: bind it to loopback only and keep + # container-to-container traffic on the Compose network (service name). ports: - - "8123:8123" - - "9000:9000" + - "127.0.0.1:8123:8123" + - "127.0.0.1:9000:9000" volumes: - ./clickhouse/schema.sql:/docker-entrypoint-initdb.d/schema.sql:ro healthcheck: @@ -27,12 +34,12 @@ services: image: grafana/grafana:latest container_name: rampart-grafana ports: - - "3000:3000" + - "127.0.0.1:3000:3000" depends_on: clickhouse: condition: service_started environment: - GF_INSTALL_PLUGINS: grafana-clickhouse-datasource + GF_SECURITY_ADMIN_PASSWORD: "${GRAFANA_PASSWORD:?GRAFANA_PASSWORD must be set (see docs/deployment.md)}" volumes: - ./grafana/datasources:/etc/grafana/provisioning/datasources:ro - ./grafana/dashboards:/etc/grafana/provisioning/dashboards:ro diff --git a/deploy/test/README.md b/deploy/test/README.md index b7a0023..52db9ad 100644 --- a/deploy/test/README.md +++ b/deploy/test/README.md @@ -70,4 +70,4 @@ rampart_attack_status ``` ClickHouse (опционально): http://localhost:8123 — долгосрочное хранение событий. -Grafana: http://localhost:3000 (admin/admin) — дашборды из `deploy/grafana/`. +Grafana: http://localhost:3000 (логин `admin`, пароль из `GRAFANA_PASSWORD`) — дашборды из `deploy/grafana/`. diff --git a/docker-compose.yml b/docker-compose.yml index 3d99ff2..4093dec 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,13 +2,17 @@ services: redis: image: redis:7-alpine container_name: rampart-redis - command: redis-server --requirepass "${REDIS_PASSWORD:-rampart_dev}" --appendonly yes + # Redis is reachable only from this Compose network and the loopback + # interface; the password is mandatory so an unset variable aborts startup. ports: - - "6379:6379" + - "127.0.0.1:6379:6379" + environment: + REDIS_PASSWORD: "${REDIS_PASSWORD:?REDIS_PASSWORD must be set (see docs/deployment.md)}" + command: redis-server --requirepass "${REDIS_PASSWORD:?REDIS_PASSWORD must be set (see docs/deployment.md)}" --appendonly yes volumes: - redis-data:/data healthcheck: - test: ["CMD", "redis-cli", "--raw", "incr", "ping"] + test: ["CMD-SHELL", "redis-cli -a \"$$REDIS_PASSWORD\" --no-auth-warning ping | grep -q PONG"] interval: 5s timeout: 3s retries: 5 @@ -17,9 +21,11 @@ services: clickhouse: image: clickhouse/clickhouse-server:24-alpine container_name: rampart-clickhouse + # ClickHouse has no authentication here: bind it to loopback only and keep + # container-to-container traffic on the Compose network (service name). ports: - - "8123:8123" # HTTP - - "9000:9000" # Native TCP + - "127.0.0.1:8123:8123" # HTTP + - "127.0.0.1:9000:9000" # Native TCP volumes: - clickhouse-data:/var/lib/clickhouse - ./deploy/clickhouse/schema.sql:/docker-entrypoint-initdb.d/schema.sql:ro @@ -34,9 +40,9 @@ services: image: grafana/grafana:latest container_name: rampart-grafana ports: - - "3000:3000" + - "127.0.0.1:3000:3000" environment: - - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD:-admin} + - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD:?GRAFANA_PASSWORD must be set (see docs/deployment.md)} - GF_INSTALL_PLUGINS=grafana-clickhouse-datasource volumes: - grafana-data:/var/lib/grafana diff --git a/docs/configuration.md b/docs/configuration.md index 23f9aa3..3ff65c5 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -92,11 +92,11 @@ max_header_bytes = 512 | Переменная | Обязательна | По умолчанию | Описание | |------------|-------------|--------------|----------| | `REDIS_URL` | нет | `redis://127.0.0.1:6379/0` | Хранилище состояния | -| `JWT_SECRET` | да | — | Минимум 32 байта | +| `JWT_SECRET` | да | — | Минимум 32 непробельных байта | | `JWT_AUDIENCE` | нет | `rampart` | Audience JWT | | `JWT_EXPIRATION_SECS` | нет | `86400` | Время жизни токена | -| `API_PASSWORD` | да | — | Пароль для `/api/v1/auth/login`; значение `changeme` запрещено | -| `CORS_ORIGIN` | нет | `http://localhost:5173` | CORS; `*` или пусто — разрешить все | +| `API_PASSWORD` | да | — | Пароль для `/api/v1/auth/login`, минимум 16 символов; `changeme` запрещён | +| `CORS_ORIGIN` | нет | — (CORS выключен) | `*` — разрешить все источники; список через запятую; пусто/не задано — запретить cross-origin | Manager всегда слушает `0.0.0.0:8080`. @@ -136,9 +136,21 @@ clang -O2 -g -target bpf -c xdp/core/universal_filter.c -o /tmp/universal_filter | `auto_ban_duration_secs` | `300` | Длительность авто-бана | | `auto_ban_conn_drop_fails` | `3` | Порог `conn->fails` для бана по `EVENT_CONN_DROP` | | `syn_challenge_enabled` | `false` | RST-challenge (Oubliette liveness proof) | -| `challenge_secret_hex` | плейсхолдер | 128-битный hex-секрет challenge | +| `challenge_secret_hex` | — (обязателен при `syn_challenge_enabled`) | 128-битный hex-секрет, например `openssl rand -hex 8`; встроенный плейсхолдер запрещён | | `challenge_timeout_ms` | `3000` | TTL pending-challenge | +Поведение XDP-фильтра: + +- **Stateless-фильтр, не firewall.** Состояние conntrack ведётся только для TCP-соединений, + начатых с SYN. Пакет без состояния (non-SYN) пропускается: включение фильтра не рвёт + ответы на исходящие соединения самого демона и keep-alive от health-check'ов. +- **VLAN/QinQ** (802.1Q и 802.1ad, до двух тегов) разбираются; **фрагменты** проверяются по + source-IP в blacklist и пропускаются (не дропаются), счётчик — `frag` в `stats_map`. +- **Whitelist** из корневого `whitelist = [...]` загружается в XDP-карты (`whitelist_map`, + `whitelist6_map`) при attach и обходит state machine, throttle и баны. +- **CIDR-баны**: XDP матчит по LPM-трие, userspace — по диапазонам адресов, поэтому бан + `/24` или `/64` покрывает всю подсеть. + Покрытие семейств адресов: - **IPv4**: blacklist, SYN-throttle, conntrack, RST-challenge, ringbuf-события, `udp_policy` diff --git a/docs/deployment.md b/docs/deployment.md index 2009310..19db2ee 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -73,24 +73,34 @@ Paper и закрой backend-порт firewall-ом от всех, кроме R ```bash export REDIS_URL="redis://127.0.0.1:6379/0" -export JWT_SECRET="$(openssl rand -hex 32)" # минимум 32 байта -export API_PASSWORD="не-changeme" # дефолт запрещён +export JWT_SECRET="$(openssl rand -hex 32)" # минимум 32 байта без пробелов +export API_PASSWORD="$(openssl rand -base64 24)" # минимум 16 символов, дефолт запрещён cargo run --release --bin rampart-manager # API на 0.0.0.0:8080 ``` +Manager refuses to start with an empty `API_PASSWORD`, a password shorter than +16 characters, the default placeholder (which must not be `changeme`), or a +`JWT_SECRET` with fewer than 32 non-whitespace bytes. + Manager listens on all interfaces by default. Permit `8080` only from an admin VPN or specific admin addresses; do not expose the API to the public Internet. The same applies to Redis (`6379`) and monitoring endpoints such as Prometheus (`9090`), ClickHouse (`8123`/`9000`) and Grafana (`3000`). The sample Compose -file publishes ClickHouse and Grafana ports on the host, so restrict them with -the host firewall or change their port bindings before deploying on a public -VDS. Redis has no published host port in that Compose file and is reachable -only by containers on its Docker network unless you deliberately expose it. +files bind published ClickHouse and Grafana ports to `127.0.0.1` only, so reach +them through an SSH tunnel or a local admin session, never through the public +interface. `deploy/docker-compose.yml` publishes no Redis port at all and the +root `docker-compose.yml` binds it to loopback; both refuse to start unless +`REDIS_PASSWORD` and `GRAFANA_PASSWORD` are set. Cross-origin access to the +manager API is disabled unless `CORS_ORIGIN` is set explicitly (`*` or a +comma-separated origin list). -Инфраструктура (Redis, ClickHouse, Grafana): +Инфраструктура (Redis, ClickHouse, Grafana) требует пароли из окружения — +Compose не стартует без них: ```bash +export REDIS_PASSWORD="$(openssl rand -base64 24)" +export GRAFANA_PASSWORD="$(openssl rand -base64 24)" docker compose up -d redis clickhouse grafana ``` @@ -103,6 +113,10 @@ Redis только через private network/VPN; при пустом `redis_ur ## 5. Docker ```bash +# Пароли обязательны: без REDIS_PASSWORD/GRAFANA_PASSWORD Compose падает +export REDIS_PASSWORD="$(openssl rand -base64 24)" +export GRAFANA_PASSWORD="$(openssl rand -base64 24)" + # Образ edge docker build -f deploy/docker/Dockerfile.edge -t rampart-edge . @@ -110,6 +124,15 @@ docker build -f deploy/docker/Dockerfile.edge -t rampart-edge . docker compose -f deploy/docker-compose.yml up -d ``` +Edge подключается к Redis по имени сервиса. Поскольку Redis в Compose требует +пароль, впиши его в URL конфига edge (`deploy/config/edge-docker.toml`) — тот +же пароль, что в `REDIS_PASSWORD`: + +```toml +[store] +redis_url = "redis://:PASSWORD@redis:6379/0" +``` + ## 6. Проверка после деплоя ```bash diff --git a/scripts/check_default_secrets.sh b/scripts/check_default_secrets.sh index fb54e13..cd8e40a 100755 --- a/scripts/check_default_secrets.sh +++ b/scripts/check_default_secrets.sh @@ -1,15 +1,21 @@ #!/usr/bin/env bash # -# check_default_secrets.sh — reject the placeholder secret "changeme" -# committed as a real value in code, deploy configs, or docs. +# check_default_secrets.sh — reject placeholder and weak secrets committed as +# real values in code, deploy configs, or docs. +# +# Catches: +# 1. the literal placeholder "changeme"; +# 2. Compose-style fallbacks to weak defaults, e.g. ${REDIS_PASSWORD:-rampart_dev} +# or ${GRAFANA_PASSWORD:-admin}; +# 3. Redis `requirepass` with a weak or empty password. # # Intentional occurrences are whitelisted in two layers: # 1. File level — the validation code itself: # src/bin/rampart-manager.rs (startup guard rejecting the default) -# src/manager/api/auth.rs (test constant) -# 2. Line level — only lines that *use* "changeme" as a value are reported. -# A matching line is skipped when it contains "must not be" or "test" -# (case-insensitive), states the prohibition ("запр" root: запрет / +# src/manager/security.rs (password/secret policy + placeholders) +# src/manager/api/auth.rs (login handler) +# 2. Line level — a matching line is skipped when it contains "must not be" or +# "test" (case-insensitive), states the prohibition ("запр" root: запрет / # запрещено / запрещён — the project docs are partly Russian), or is a # comment / markdown table row (starts with '#', '//', '*', '|', '--'). # @@ -17,47 +23,62 @@ set -u -PATTERN="changeme" SCAN_DIRS="src deploy docs" -FILE_WHITELIST="src/bin/rampart-manager.rs src/manager/api/auth.rs" +FILE_WHITELIST="src/bin/rampart-manager.rs src/manager/security.rs src/manager/api/auth.rs" + +# "@@" pairs. The `|` inside alternations is +# safe because entries are split on the first `@@`. +PATTERNS=( + 'changeme@@placeholder password "changeme"' + '\$\{[A-Za-z_][A-Za-z0-9_]*:-[[:space:]]*(rampart_dev|admin|password|secret|root)@@fallback to a weak default value' + "requirepass[[:space:]]+[\"']?[[:space:]]*(rampart_dev|admin|password|redis|secret|root)@@requirepass with a weak password" + "requirepass[[:space:]]*([\"']{2})?[[:space:]]*\$@@requirepass without a password" +) cd "$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" || exit 1 -found=0 +offenders=$(mktemp) +trap 'rm -f "$offenders"' EXIT + for dir in $SCAN_DIRS; do [ -d "$dir" ] || continue - while IFS= read -r match; do - file="${match%%:*}" - rest="${match#*:}" - lineno="${rest%%:*}" - content="${rest#*:}" + for entry in "${PATTERNS[@]}"; do + pattern="${entry%%@@*}" + description="${entry#*@@}" + while IFS= read -r match; do + [ -n "$match" ] || continue + file="${match%%:*}" + rest="${match#*:}" + lineno="${rest%%:*}" + content="${rest#*:}" - skip=0 - for whitelisted in $FILE_WHITELIST; do - [ "$file" = "$whitelisted" ] && skip=1 - done - if printf '%s' "$content" | grep -qiE 'must not be|test|запр'; then - skip=1 - fi - trimmed="${content#"${content%%[![:space:]]*}"}" - if [ "${trimmed:0:2}" = "//" ] || [ "${trimmed:0:2}" = "--" ]; then - skip=1 - fi - case "${trimmed:0:1}" in - '#'|'*'|'|') skip=1 ;; - esac + skip=0 + for whitelisted in $FILE_WHITELIST; do + [ "$file" = "$whitelisted" ] && skip=1 + done + if printf '%s' "$content" | grep -qiE 'must not be|test|запр'; then + skip=1 + fi + trimmed="${content#"${content%%[![:space:]]*}"}" + if [ "${trimmed:0:2}" = "//" ] || [ "${trimmed:0:2}" = "--" ]; then + skip=1 + fi + case "${trimmed:0:1}" in + '#'|'*'|'|') skip=1 ;; + esac - if [ "$skip" -eq 0 ]; then - found=1 - printf 'FAIL %s:%s: %s\n' "$file" "$lineno" "$trimmed" >&2 - fi - done < <(grep -rn --binary-files=without-match "$PATTERN" "$dir" 2>/dev/null) + if [ "$skip" -eq 0 ]; then + printf 'FAIL %s:%s: %s [%s]\n' "$file" "$lineno" "$trimmed" "$description" >> "$offenders" + fi + done < <(grep -rnE --binary-files=without-match "$pattern" "$dir" 2>/dev/null) + done done -if [ "$found" -ne 0 ]; then - echo "secrets gate: FAIL — 'changeme' used as a value (see lines above)" >&2 +if [ -s "$offenders" ]; then + sort -u "$offenders" >&2 + echo "secrets gate: FAIL — placeholder/weak secret used as a value (see lines above)" >&2 exit 1 fi -echo "secrets gate: PASS — no unexpected '$PATTERN' occurrences in: $SCAN_DIRS" +echo "secrets gate: PASS — no placeholder or weak default secrets in: $SCAN_DIRS" exit 0 diff --git a/src/app/runtime.rs b/src/app/runtime.rs index c576c34..91142d2 100644 --- a/src/app/runtime.rs +++ b/src/app/runtime.rs @@ -132,6 +132,9 @@ pub async fn run() -> anyhow::Result { let subnet_tracker = start_subnet_tracker(&config); monitor::spawn( &config.detect.prefix, + Duration::from_secs(config.ban.ban_duration_secs), + blacklist.clone(), + !config.proxy_protocol.ingress_enabled, subnet_tracker.clone(), xdp_filter.clone(), shutdown_rx.clone(), @@ -190,7 +193,9 @@ pub async fn run() -> anyhow::Result { tracing::info!("Rampart edge starting on {}:{}", config.bind.address, config.bind.port); tracing::info!("Upstreams: {:?}", config.backend.upstreams); - let adjuster = Arc::new(Mutex::new(DifficultyAdjuster::default())); + // The configured difficulty is the floor and the starting point; the + // adjuster only raises it under load. + let adjuster = Arc::new(Mutex::new(DifficultyAdjuster::with_floor(config.pow.difficulty))); let gateway = Arc::new(Gateway::new( config.clone(), rate_limiter, diff --git a/src/app/shutdown.rs b/src/app/shutdown.rs index bcba0fd..198a7a4 100644 --- a/src/app/shutdown.rs +++ b/src/app/shutdown.rs @@ -18,11 +18,14 @@ pub enum RunExit { impl RunExit { /// Process exit code for this outcome. + /// + /// A forced drain timeout loses connections that did not finish in time, so + /// it must not look like a clean shutdown to supervisors and scripts. #[must_use] pub const fn code(self) -> i32 { match self { Self::Ok => 0, - Self::DrainTimeout => 0, + Self::DrainTimeout => 1, } } } @@ -53,15 +56,3 @@ async fn wait_for_signal() { _ = term.recv() => {} } } - -#[cfg(test)] -mod tests { - use super::RunExit; - - #[test] - fn run_exit_codes_match_previous_process_exit() { - // Both the clean path and the former drain timeout (`process::exit(0)`) exited with code 0. - assert_eq!(RunExit::Ok.code(), 0); - assert_eq!(RunExit::DrainTimeout.code(), 0); - } -} diff --git a/src/bin/rampart-manager.rs b/src/bin/rampart-manager.rs index 4d04421..4f6c6bb 100644 --- a/src/bin/rampart-manager.rs +++ b/src/bin/rampart-manager.rs @@ -1,17 +1,14 @@ //! Rampart manager daemon: HTTP API in front of the redis state store. use axum::{ - Router, - http::HeaderValue, - middleware, + Router, middleware, routing::{get, post}, }; use dashmap::DashMap; #[cfg(feature = "store-redis")] use rampart::manager::sync; -use rampart::manager::{AppState, api, auth}; +use rampart::manager::{AppState, api, auth, security}; use std::sync::Arc; -use tower_http::cors::CorsLayer; use tracing_subscriber::EnvFilter; #[tokio::main] @@ -32,9 +29,7 @@ async fn main() -> anyhow::Result<()> { let redis_client = redis::Client::open(redis_url)?; let jwt_secret = std::env::var("JWT_SECRET").map_err(|_| anyhow::anyhow!("JWT_SECRET must be set"))?; - if jwt_secret.len() < 32 { - return Err(anyhow::anyhow!("JWT_SECRET must be at least 32 bytes")); - } + security::validate_jwt_secret(&jwt_secret)?; let jwt_audience = std::env::var("JWT_AUDIENCE").unwrap_or_else(|_| "rampart".to_string()); let jwt_expiration = std::env::var("JWT_EXPIRATION_SECS") .unwrap_or_else(|_| "86400".to_string()) @@ -42,9 +37,7 @@ async fn main() -> anyhow::Result<()> { .map_err(|_| anyhow::anyhow!("JWT_EXPIRATION_SECS must be a valid u64"))?; let api_password = std::env::var("API_PASSWORD").map_err(|_| anyhow::anyhow!("API_PASSWORD must be set"))?; - if api_password == "changeme" { - return Err(anyhow::anyhow!("API_PASSWORD must not be the default 'changeme'")); - } + security::validate_api_password(&api_password)?; let state = Arc::new(AppState { #[cfg(feature = "store-redis")] @@ -78,17 +71,13 @@ async fn main() -> anyhow::Result<()> { .route("/api/v1/nodes", get(api::inventory::nodes::list_nodes)); let protected = protected.route_layer(middleware::from_fn_with_state(state.clone(), auth::auth_middleware)); - let cors = match std::env::var("CORS_ORIGIN") { - Ok(origin) if origin.is_empty() || origin == "*" => CorsLayer::new().allow_origin(tower_http::cors::Any), - Ok(origin) => CorsLayer::new().allow_origin(HeaderValue::from_str(&origin)?), - Err(_) => CorsLayer::new().allow_origin(HeaderValue::from_static("http://localhost:5173")), - }; + let cors_policy = security::CorsPolicy::parse(std::env::var("CORS_ORIGIN").ok().as_deref())?; + tracing::info!("CORS policy: {cors_policy:?}"); - let app = Router::new() - .merge(public) - .merge(protected) - .layer(cors) - .with_state(state); + let mut app = Router::new().merge(public).merge(protected).with_state(state); + if let Some(cors) = cors_policy.cors_layer() { + app = app.layer(cors); + } let addr = "0.0.0.0:8080"; tracing::info!("Manager API listening on {addr}"); diff --git a/src/cli/commands/config.rs b/src/cli/commands/config.rs index 45c716c..17db4c6 100644 --- a/src/cli/commands/config.rs +++ b/src/cli/commands/config.rs @@ -14,7 +14,7 @@ pub async fn run(key: Option, value: Option) -> anyhow::Result<( println!(); println!("Example keys:"); println!(" workers.count"); - println!(" limits.rate_limit_login_pps"); + println!(" limits.rate_limit_pps"); println!(" limits.max_connections_per_ip"); Ok(()) }, diff --git a/src/cli/commands/doctor.rs b/src/cli/commands/doctor.rs index 95e28db..0836ac5 100644 --- a/src/cli/commands/doctor.rs +++ b/src/cli/commands/doctor.rs @@ -2,6 +2,11 @@ use crate::xdp::{EnvironmentReport, FilesystemProbe, SystemProbe}; +/// Environment variable holding a ready-to-use manager bearer token. +const TOKEN_ENV: &str = "RAMPART_TOKEN"; +/// Environment variable holding the manager API password used to log in. +const PASSWORD_ENV: &str = "RAMPART_API_PASSWORD"; + /// Finds the interface that owns the default route in `/proc/net/route` /// (destination `00000000`). Works without root. fn default_route_interface(probe: &impl SystemProbe) -> Option { @@ -12,6 +17,51 @@ fn default_route_interface(probe: &impl SystemProbe) -> Option { }) } +/// Obtains a bearer token for the protected manager endpoints. +/// +/// `RAMPART_TOKEN` is used as is; otherwise `RAMPART_API_PASSWORD` is exchanged +/// for a token via `/api/v1/auth/login`. Protected endpoints are never called +/// anonymously, so `None` means the caller has to be told to provide a +/// credential instead of seeing a misleading failure. +async fn manager_token(client: &reqwest::Client, manager_url: &str) -> Option { + if let Ok(token) = std::env::var(TOKEN_ENV) { + if !token.trim().is_empty() { + return Some(token); + } + } + let password = std::env::var(PASSWORD_ENV).ok().filter(|value| !value.is_empty())?; + let response = client + .post(format!("{manager_url}/api/v1/auth/login")) + .json(&serde_json::json!({"password": password})) + .send() + .await + .ok()?; + if !response.status().is_success() { + println!("[WARN] Manager login rejected ({})", response.status()); + return None; + } + let body = response.json::().await.ok()?; + body["token"].as_str().map(str::to_owned) +} + +/// Probes one protected manager endpoint with a bearer token. +async fn check_protected(client: &reqwest::Client, manager_url: &str, path: &str, label: &str, token: Option<&str>) { + let Some(token) = token else { + println!("[SKIP] {label}: set {TOKEN_ENV} or {PASSWORD_ENV} to check it"); + return; + }; + match client + .get(format!("{manager_url}{path}")) + .bearer_auth(token) + .send() + .await + { + Ok(resp) if resp.status().is_success() => println!("[OK] {label}"), + Ok(resp) => println!("[WARN] {label} unavailable ({})", resp.status()), + Err(e) => println!("[WARN] {label} unavailable: {e}"), + } +} + /// Network/XDP section: what the environment exposes without root. fn print_network_section(all_ok: &mut bool) { println!("\nNetwork / XDP"); @@ -60,6 +110,10 @@ fn print_network_section(all_ok: &mut bool) { } /// Runs every diagnostic and prints the report. +/// +/// # Errors +/// Returns an error when any check failed, so scripts and CI see a non-zero +/// exit status instead of a report that only looks alarming. pub async fn run() -> anyhow::Result<()> { println!("Rampart Diagnostics"); println!("=====================\n"); @@ -67,8 +121,9 @@ pub async fn run() -> anyhow::Result<()> { let mut all_ok = true; let manager_url = std::env::var("RAMPART_MANAGER").unwrap_or_else(|_| "http://localhost:8080".to_string()); + let client = reqwest::Client::new(); - match reqwest::get(format!("{manager_url}/api/v1/health")).await { + match client.get(format!("{manager_url}/api/v1/health")).send().await { Ok(resp) if resp.status().is_success() => { println!("[OK] Manager API"); }, @@ -78,32 +133,31 @@ pub async fn run() -> anyhow::Result<()> { }, } - match reqwest::get(format!("{manager_url}/api/v1/blacklist")).await { - Ok(resp) if resp.status().is_success() => { - println!("[OK] Blacklist API"); - }, - _ => { - println!("[WARN] Blacklist API unavailable"); - }, - } - - match reqwest::get(format!("{manager_url}/api/v1/servers")).await { - Ok(resp) if resp.status().is_success() => { - println!("[OK] Servers API"); - }, - _ => { - println!("[WARN] Servers API unavailable"); - }, - } + let token = manager_token(&client, &manager_url).await; + check_protected( + &client, + &manager_url, + "/api/v1/blacklist", + "Blacklist API", + token.as_deref(), + ) + .await; + check_protected( + &client, + &manager_url, + "/api/v1/servers", + "Servers API", + token.as_deref(), + ) + .await; print_network_section(&mut all_ok); println!(); - if all_ok { - println!("All checks passed."); - } else { + if !all_ok { println!("Some checks failed. Run with --verbose for details."); + anyhow::bail!("doctor: one or more checks failed"); } - + println!("All checks passed."); Ok(()) } diff --git a/src/config/mod.rs b/src/config/mod.rs index 03aa1c2..6f9996e 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -19,6 +19,14 @@ pub use sections::{ use serde::Deserialize; use std::fs; +/// Upper bound of `ban.ban_duration_secs`: one year. +const MAX_BAN_DURATION_SECS: u64 = 31_536_000; +/// Upper bound of `pow.difficulty`. +const MAX_POW_DIFFICULTY: u8 = 10; +/// Placeholder `xdp.challenge_secret_hex` values baked into the XDP defaults +/// (`xdp/core/config.h` and `XdpGlobals::default()`; both spellings exist). +const PLACEHOLDER_CHALLENGE_SECRETS: [&str; 2] = ["a5a5a5a5a5a5a5a5", "a5a55a5a5a5aa5a5"]; + #[derive(Debug, Clone, Default, Deserialize)] pub struct Config { #[serde(default)] @@ -90,9 +98,38 @@ impl Config { if self.detect.prefix.enabled && self.detect.prefix.syn_threshold == 0 { anyhow::bail!("detect.prefix.syn_threshold must be positive when enabled"); } + if self.xdp.syn_challenge_enabled { + let secret = self.xdp.challenge_secret_hex.as_deref().ok_or_else(|| { + anyhow::anyhow!( + "xdp.syn_challenge_enabled requires xdp.challenge_secret_hex; \ + generate one with `openssl rand -hex 8`" + ) + })?; + if is_placeholder_challenge_secret(secret) { + anyhow::bail!( + "xdp.challenge_secret_hex is the built-in placeholder; \ + generate a random secret with `openssl rand -hex 8`" + ); + } + } if self.limits.max_connections_per_ip == 0 { anyhow::bail!("limits.max_connections_per_ip must be positive"); } + if !self.limits.rate_limit_pps.is_finite() || self.limits.rate_limit_pps <= 0.0 { + anyhow::bail!("limits.rate_limit_pps must be a finite number greater than 0"); + } + if !self.limits.rate_limit_burst.is_finite() || self.limits.rate_limit_burst < 1.0 { + anyhow::bail!("limits.rate_limit_burst must be a finite number at least 1"); + } + if self.ban.ban_duration_secs == 0 || self.ban.ban_duration_secs > MAX_BAN_DURATION_SECS { + anyhow::bail!("ban.ban_duration_secs must be between 1 and {MAX_BAN_DURATION_SECS}"); + } + if self.pow.difficulty == 0 || self.pow.difficulty > MAX_POW_DIFFICULTY { + anyhow::bail!("pow.difficulty must be between 1 and {MAX_POW_DIFFICULTY}"); + } + if self.workers.count == 0 { + anyhow::bail!("workers.count must be at least 1: otherwise no accept loops are started"); + } if self.proxy_protocol.ingress_enabled { if self.proxy_protocol.trusted_cidrs.is_empty() { anyhow::bail!("proxy_protocol.ingress_enabled requires at least one trusted_cidrs entry"); @@ -138,109 +175,17 @@ impl Config { } } -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn empty_config_uses_defaults() { - let config = Config::parse_str("").expect("empty config should parse"); - assert_eq!(config.bind.address, "0.0.0.0"); - assert_eq!(config.bind.port, 25565); - assert_eq!(config.backend.upstreams, vec!["127.0.0.1:25566"]); - assert!(!config.pow.enabled); - assert!(!config.xdp.enabled); - } - - #[test] - fn full_config_parses() { - let config = Config::parse_str( - r#" -[bind] -address = "10.0.0.1" -port = 443 - -[backend] -upstreams = ["192.168.1.10:8080", "192.168.1.11:8080"] - -[pow] -enabled = true -difficulty = 6 - -whitelist = ["203.0.113.7"] -"#, - ) - .expect("config should parse"); - assert_eq!(config.bind.port, 443); - assert_eq!(config.backend.upstreams.len(), 2); - assert!(config.pow.enabled); - assert_eq!(config.pow.difficulty, 6); - } - - #[test] - fn empty_upstreams_rejected() { - let result = Config::parse_str( - r#" -[backend] -upstreams = [] -"#, - ); - assert!(result.is_err()); - } - - #[test] - fn invalid_upstream_rejected() { - let result = Config::parse_str( - r#" -[backend] -upstreams = ["not-an-addr"] -"#, - ); - assert!(result.is_err()); - } - - #[test] - fn invalid_whitelist_entry_rejected() { - let result = Config::parse_str("whitelist = [\"999.999.1.1\"]"); - assert!(result.is_err()); - } - - #[test] - fn prefix_detect_defaults_disabled() { - let config = Config::parse_str("").expect("empty config should parse"); - assert!(!config.detect.prefix.enabled); - assert_eq!(config.detect.prefix.syn_threshold, 500); - assert_eq!(config.detect.prefix.window_secs, 10); - assert_eq!(config.detect.prefix.min_unique_sources, 16); - } - - #[test] - fn prefix_detect_section_parses() { - let config = Config::parse_str( - r#" -[detect.prefix] -enabled = true -syn_threshold = 1000 -window_secs = 5 -min_unique_sources = 8 -"#, - ) - .expect("detect.prefix section should parse"); - assert!(config.detect.prefix.enabled); - assert_eq!(config.detect.prefix.syn_threshold, 1000); - assert_eq!(config.detect.prefix.window_secs, 5); - assert_eq!(config.detect.prefix.min_unique_sources, 8); - } - - #[test] - fn prefix_detect_zero_threshold_rejected_when_enabled() { - let result = Config::parse_str( - r#" -[detect.prefix] -enabled = true -syn_threshold = 0 -"#, - ); - assert!(result.is_err()); - } +/// Returns whether `secret` spells a built-in placeholder challenge secret. +/// +/// Hex digits are compared case-insensitively, with an optional `0x` prefix and +/// `_`, `:` and `-` separators ignored, so every spelling of the placeholder is +/// caught. +fn is_placeholder_challenge_secret(secret: &str) -> bool { + let normalized = secret + .trim() + .trim_start_matches("0x") + .trim_start_matches("0X") + .replace(['_', ':', '-'], "") + .to_ascii_lowercase(); + PLACEHOLDER_CHALLENGE_SECRETS.contains(&normalized.as_str()) } diff --git a/src/engine/challenge/difficulty.rs b/src/engine/challenge/difficulty.rs index 26a4bb1..47da3d5 100644 --- a/src/engine/challenge/difficulty.rs +++ b/src/engine/challenge/difficulty.rs @@ -4,10 +4,20 @@ use crate::metrics; use std::collections::VecDeque; use std::time::Instant; +/// Lowest usable difficulty: 0 disables the PoW gate before the adjuster runs. +pub const MIN_DIFFICULTY: u8 = 1; + +/// Highest adaptive difficulty. Every extra hex digit quadruples solver work. +pub const MAX_DIFFICULTY: u8 = 12; + +/// Difficulty the adjuster climbs to under the heaviest observed load. +const DEFAULT_MAX_DIFFICULTY: u8 = 10; + /// Adapts the PoW difficulty to the current connection rate. /// /// The rate is counted over a sliding one-second window and the resulting -/// difficulty is clamped to `[min, max]`. +/// difficulty is clamped to `[min, max]`. The lower bound is also the floor and +/// the starting difficulty, so a configured value takes effect immediately. pub struct DifficultyAdjuster { window: VecDeque, min: u8, @@ -22,17 +32,36 @@ impl Default for DifficultyAdjuster { } impl DifficultyAdjuster { - /// Creates an adjuster clamped to `min.max(4)` and `max.min(10)`. + /// Creates an adjuster with the explicit `[min, max]` range. + /// + /// Inverted bounds are swapped, and both ends are clamped to + /// [`MIN_DIFFICULTY`]..=[`MAX_DIFFICULTY`]. The lower bound doubles as the + /// starting difficulty. #[must_use] pub fn new(min: u8, max: u8) -> Self { + let (min, max) = if min > max { (max, min) } else { (min, max) }; + let min = min.clamp(MIN_DIFFICULTY, MAX_DIFFICULTY); + let max = max.clamp(min, MAX_DIFFICULTY); Self { window: VecDeque::new(), - min: min.max(4), - max: max.min(10), - current: min.max(4), + min, + max, + current: min, } } + /// Creates an adjuster whose floor and starting point is the configured PoW + /// difficulty. + /// + /// The ceiling stays at the default maximum, or rises to the configured + /// value when that value is higher. Values outside + /// [`MIN_DIFFICULTY`]..=[`MAX_DIFFICULTY`] are clamped. + #[must_use] + pub fn with_floor(difficulty: u8) -> Self { + let floor = difficulty.clamp(MIN_DIFFICULTY, MAX_DIFFICULTY); + Self::new(floor, floor.max(DEFAULT_MAX_DIFFICULTY)) + } + /// Records one connection and recomputes the difficulty for the window. pub fn record_connection(&mut self) { let now = Instant::now(); @@ -64,33 +93,18 @@ impl DifficultyAdjuster { self.current } - fn compute_difficulty(&self) -> u8 { + /// Rate step before it is clamped to the configured range. + fn rate_difficulty(&self) -> u8 { match self.window.len() { - cps if cps > 500 => self.max.max(self.min), + cps if cps > 500 => MAX_DIFFICULTY, cps if cps > 200 => 8, cps if cps > 50 => 6, _ => self.min, } } -} -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn adjuster_raises_difficulty_under_load() { - let mut adjuster = DifficultyAdjuster::new(4, 10); - for _ in 0..600 { - adjuster.record_connection(); - } - assert_eq!(adjuster.current_difficulty(), 10); - } - - #[test] - fn adjuster_stays_minimal_when_idle() { - let mut adjuster = DifficultyAdjuster::new(4, 10); - adjuster.record_connection(); - assert_eq!(adjuster.current_difficulty(), 4); + fn compute_difficulty(&self) -> u8 { + // The rate ladder is global; the configured range is authoritative. + self.rate_difficulty().clamp(self.min, self.max) } } diff --git a/src/engine/challenge/mod.rs b/src/engine/challenge/mod.rs index 461a3ce..e6bba73 100644 --- a/src/engine/challenge/mod.rs +++ b/src/engine/challenge/mod.rs @@ -4,5 +4,5 @@ mod difficulty; mod pow; -pub use difficulty::DifficultyAdjuster; +pub use difficulty::{DifficultyAdjuster, MAX_DIFFICULTY, MIN_DIFFICULTY}; pub use pow::{Challenge, enforce, solve}; diff --git a/src/engine/challenge/pow.rs b/src/engine/challenge/pow.rs index 057bc7f..79eaaac 100644 --- a/src/engine/challenge/pow.rs +++ b/src/engine/challenge/pow.rs @@ -1,3 +1,5 @@ +//! Universal SHA-256 hashcash challenge generation, solver and verifier. + use rand::Rng; use sha2::{Digest, Sha256}; use std::net::IpAddr; @@ -5,11 +7,15 @@ use std::time::{Duration, Instant}; use subtle::ConstantTimeEq; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpStream; +use tokio::time::timeout; const CHALLENGE_TTL_SECS: u64 = 30; const MAX_NONCE_LEN: usize = 64; const ALLOWED_HEX_PREFIX: [u8; 4] = *b"0123"; +/// Maximum time a peer gets to read the challenge and return a nonce. +const NONCE_TIMEOUT: Duration = Duration::from_secs(10); + /// Active PoW challenge issued to one connection. pub struct Challenge { token: [u8; 32], @@ -32,7 +38,9 @@ impl Challenge { } /// Builds a challenge with an explicit token. + /// /// Intended for deterministic tests, where a random token makes the outcome + /// hard to predict. #[must_use] pub fn with_token(token: [u8; 32], difficulty: u8) -> Self { Self { @@ -104,8 +112,14 @@ pub fn solve(challenge: &str, difficulty: u8) -> Option { /// Runs the textual PoW gate on `stream`: issues a challenge and checks the /// answer. /// +/// Wire format: the server writes the hex challenge followed by `\n`; the +/// client answers with the nonce followed by `\n`. A reply terminated by EOF is +/// also accepted for compatibility with clients that close their write half +/// after sending the nonce. Anything the client sent after the delimiter stays +/// in the socket buffer and is seen by the protocol handshake that runs next. +/// /// # Errors -/// Only I/O failures are reported; a wrong solution is `Ok(false)`. +/// Only I/O failures are reported; a wrong or missing solution is `Ok(false)`. pub async fn enforce(stream: &mut TcpStream, peer_ip: IpAddr, difficulty: u8) -> anyhow::Result { if difficulty == 0 { tracing::debug!("pow: difficulty 0, skipping for {peer_ip}"); @@ -114,22 +128,21 @@ pub async fn enforce(stream: &mut TcpStream, peer_ip: IpAddr, difficulty: u8) -> let mut challenge = Challenge::generate(difficulty); let line = format!("{}\n", challenge.challenge_string()); - stream.write_all(line.as_bytes()).await?; + timeout(NONCE_TIMEOUT, stream.write_all(line.as_bytes())).await??; - let mut buf = [0u8; MAX_NONCE_LEN + 1]; - let n = tokio::time::timeout(Duration::from_secs(10), stream.read(&mut buf)).await??; - if n == 0 { - tracing::debug!("pow: no response from {peer_ip}"); + let nonce = match timeout(NONCE_TIMEOUT, read_nonce_line(stream)).await { + Ok(result) => result?, + Err(_) => { + tracing::debug!("pow: nonce read timed out for {peer_ip}"); + return Ok(false); + }, + }; + let Some(nonce) = nonce else { + tracing::debug!("pow: invalid or missing nonce from {peer_ip}"); return Ok(false); - } + }; - let nonce = std::str::from_utf8(&buf[..n.min(MAX_NONCE_LEN)]).unwrap_or("").trim(); - if nonce.is_empty() || nonce.len() > MAX_NONCE_LEN { - tracing::debug!("pow: invalid nonce from {peer_ip}"); - return Ok(false); - } - - let valid = challenge.verify(nonce); + let valid = challenge.verify(&nonce); tracing::debug!( "pow: verification {} for {peer_ip}", if valid { "passed" } else { "failed" } @@ -137,54 +150,43 @@ pub async fn enforce(stream: &mut TcpStream, peer_ip: IpAddr, difficulty: u8) -> Ok(valid) } -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn solve_then_verify_roundtrip() { - let mut challenge = Challenge::generate(2); - let nonce = solve(&challenge.challenge_string(), 2).expect("solver should find a nonce"); - assert!(challenge.verify(&nonce), "verifier must accept solver output"); - } - - #[test] - fn verify_rejects_wrong_nonce() { - let mut challenge = Challenge::generate(4); - let challenge_str = challenge.challenge_string(); - - // Search for a nonce that an independently computed SHA-256 provably rejects, - // so the test stays deterministic. - let mut wrong_nonce = None; - for i in 0..10_000u32 { - let candidate = format!("bad-{i}"); - let hex_hash = hex::encode(Sha256::digest(format!("{challenge_str}{candidate}").as_bytes())); - if !hex_hash - .as_bytes() - .iter() - .take(4) - .all(|c| ALLOWED_HEX_PREFIX.contains(c)) - { - wrong_nonce = Some(candidate); - break; - } +/// Reads a newline-terminated nonce without consuming the bytes that follow the +/// delimiter. +/// +/// `peek` inspects the buffer first; only the nonce and its `\n` are consumed, so +/// a client that coalesces the nonce and the handshake into one TCP segment is +/// not truncated. A nonce split across segments is accumulated until the +/// delimiter arrives or the connection reports EOF. +/// +/// Returns `None` for an empty, oversized or non-UTF-8 reply. +async fn read_nonce_line(stream: &mut TcpStream) -> anyhow::Result> { + let mut buf = [0u8; MAX_NONCE_LEN + 1]; + let mut nonce = Vec::with_capacity(MAX_NONCE_LEN); + loop { + let n = stream.peek(&mut buf).await?; + if n == 0 { + // EOF without a delimiter: accept the bytes received so far. + break; } - let wrong_nonce = wrong_nonce.expect("a failing nonce must exist among candidates"); - assert!(!challenge.verify(&wrong_nonce)); + if let Some(pos) = buf[..n].iter().position(|&b| b == b'\n') { + if nonce.len() + pos > MAX_NONCE_LEN { + return Ok(None); + } + nonce.extend_from_slice(&buf[..pos]); + // Consume exactly the nonce line; the handshake bytes stay buffered. + stream.read_exact(&mut buf[..=pos]).await?; + break; + } + if nonce.len() + n > MAX_NONCE_LEN { + return Ok(None); + } + stream.read_exact(&mut buf[..n]).await?; + nonce.extend_from_slice(&buf[..n]); } - #[test] - fn verify_rejects_replay() { - let mut challenge = Challenge::generate(2); - let nonce = solve(&challenge.challenge_string(), 2).expect("solved"); - assert!(challenge.verify(&nonce)); - assert!(!challenge.verify(&nonce), "challenge must be single-use"); - } - - #[test] - fn verify_rejects_oversized_nonce() { - let mut challenge = Challenge::generate(2); - let big_nonce = "0".repeat(MAX_NONCE_LEN + 1); - assert!(!challenge.verify(&big_nonce)); + let text = std::str::from_utf8(&nonce).unwrap_or("").trim(); + if text.is_empty() || text.len() > MAX_NONCE_LEN { + return Ok(None); } + Ok(Some(text.to_string())) } diff --git a/src/engine/listener.rs b/src/engine/listener.rs index 5e1fc2d..7bc85e2 100644 --- a/src/engine/listener.rs +++ b/src/engine/listener.rs @@ -4,10 +4,107 @@ use crate::config::Config; use crate::engine::tunnel::Gateway; use crate::traffic::hook::TrafficHook; use socket2::{Domain, Socket, Type}; +use std::io::ErrorKind; use std::sync::Arc; +use std::time::Duration; use tokio::net::TcpListener; use tokio::sync::watch; +const ACCEPT_BACKOFF_MIN: Duration = Duration::from_millis(50); +const ACCEPT_BACKOFF_MAX: Duration = Duration::from_millis(500); +/// Consecutive accept failures between full-rate ERROR logs. +/// +/// Every intervening failure is still reported at `debug`, so the log stays +/// diagnosable without amplifying a spin into a disk-filling storm. +const ACCEPT_ERROR_LOG_EVERY: u32 = 32; + +/// Linux errno values that mean "the listener is temporarily unable to accept". +/// +/// `libc` is an optional dependency of the `xdp` feature, so the values are +/// spelled out here; they are ABI constants on the platforms Rampart targets. +#[cfg(target_os = "linux")] +mod errno { + pub const ENOMEM: i32 = 12; + pub const ENFILE: i32 = 23; + pub const EMFILE: i32 = 24; + pub const ENOBUFS: i32 = 105; +} + +/// Exponential backoff for transient `accept()` failures. +/// +/// File-descriptor exhaustion (`EMFILE`/`ENFILE`) makes `accept()` fail +/// synchronously and forever: a plain `continue` turns every worker into a +/// 100% CPU spin that also floods the log. Each transient failure therefore +/// yields a mandatory sleep, growing from `ACCEPT_BACKOFF_MIN` to +/// `ACCEPT_BACKOFF_MAX` and resetting after the next successful accept. +pub struct AcceptBackoff { + delay: Duration, + failures: u32, +} + +impl Default for AcceptBackoff { + fn default() -> Self { + Self::new() + } +} + +impl AcceptBackoff { + #[must_use] + pub fn new() -> Self { + Self { + delay: Duration::ZERO, + failures: 0, + } + } + + /// Returns how long the accept loop must sleep before retrying `error`. + /// + /// `None` means the error is not a capacity or readiness condition + /// (`ECONNABORTED`, `EPROTO`, ...): the connection was abandoned by the peer + /// or the kernel, so an immediate retry is correct. + pub fn delay(&mut self, error: &std::io::Error) -> Option { + if !is_transient(error) { + return None; + } + self.failures = self.failures.saturating_add(1); + self.delay = if self.delay.is_zero() { + ACCEPT_BACKOFF_MIN + } else { + (self.delay * 2).min(ACCEPT_BACKOFF_MAX) + }; + Some(self.delay) + } + + /// Clears the accumulated backoff after a successful accept. + pub fn reset(&mut self) { + self.delay = Duration::ZERO; + self.failures = 0; + } + + /// Reports whether the current failure deserves a full-rate ERROR log. + #[must_use] + pub fn should_log(&self) -> bool { + self.failures == 1 || self.failures % ACCEPT_ERROR_LOG_EVERY == 0 + } +} + +fn is_transient(error: &std::io::Error) -> bool { + if matches!( + error.kind(), + ErrorKind::WouldBlock | ErrorKind::Interrupted | ErrorKind::OutOfMemory + ) { + return true; + } + #[cfg(target_os = "linux")] + if matches!( + error.raw_os_error(), + Some(errno::EMFILE | errno::ENFILE | errno::ENOBUFS | errno::ENOMEM) + ) { + return true; + } + false +} + /// Starts `workers.count` accept loops on a single address. /// /// Each worker binds its own socket with `SO_REUSEPORT` and `SO_REUSEADDR`, so @@ -58,22 +155,45 @@ async fn accept_loop( hook: Arc, mut shutdown: watch::Receiver, ) { + let mut backoff = AcceptBackoff::new(); loop { tokio::select! { biased; - _ = shutdown.changed() => { - if *shutdown.borrow() { - tracing::info!("shutdown signal received, stopping accept loop"); - return; + changed = shutdown.changed() => { + match changed { + Ok(()) if *shutdown.borrow() => { + tracing::info!("shutdown signal received, stopping accept loop"); + return; + }, + // An update that does not raise the flag: keep serving. + Ok(()) => {}, + // The sender is gone. `changed()` would return `Err` immediately + // and forever, so treat it as shutdown instead of spinning. + Err(_) => { + tracing::info!("shutdown channel closed, stopping accept loop"); + return; + }, } } result = listener.accept() => { let (stream, peer_addr) = match result { - Ok(conn) => conn, + Ok(conn) => { + backoff.reset(); + conn + }, Err(e) => { - tracing::error!("accept error: {e}"); + if let Some(delay) = backoff.delay(&e) { + if backoff.should_log() { + tracing::error!("accept error, backing off {delay:?}: {e}"); + } else { + tracing::debug!("accept error, backing off {delay:?}: {e}"); + } + tokio::time::sleep(delay).await; + } else { + tracing::debug!("accept error: {e}"); + } continue; - } + }, }; let gateway = Arc::clone(&gateway); let hook = Arc::clone(&hook); diff --git a/src/engine/subnet/monitor.rs b/src/engine/subnet/monitor.rs index e2a0ec2..00fd93e 100644 --- a/src/engine/subnet/monitor.rs +++ b/src/engine/subnet/monitor.rs @@ -6,6 +6,7 @@ use crate::config::DetectPrefixConfig; use crate::engine::subnet_tracker::{SubnetTracker, TrackerSource}; +use crate::filter::blacklist::Blacklist; use crate::metrics; #[cfg(feature = "xdp")] use crate::traffic::prefix::XdpPrefixStats; @@ -30,6 +31,22 @@ impl Source { Source::Tracker(tracker) => tracker.snapshot().await, } } + + /// Whether the source reports distinct source addresses per prefix. + fn counts_unique_sources(&self) -> bool { + match self { + #[cfg(feature = "xdp")] + Source::Xdp(_) => false, + Source::Tracker(_) => true, + } + } +} + +/// Verdict enforcement settings shared by every snapshot tick. +struct BanPolicy { + blacklist: Arc, + ttl: Duration, + trusted: bool, } /// Spawns the background subnet monitor when `[detect.prefix] enabled`. @@ -37,8 +54,16 @@ impl Source { /// Source priority: the XDP map when a filter is loaded, otherwise the userspace /// aggregator. Without either source the monitor stays disabled and logs a /// warning. +/// +/// `ban_ttl` comes from `[ban].ban_duration_secs`. `trusted_peer` is false when +/// the observed address comes from an unverified ingress path (inbound PROXY +/// protocol), where banning it could punish the transport proxy instead of the +/// real client. pub fn spawn( detect: &DetectPrefixConfig, + ban_ttl: Duration, + blacklist: Arc, + trusted_peer: bool, tracker: Option>, xdp: Option>>, mut shutdown: watch::Receiver, @@ -52,6 +77,12 @@ pub fn spawn( }; let detector = SubnetDetector::from_config(detect); let window = Duration::from_secs(detect.window_secs.max(1)); + let policy = BanPolicy { + blacklist, + ttl: ban_ttl, + trusted: trusted_peer, + }; + let extreme_threshold = detect.syn_threshold.saturating_mul(4); tokio::spawn(async move { let mut tick = tokio::time::interval(window); @@ -66,7 +97,10 @@ pub fn spawn( } _ = tick.tick() => { match source.snapshot().await { - Ok(snaps) => apply_verdicts(&detector.evaluate(&snaps), &source).await, + Ok(snaps) => { + let verdicts = detector.evaluate(&snaps); + apply_verdicts(&verdicts, &source, &policy, extreme_threshold); + }, Err(e) => tracing::debug!("prefix stats unavailable: {e}"), } } @@ -85,7 +119,7 @@ fn build_source(tracker: Option>, xdp: Option= extreme_threshold + { + // XDP does not count distinct sources, so the spoof gate can never + // confirm `Block` on that path; make the cap visible instead of silent. + tracing::info!( + prefix = %pv.prefix, + syn = pv.snapshot.syn_count, + "spoof gate suppressed block: XDP has no source diversity data; escalating at L7 only" + ); + } + } + match source { + #[cfg(feature = "xdp")] + Source::Xdp(xdp) if policy.trusted => ban_xdp_verdicts(xdp, verdicts, policy.ttl), + #[cfg(feature = "xdp")] + Source::Xdp(_) => { + if verdicts.iter().any(|pv| pv.verdict == SubnetVerdict::Block) { + tracing::warn!("subnet kernel bans suppressed: peer identity comes from an untrusted ingress path"); + } + }, + Source::Tracker(_) => { + enforce_tracker_verdicts(verdicts, &policy.blacklist, policy.ttl, policy.trusted); + }, + } +} + +/// Applies userspace `Block` verdicts to the shared blacklist. +/// +/// Returns the number of banned prefixes. When `trusted` is false the observed +/// addresses belong to an unverified ingress path, so the verdict is only logged +/// at warn level and no ban is installed. +/// +/// The ban is inserted as a CIDR range, so every address inside the detected +/// `/24` or `/64` is blocked, not just its network address. +pub fn enforce_tracker_verdicts( + verdicts: &[PrefixVerdict], + blacklist: &Blacklist, + ttl: Duration, + trusted: bool, +) -> usize { + let mut banned = 0; + for pv in verdicts.iter().filter(|pv| pv.verdict == SubnetVerdict::Block) { + if !trusted { + tracing::warn!( + prefix = %pv.prefix, + "subnet block verdict on an untrusted peer path; not adding to the blacklist" + ); + continue; + } + blacklist.add_cidr(pv.prefix, prefix_len(pv.prefix), ttl, "subnet_attack"); + banned += 1; + tracing::warn!( + prefix = %pv.prefix, + ttl_secs = ttl.as_secs(), + "subnet blocked in the userspace blacklist" + ); + } + banned +} + +#[cfg(feature = "xdp")] +fn ban_xdp_verdicts(xdp: &XdpPrefixStats, verdicts: &[PrefixVerdict], ttl: Duration) { + for pv in verdicts.iter().filter(|pv| pv.verdict == SubnetVerdict::Block) { + let result = { + let guard = xdp.filter_lock(); + guard.ban_cidr(pv.prefix, prefix_len(pv.prefix), ttl.as_secs()) + }; + match result { + Ok(()) => tracing::warn!( + prefix = %pv.prefix, + ttl_secs = ttl.as_secs(), + "subnet blocked in XDP" + ), + Err(e) => tracing::warn!("subnet block failed for {}: {e}", pv.prefix), } } } -async fn enforce_block(source: &Source, pv: &PrefixVerdict) { - match source { - #[cfg(feature = "xdp")] - Source::Xdp(xdp) => { - let result = { - let guard = xdp.filter_lock(); - guard.ban_cidr(pv.prefix, prefix_len(pv.prefix), 3600) - }; - if let Err(e) = result { - tracing::warn!("subnet block failed for {}: {e}", pv.prefix); - } - }, - Source::Tracker(_) => { - tracing::warn!( - prefix = %pv.prefix, - "block verdict without kernel enforcement; escalate at L7" - ); - }, - } -} - /// Prefix length matching the detection granularity (v4 /24, v6 /64). -#[cfg(feature = "xdp")] fn prefix_len(prefix: IpAddr) -> u8 { match prefix { IpAddr::V4(_) => 24, diff --git a/src/engine/subnet/tracker.rs b/src/engine/subnet/tracker.rs index 817fd76..6f3d3b2 100644 --- a/src/engine/subnet/tracker.rs +++ b/src/engine/subnet/tracker.rs @@ -4,25 +4,49 @@ //! Fallback path of the subnet detector when XDP is disabled: it feeds //! [`crate::traffic::prefix::SubnetDetector`] with the same snapshot shape the //! kernel map produces. +//! +//! Memory is bounded on purpose. Both the prefix table and the per-prefix +//! source set have hard caps, so a source rotating through a /48 (or a /64 with +//! millions of addresses) cannot grow the tracker without limit during one +//! detection window; over-cap events are counted and reported. use crate::traffic::prefix::{PrefixSnapshot, PrefixStatsSource, prefix_of}; -use dashmap::DashMap; +use std::collections::{HashMap, HashSet}; use std::net::IpAddr; -use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; use std::time::Instant; +/// Upper bound on tracked prefixes per window. +pub const DEFAULT_MAX_PREFIXES: usize = 4096; + +/// Upper bound on distinct sources remembered per prefix. +/// +/// 256 is well above the default `min_unique_sources = 16` spoof gate while +/// keeping the worst-case table at a few megabytes. +pub const DEFAULT_MAX_SOURCES_PER_PREFIX: usize = 256; + struct WindowState { syn_count: u64, - sources: std::collections::HashSet, + sources: HashSet, started: Instant, } +#[derive(Debug, Clone, Copy)] +struct Limits { + max_prefixes: usize, + max_sources: usize, +} + /// Aggregates new connections per prefix. /// /// `take_snapshot` drains the accumulated state, so every call closes exactly /// one detection window. pub struct SubnetTracker { - windows: DashMap, + windows: Mutex>, + limits: Limits, + dropped_prefixes: AtomicU64, + dropped_sources: AtomicU64, } impl Default for SubnetTracker { @@ -34,47 +58,108 @@ impl Default for SubnetTracker { impl SubnetTracker { #[must_use] pub fn new() -> Self { + Self::with_limits(DEFAULT_MAX_PREFIXES, DEFAULT_MAX_SOURCES_PER_PREFIX) + } + + /// Creates a tracker with explicit caps. + /// + /// Both limits are clamped to at least one entry. The constructor mainly + /// lets tests exercise the caps without building thousands of prefixes. + #[must_use] + pub fn with_limits(max_prefixes: usize, max_sources_per_prefix: usize) -> Self { Self { - windows: DashMap::new(), + windows: Mutex::new(HashMap::new()), + limits: Limits { + max_prefixes: max_prefixes.max(1), + max_sources: max_sources_per_prefix.max(1), + }, + dropped_prefixes: AtomicU64::new(0), + dropped_sources: AtomicU64::new(0), } } /// Records a new inbound connection (userspace equivalent of a SYN). + /// + /// Call it only after the source passed the blacklist check, so banned + /// sources cannot allocate tracker memory. pub fn record_connection(&self, ip: IpAddr) { let prefix = prefix_of(ip); - let mut state = self.windows.entry(prefix).or_insert_with(|| WindowState { - syn_count: 0, - sources: std::collections::HashSet::new(), - started: Instant::now(), - }); - state.syn_count = state.syn_count.saturating_add(1); - state.sources.insert(ip); + let mut windows = self.windows.lock().expect("subnet tracker lock poisoned"); + if let Some(state) = windows.get_mut(&prefix) { + state.syn_count = state.syn_count.saturating_add(1); + Self::remember_source(state, ip, self.limits.max_sources, &self.dropped_sources); + return; + } + if windows.len() >= self.limits.max_prefixes { + self.dropped_prefixes.fetch_add(1, Ordering::Relaxed); + return; + } + let mut sources = HashSet::new(); + sources.insert(ip); + windows.insert( + prefix, + WindowState { + syn_count: 1, + sources, + started: Instant::now(), + }, + ); } /// Takes and resets the aggregates of the current window. #[must_use] pub fn take_snapshot(&self) -> Vec<(IpAddr, PrefixSnapshot)> { - let mut out = Vec::new(); - // Drain every key: the next window starts from an empty map, so memory does not - // grow when an attacker spoofs many distinct prefixes. - self.windows.retain(|prefix, state| { - if state.syn_count == 0 { - return false; - } - out.push(( - *prefix, - PrefixSnapshot { - // Userspace only sees established connections, so `pkt_count` mirrors the same - // flow as `syn_count`. - pkt_count: state.syn_count, - syn_count: state.syn_count, - last_seen_ns: state.started.elapsed().as_nanos() as u64, - unique_sources: state.sources.len() as u64, - }, - )); - false - }); - out + let windows = std::mem::take(&mut *self.windows.lock().expect("subnet tracker lock poisoned")); + let dropped_prefixes = self.dropped_prefixes.load(Ordering::Relaxed); + let dropped_sources = self.dropped_sources.load(Ordering::Relaxed); + if dropped_prefixes > 0 || dropped_sources > 0 { + tracing::debug!( + dropped_prefixes, + dropped_sources, + "subnet tracker caps reached; aggregates are bounded" + ); + } + windows + .into_iter() + .filter(|(_, state)| state.syn_count > 0) + .map(|(prefix, state)| { + ( + prefix, + PrefixSnapshot { + // Userspace only sees established connections, so `pkt_count` mirrors the + // same flow as `syn_count`. + pkt_count: state.syn_count, + syn_count: state.syn_count, + last_seen_ns: state.started.elapsed().as_nanos() as u64, + unique_sources: state.sources.len() as u64, + }, + ) + }) + .collect() + } + + /// Number of connections from new prefixes dropped because the prefix cap + /// was reached. + #[must_use] + pub fn dropped_prefixes(&self) -> u64 { + self.dropped_prefixes.load(Ordering::Relaxed) + } + + /// Number of connections from new source addresses dropped because the + /// per-prefix source cap was reached. + #[must_use] + pub fn dropped_sources(&self) -> u64 { + self.dropped_sources.load(Ordering::Relaxed) + } + + fn remember_source(state: &mut WindowState, ip: IpAddr, cap: usize, dropped: &AtomicU64) { + if state.sources.len() < cap { + state.sources.insert(ip); + return; + } + if !state.sources.contains(&ip) { + dropped.fetch_add(1, Ordering::Relaxed); + } } } @@ -88,48 +173,3 @@ impl PrefixStatsSource for TrackerSource { Ok(self.tracker.take_snapshot()) } } - -#[cfg(test)] -mod tests { - use super::*; - use std::net::{Ipv4Addr, Ipv6Addr}; - - fn v4(octets: [u8; 4]) -> IpAddr { - IpAddr::V4(Ipv4Addr::from(octets)) - } - - #[test] - fn groups_connections_by_prefix() { - let tracker = SubnetTracker::new(); - tracker.record_connection(v4([203, 0, 113, 1])); - tracker.record_connection(v4([203, 0, 113, 2])); - tracker.record_connection(v4([198, 51, 100, 9])); - let snaps = tracker.take_snapshot(); - assert_eq!(snaps.len(), 2); - let by_prefix: std::collections::HashMap = snaps.into_iter().collect(); - assert_eq!(by_prefix[&v4([203, 0, 113, 0])].syn_count, 2); - assert_eq!(by_prefix[&v4([203, 0, 113, 0])].unique_sources, 2); - assert_eq!(by_prefix[&v4([198, 51, 100, 0])].syn_count, 1); - } - - #[test] - fn snapshot_drains_window() { - let tracker = SubnetTracker::new(); - tracker.record_connection(v4([10, 0, 0, 1])); - assert_eq!(tracker.take_snapshot().len(), 1); - assert!(tracker.take_snapshot().is_empty()); - } - - #[test] - fn mapped_addresses_share_v4_prefix() { - let tracker = SubnetTracker::new(); - tracker.record_connection(v4([203, 0, 113, 5])); - // ::ffff:203.0.113.6 - let mapped = Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xcb00, 0x7106); - tracker.record_connection(IpAddr::V6(mapped)); - let snaps = tracker.take_snapshot(); - assert_eq!(snaps.len(), 1); - assert_eq!(snaps[0].0, v4([203, 0, 113, 0])); - assert_eq!(snaps[0].1.unique_sources, 2); - } -} diff --git a/src/engine/tunnel/gateway.rs b/src/engine/tunnel/gateway.rs index 670392f..ebf6d34 100644 --- a/src/engine/tunnel/gateway.rs +++ b/src/engine/tunnel/gateway.rs @@ -1,7 +1,16 @@ //! Per-connection pipeline of the edge engine: filters, PoW gate, protocol //! handshake and upstream relay. +//! +//! The module root owns gateway construction and the entry point; the stages +//! live in `pipeline` (filters, PoW, handshake, upstream), `admission` +//! (reputation escalation, bans and events) and `relay` (idle-bounded relay). +mod admission; mod challenge; +mod pipeline; +mod relay; + +pub use relay::RELAY_IDLE_TIMEOUT; #[cfg(feature = "protocol-minecraft")] use super::proxy; @@ -12,17 +21,14 @@ use crate::engine::subnet::tracker::SubnetTracker; use crate::filter::blacklist::Blacklist; use crate::filter::connection_limit::ConnectionLimiter; use crate::filter::rate_limit::RateLimiter; -use crate::metrics; use crate::protocol::ProtocolRegistry; -use crate::store::clickhouse::{ClickHouseEvent, ClickHouseWriter}; +use crate::store::clickhouse::ClickHouseWriter; use crate::traffic::reputation::IpReputation; use crate::xdp::XdpFilter; use std::collections::HashSet; use std::net::IpAddr; -use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::sync::atomic::AtomicU64; use std::sync::{Arc, Mutex}; -use std::time::Duration; -use tokio::io::AsyncWriteExt; use tokio::net::TcpStream; use tokio::sync::Mutex as TokioMutex; @@ -55,7 +61,7 @@ impl Gateway { allowed_1s, registry, subnet_tracker, - upstream_cursor: AtomicUsize::new(0), + upstream_cursor: std::sync::atomic::AtomicUsize::new(0), } } @@ -69,25 +75,7 @@ impl Gateway { let client_addr = if self.config.proxy_protocol.ingress_enabled { #[cfg(feature = "protocol-minecraft")] { - if !self - .config - .proxy_protocol - .trusted_cidrs - .iter() - .any(|cidr| cidr.contains(peer_addr.ip())) - { - return GatewayOutcome { - client_ip: None, - result: Err(anyhow::anyhow!("untrusted peer attempted inbound PROXY protocol")), - }; - } - match proxy::read_v2_header( - &mut client, - Duration::from_millis(self.config.proxy_protocol.ingress_timeout_ms), - self.config.proxy_protocol.max_header_bytes, - ) - .await - { + match proxy::resolve_client_addr(&self.config.proxy_protocol, &mut client, peer_addr).await { Ok(client_addr) => client_addr, Err(error) => { return GatewayOutcome { @@ -114,122 +102,4 @@ impl Gateway { result, } } - - async fn handle_client( - &self, - mut client: TcpStream, - peer_addr: std::net::SocketAddr, - client_addr: std::net::SocketAddr, - ) -> anyhow::Result<()> { - let peer_ip = client_addr.ip(); - - if let Some(tracker) = &self.subnet_tracker { - tracker.record_connection(peer_ip); - } - - if self.blacklist.is_blocked(peer_ip) { - metrics::CONNECTIONS_TOTAL.with_label_values(&["blocked"]).inc(); - return Ok(()); - } - - let Some(_connection_permit) = self.connection_limiter.try_acquire(peer_ip) else { - self.block_rate_limit(peer_ip).await; - return Ok(()); - }; - - if !self.pass_pow_gate(&mut client, peer_ip).await? { - return Ok(()); - } - - if !self.rate_limiter.check(peer_ip) { - self.block_rate_limit(peer_ip).await; - return Ok(()); - } - - let handler = self.registry.primary()?; - let handshake = match handler.handshake(&mut client).await { - Ok(handshake) => handshake, - Err(e) => { - tracing::debug!("handshake rejected from {peer_addr}: {e}"); - metrics::CONNECTIONS_TOTAL.with_label_values(&["blocked"]).inc(); - return Ok(()); - }, - }; - - metrics::CONNECTIONS_TOTAL.with_label_values(&["allowed"]).inc(); - self.allowed_1s.fetch_add(1, Ordering::Relaxed); - self.reputation.record_good(peer_ip); - - let mut backend = TcpStream::connect(self.next_upstream(&handshake.upstream.addr)).await?; - #[cfg(feature = "protocol-minecraft")] - if self.config.protocol.minecraft.enabled && self.config.protocol.minecraft.send_proxy_protocol { - let source = client_addr; - let destination = client.local_addr()?; - backend.write_all(&proxy::protocol_v2(source, destination)?).await?; - } - if !handshake.initial_backend_data.is_empty() { - backend.write_all(&handshake.initial_backend_data).await?; - } - tokio::io::copy_bidirectional(&mut client, &mut backend).await?; - Ok(()) - } - - fn next_upstream<'a>(&'a self, negotiated: &'a str) -> &'a str { - let upstreams = &self.config.backend.upstreams; - if upstreams.len() > 1 { - let idx = self.upstream_cursor.fetch_add(1, Ordering::Relaxed) % upstreams.len(); - return upstreams[idx].as_str(); - } - if let Some(first) = upstreams.first() { - return first.as_str(); - } - negotiated - } - - async fn block_rate_limit(&self, ip: IpAddr) { - metrics::RATE_LIMIT_HITS.with_label_values(&["hit"]).inc(); - metrics::CONNECTIONS_TOTAL.with_label_values(&["blocked"]).inc(); - self.reputation.record_bad(ip); - if self.reputation.score(ip) < -40 { - let duration_secs = Duration::from_secs(self.config.ban.ban_duration_secs); - self.blacklist.add(ip, duration_secs, "low_reputation"); - if !self.config.proxy_protocol.ingress_enabled { - self.xdp_ban(ip, duration_secs.as_secs()); - } - self.push_event("block", ip, "low_reputation").await; - tracing::info!("low reputation ban {ip}: rate-limit abuse"); - } - } - - fn xdp_ban(&self, ip: IpAddr, duration_secs: u64) { - let Some(xdp) = &self.xdp else { - return; - }; - let prefix_len = if ip.is_ipv4() { 32 } else { 128 }; - match xdp - .lock() - .expect("xdp lock poisoned") - .ban_cidr(ip, prefix_len, duration_secs) - { - Ok(()) => tracing::debug!("xdp ban {ip} for {duration_secs}s"), - Err(e) => tracing::warn!("xdp ban failed for {ip}: {e}"), - } - } - - async fn push_event(&self, event_type: &str, ip: IpAddr, data_string: &str) { - let Some(writer) = &self.clickhouse else { - return; - }; - let event = ClickHouseEvent { - timestamp: chrono::Utc::now(), - event_type: event_type.to_string(), - ip: ip.to_string(), - data_float: 0.0, - data_int: 0, - data_string: data_string.to_string(), - }; - if let Err(e) = writer.lock().await.push(event).await { - tracing::debug!("clickhouse push error: {e}"); - } - } } diff --git a/src/engine/tunnel/gateway/admission.rs b/src/engine/tunnel/gateway/admission.rs new file mode 100644 index 0000000..92d6227 --- /dev/null +++ b/src/engine/tunnel/gateway/admission.rs @@ -0,0 +1,67 @@ +//! Reputation escalation, bans and telemetry for dropped connections. + +use super::Gateway; +use crate::metrics; +use crate::store::clickhouse::ClickHouseEvent; +use std::net::IpAddr; +use std::time::Duration; + +impl Gateway { + /// Accounts for a connection dropped by an admission filter and escalates to + /// a ban when the source reputation is exhausted. + /// + /// Whitelisted peers are exempt: a ban is replicated to Redis and XDP on + /// every node, which would lock a trusted operator out of the whole cluster. + pub(super) async fn block_rate_limit(&self, ip: IpAddr) { + if self.whitelist.contains(&ip) { + metrics::RATE_LIMIT_HITS.with_label_values(&["trusted"]).inc(); + return; + } + metrics::RATE_LIMIT_HITS.with_label_values(&["hit"]).inc(); + metrics::CONNECTIONS_TOTAL.with_label_values(&["blocked"]).inc(); + self.reputation.record_bad(ip); + if self.reputation.score(ip) < -40 { + let duration_secs = Duration::from_secs(self.config.ban.ban_duration_secs); + self.blacklist.add(ip, duration_secs, "low_reputation"); + if !self.config.proxy_protocol.ingress_enabled { + self.xdp_ban(ip, duration_secs.as_secs()); + } + self.push_event("block", ip, "low_reputation").await; + tracing::info!("low reputation ban {ip}: rate-limit abuse"); + } + } + + /// Mirrors a ban into the XDP map, if the kernel filter is attached. + fn xdp_ban(&self, ip: IpAddr, duration_secs: u64) { + let Some(xdp) = &self.xdp else { + return; + }; + let prefix_len = if ip.is_ipv4() { 32 } else { 128 }; + match xdp + .lock() + .expect("xdp lock poisoned") + .ban_cidr(ip, prefix_len, duration_secs) + { + Ok(()) => tracing::debug!("xdp ban {ip} for {duration_secs}s"), + Err(e) => tracing::warn!("xdp ban failed for {ip}: {e}"), + } + } + + /// Forwards an event to ClickHouse when the writer is configured. + async fn push_event(&self, event_type: &str, ip: IpAddr, data_string: &str) { + let Some(writer) = &self.clickhouse else { + return; + }; + let event = ClickHouseEvent { + timestamp: chrono::Utc::now(), + event_type: event_type.to_string(), + ip: ip.to_string(), + data_float: 0.0, + data_int: 0, + data_string: data_string.to_string(), + }; + if let Err(e) = writer.lock().await.push(event).await { + tracing::debug!("clickhouse push error: {e}"); + } + } +} diff --git a/src/engine/tunnel/gateway/challenge.rs b/src/engine/tunnel/gateway/challenge.rs index cc5b159..c4f88e0 100644 --- a/src/engine/tunnel/gateway/challenge.rs +++ b/src/engine/tunnel/gateway/challenge.rs @@ -31,6 +31,10 @@ impl Gateway { adjuster.record_connection(); adjuster.current_difficulty() }; + // The configured difficulty is a hard floor even if the adjuster was + // constructed without it; recording the connection above still adapts + // the value for later peers. + let diff = diff.max(pow_config.difficulty); let result = enforce_pow(client, peer_ip, diff).await?; if !result { metrics::POW_CHALLENGES_TOTAL.with_label_values(&["failed"]).inc(); diff --git a/src/engine/tunnel/gateway/pipeline.rs b/src/engine/tunnel/gateway/pipeline.rs new file mode 100644 index 0000000..2205556 --- /dev/null +++ b/src/engine/tunnel/gateway/pipeline.rs @@ -0,0 +1,208 @@ +//! Filter order and upstream establishment for one accepted connection. + +use super::Gateway; +use super::RELAY_IDLE_TIMEOUT; +use crate::metrics; +use std::sync::OnceLock; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; +use tokio::io::AsyncWriteExt; +use tokio::net::TcpStream; +use tokio::sync::Semaphore; + +/// Multiplier applied to the per-IP limit when sizing the global connection +/// budget: `4 x workers x max_connections_per_ip` in-flight connections. +const GLOBAL_BUDGET_FACTOR: usize = 4; +/// Ceiling of the global budget, so a large worker count cannot exhaust the +/// file-descriptor table of the host. +const GLOBAL_BUDGET_MAX: usize = 100_000; +/// One in every `BUDGET_LOG_EVERY` budget rejections is logged. +const BUDGET_LOG_EVERY: u64 = 128; + +/// Process-wide in-flight connection budget. +/// +/// Initialised on first use from the worker count and the per-IP limit of the +/// gateway configuration. A process-wide value is intentional: the budget caps +/// descriptors and tasks for the whole edge node, and there is exactly one +/// gateway per process in production. +static GLOBAL_BUDGET: OnceLock = OnceLock::new(); +/// Total connections rejected by the global budget, used for log rate limiting. +static BUDGET_REJECTIONS: AtomicU64 = AtomicU64::new(0); + +impl Gateway { + /// Returns the process-wide in-flight connection budget. + fn global_budget(&self) -> &'static Semaphore { + GLOBAL_BUDGET.get_or_init(|| { + let workers = self.config.workers.count.max(1); + let per_ip = self.config.limits.max_connections_per_ip.max(1) as usize; + let capacity = workers + .saturating_mul(per_ip) + .saturating_mul(GLOBAL_BUDGET_FACTOR) + .clamp(1, GLOBAL_BUDGET_MAX); + Semaphore::new(capacity) + }) + } + + /// Runs the filter stages, PoW gate, protocol handshake and relay. + /// + /// # Errors + /// Only upstream I/O failures are surfaced; every filter drop is `Ok(())`. + pub(super) async fn handle_client( + &self, + mut client: TcpStream, + peer_addr: std::net::SocketAddr, + client_addr: std::net::SocketAddr, + ) -> anyhow::Result<()> { + let peer_ip = client_addr.ip(); + let trusted = self.whitelist.contains(&peer_ip); + + // Trusted peers bypass every ingress filter and are never auto-banned: + // blacklist entries and XDP/Redis bans propagate across all nodes, so a + // whitelisted address must never enter that path. + if !trusted { + if self.blacklist.is_blocked(peer_ip) { + metrics::CONNECTIONS_TOTAL.with_label_values(&["blocked"]).inc(); + return Ok(()); + } + + // Feed the subnet detector only after the ban decision: a source we + // already block must not allocate tracker memory. + if let Some(tracker) = &self.subnet_tracker { + tracker.record_connection(peer_ip); + } + + // The cheap token check runs before the expensive PoW exchange: a + // client that never solves the challenge must still exhaust its own + // rate budget instead of holding a task for the PoW timeout unpaid. + if !self.rate_limiter.check(peer_ip) { + self.block_rate_limit(peer_ip).await; + return Ok(()); + } + } + + // Global budget: `try_acquire` keeps an overloaded node responsive (the + // accept loop never queues unbounded per-connection tasks) and the + // connection is closed immediately on refusal. + let _global_permit = if trusted { + None + } else { + match self.global_budget().try_acquire() { + Ok(permit) => Some(permit), + Err(_) => { + metrics::CONNECTIONS_TOTAL.with_label_values(&["overloaded"]).inc(); + let seen = BUDGET_REJECTIONS.fetch_add(1, Ordering::Relaxed); + if seen % BUDGET_LOG_EVERY == 0 { + tracing::warn!( + "global connection budget exhausted after {seen} rejections, dropping {peer_ip}" + ); + } + return Ok(()); + }, + } + }; + + let Some(_connection_permit) = self.connection_limiter.try_acquire(peer_ip) else { + if trusted { + tracing::debug!("per-IP connection limit reached for trusted {peer_ip}"); + } else { + self.block_rate_limit(peer_ip).await; + } + return Ok(()); + }; + + if !self.pass_pow_gate(&mut client, peer_ip).await? { + return Ok(()); + } + + let handshake_budget = Duration::from_secs(self.config.limits.handshake_timeout_secs.max(1)); + let handler = self.registry.primary()?; + let handshake = match tokio::time::timeout(handshake_budget, handler.handshake(&mut client)).await { + Ok(Ok(handshake)) => handshake, + Ok(Err(e)) => { + tracing::debug!("handshake rejected from {peer_addr}: {e}"); + metrics::CONNECTIONS_TOTAL.with_label_values(&["blocked"]).inc(); + return Ok(()); + }, + Err(_) => { + tracing::debug!("handshake timed out after {handshake_budget:?} from {peer_addr}"); + metrics::CONNECTIONS_TOTAL.with_label_values(&["blocked"]).inc(); + return Ok(()); + }, + }; + + metrics::CONNECTIONS_TOTAL.with_label_values(&["allowed"]).inc(); + self.allowed_1s.fetch_add(1, Ordering::Relaxed); + self.reputation.record_good(peer_ip); + + let mut backend = self + .connect_upstream(&handshake.upstream.addr, handshake_budget) + .await?; + self.send_preamble( + &mut client, + &mut backend, + client_addr, + &handshake.initial_backend_data, + handshake_budget, + ) + .await?; + self.relay_with_idle(client, backend, RELAY_IDLE_TIMEOUT).await + } + + /// Opens the upstream connection under the handshake timeout. + /// + /// # Errors + /// Fails when the connect is refused or exceeds `budget`. + async fn connect_upstream(&self, negotiated: &str, budget: Duration) -> anyhow::Result { + let upstream = self.next_upstream(negotiated); + match tokio::time::timeout(budget, TcpStream::connect(upstream)).await { + Ok(Ok(stream)) => Ok(stream), + Ok(Err(e)) => Err(anyhow::anyhow!("upstream connect to {upstream} failed: {e}")), + Err(_) => Err(anyhow::anyhow!( + "upstream connect to {upstream} timed out after {budget:?}" + )), + } + } + + /// Writes the PROXY protocol header and the protocol's replay bytes to the + /// backend under the handshake timeout. + /// + /// # Errors + /// Fails on address resolution and on a write that exceeds `budget`. + #[allow(unused_variables)] + async fn send_preamble( + &self, + client: &mut TcpStream, + backend: &mut TcpStream, + client_addr: std::net::SocketAddr, + initial_data: &[u8], + budget: Duration, + ) -> anyhow::Result<()> { + let mut preamble = Vec::new(); + #[cfg(feature = "protocol-minecraft")] + if self.config.protocol.minecraft.enabled && self.config.protocol.minecraft.send_proxy_protocol { + let destination = client.local_addr()?; + preamble.extend_from_slice(&super::super::proxy::protocol_v2(client_addr, destination)?); + } + preamble.extend_from_slice(initial_data); + if preamble.is_empty() { + return Ok(()); + } + match tokio::time::timeout(budget, backend.write_all(&preamble)).await { + Ok(Ok(())) => Ok(()), + Ok(Err(e)) => Err(e.into()), + Err(_) => Err(anyhow::anyhow!("upstream preamble write timed out after {budget:?}")), + } + } + + fn next_upstream<'a>(&'a self, negotiated: &'a str) -> &'a str { + let upstreams = &self.config.backend.upstreams; + if upstreams.len() > 1 { + let idx = self.upstream_cursor.fetch_add(1, Ordering::Relaxed) % upstreams.len(); + return upstreams[idx].as_str(); + } + if let Some(first) = upstreams.first() { + return first.as_str(); + } + negotiated + } +} diff --git a/src/engine/tunnel/gateway/relay.rs b/src/engine/tunnel/gateway/relay.rs new file mode 100644 index 0000000..9ecd994 --- /dev/null +++ b/src/engine/tunnel/gateway/relay.rs @@ -0,0 +1,75 @@ +//! Idle-aware bidirectional relay between the accepted client and the backend. + +use super::Gateway; +use std::io; +use std::time::Duration; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; +use tokio::net::TcpStream; + +const RELAY_BUFFER_BYTES: usize = 16 * 1024; + +/// Idle budget of the client/backend relay. +/// +/// Minecraft keeps one connection alive for a whole session and a legitimately +/// silent gap (menus, chunk loading, AFK timeouts) can last minutes, so 300s is +/// the safe default: it never cuts a real session and still reaps dead peers. +/// The value is a constant because `[limits]` is owned by a parallel change; it +/// is plumbed as a parameter through [`Gateway::relay_with_idle`] so a config +/// field can be wired in without touching the relay itself. +pub const RELAY_IDLE_TIMEOUT: Duration = Duration::from_secs(300); + +impl Gateway { + /// Relays bytes in both directions until either side closes, an I/O error + /// occurs or a direction stays silent for `idle`. + /// + /// The idle budget restarts on every successful read/write, so a long-lived + /// session with steady traffic is never cut, while a dead or half-open peer + /// is reaped instead of pinning two descriptors forever. Both directions run + /// under one `select!`: when one pump finishes the other is dropped and both + /// sockets close, preserving the cancellation semantics of + /// `copy_bidirectional`. + /// + /// `idle` is a parameter rather than a constant so tests can exercise the + /// timeout with a short budget and per-upstream tuning can override it. + /// + /// # Errors + /// Fails on the first I/O error and on an idle timeout; a clean half-close is + /// `Ok`. + pub async fn relay_with_idle(&self, client: TcpStream, backend: TcpStream, idle: Duration) -> anyhow::Result<()> { + let (client_read, mut client_write) = client.into_split(); + let (backend_read, mut backend_write) = backend.into_split(); + tokio::select! { + result = pump(client_read, &mut backend_write, idle) => result?, + result = pump(backend_read, &mut client_write, idle) => result?, + } + Ok(()) + } +} + +/// Copies one direction, bounded by `idle` between chunks. +/// +/// # Errors +/// Fails on a read/write error and when a direction makes no progress within +/// `idle`. +async fn pump(mut source: R, mut sink: W, idle: Duration) -> io::Result<()> +where + R: AsyncRead + Unpin, + W: AsyncWrite + Unpin, +{ + let mut buffer = vec![0u8; RELAY_BUFFER_BYTES]; + loop { + let read = tokio::time::timeout(idle, source.read(&mut buffer)) + .await + .map_err(|_| idle_error())??; + if read == 0 { + return Ok(()); + } + tokio::time::timeout(idle, sink.write_all(&buffer[..read])) + .await + .map_err(|_| idle_error())??; + } +} + +fn idle_error() -> io::Error { + io::Error::new(io::ErrorKind::TimedOut, "relay idle timeout") +} diff --git a/src/engine/tunnel/proxy.rs b/src/engine/tunnel/proxy.rs index 892f0d6..d681845 100644 --- a/src/engine/tunnel/proxy.rs +++ b/src/engine/tunnel/proxy.rs @@ -1,10 +1,38 @@ -//! Encoding of upstream PROXY protocol v2 headers. +//! PROXY protocol v2 support: header encoding for the backend and strict +//! parsing of headers injected by trusted ingress peers. +use crate::config::ProxyProtocolConfig; use std::net::SocketAddr; +use std::time::Duration; +use tokio::net::TcpStream; mod inbound; pub(super) use inbound::read_v2_header; +/// Resolves the effective client address from an inbound PROXY v2 header. +/// +/// Only peers inside `config.trusted_cidrs` may inject a header; anyone else +/// is rejected instead of being believed. The whole header must arrive within +/// `config.ingress_timeout_ms` and fit in `config.max_header_bytes`. +/// +/// # Errors +/// Fails when the peer is not trusted, on timeout and on any header parse error. +pub(super) async fn resolve_client_addr( + config: &ProxyProtocolConfig, + client: &mut TcpStream, + peer_addr: SocketAddr, +) -> anyhow::Result { + if !config.trusted_cidrs.iter().any(|cidr| cidr.contains(peer_addr.ip())) { + anyhow::bail!("untrusted peer attempted inbound PROXY protocol"); + } + read_v2_header( + client, + Duration::from_millis(config.ingress_timeout_ms), + config.max_header_bytes, + ) + .await +} + /// Encodes a PROXY protocol v2 TCP header for the backend. /// /// The backend must accept connections only from trusted Rampart nodes, since diff --git a/src/filter/blacklist/mod.rs b/src/filter/blacklist/mod.rs index e18b533..aff0ab7 100644 --- a/src/filter/blacklist/mod.rs +++ b/src/filter/blacklist/mod.rs @@ -1,4 +1,15 @@ +//! In-memory ban tables. +//! +//! Two tables are kept: bans detected locally and bans mirrored from Redis. +//! Expired entries are removed lazily on lookup and periodically by +//! [`Blacklist::clear_expired`]. Deadlines are monotonic and clamped, so a +//! hostile expiry can never overflow or panic. + +mod prefix; + +use self::prefix::PrefixBans; use dashmap::DashMap; +use std::collections::HashSet; use std::net::IpAddr; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -6,9 +17,14 @@ use tokio::sync::broadcast; const LOCAL_BAN_EVENT_CAPACITY: usize = 4096; +/// Upper bound for a single ban; longer requests are clamped. +pub const MAX_BAN_DURATION_SECS: u64 = 31_536_000; + +/// A locally detected ban, forwarded to the Redis publisher. #[derive(Clone, Debug)] pub struct LocalBanEvent { pub ip: IpAddr, + /// Wall-clock expiry as Unix seconds. pub expires_at_unix: i64, pub reason: String, } @@ -18,9 +34,11 @@ struct BanEntry { _reason: String, } +/// Combined view of local and Redis-mirrored bans. pub struct Blacklist { entries: Arc>, redis_entries: Arc>, + prefixes: PrefixBans, local_ban_events: broadcast::Sender, } @@ -31,29 +49,53 @@ impl Default for Blacklist { } impl Blacklist { + /// Creates an empty blacklist. pub fn new() -> Self { let (local_ban_events, _) = broadcast::channel(LOCAL_BAN_EVENT_CAPACITY); Self { entries: Arc::new(DashMap::new()), redis_entries: Arc::new(DashMap::new()), + prefixes: PrefixBans::new(), local_ban_events, } } + /// Returns whether `ip` is banned: exact match or inside a CIDR ban. pub fn is_blocked(&self, ip: IpAddr) -> bool { - Self::is_blocked_in(&self.entries, ip) || Self::is_blocked_in(&self.redis_entries, ip) + let ip = normalize(ip); + Self::is_blocked_in(&self.entries, ip) + || Self::is_blocked_in(&self.redis_entries, ip) + || self.prefixes.is_blocked(ip, Instant::now()) } + /// Adds a CIDR ban covering `ip/prefix_len` and broadcasts its network + /// address, so other nodes mirror at least that host. + /// + /// Invalid prefix lengths (`0`, `> 32` for IPv4, `> 128` for IPv6) are + /// ignored: a `/0` ban would blackhole the whole address family. + pub fn add_cidr(&self, ip: IpAddr, prefix_len: u8, duration: Duration, reason: &str) { + let duration = clamp_duration(duration); + let Some(start) = self.prefixes.add(ip, prefix_len, deadline(duration)) else { + tracing::warn!(%ip, prefix_len, "ignoring CIDR ban with an invalid prefix length"); + return; + }; + let expires_at_unix = now_unix().saturating_add(duration.as_secs()).min(i64::MAX as u64) as i64; + let _ = self.local_ban_events.send(LocalBanEvent { + ip: start, + expires_at_unix, + reason: reason.to_string(), + }); + } + + /// Adds a local ban and broadcasts it for the Redis publisher. pub fn add(&self, ip: IpAddr, duration: Duration, reason: &str) { - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_secs(); - let expires_at_unix = now.saturating_add(duration.as_secs()).min(i64::MAX as u64) as i64; + let ip = normalize(ip); + let duration = clamp_duration(duration); + let expires_at_unix = now_unix().saturating_add(duration.as_secs()).min(i64::MAX as u64) as i64; self.entries.insert( ip, BanEntry { - expires: Instant::now() + duration, + expires: deadline(duration), _reason: reason.to_string(), }, ); @@ -64,120 +106,130 @@ impl Blacklist { }); } + /// Subscribes to locally detected bans. pub fn subscribe_local_bans(&self) -> broadcast::Receiver { self.local_ban_events.subscribe() } + /// Returns all active local bans. pub fn local_snapshot(&self) -> Vec { + let now = Instant::now(); + let now_unix = now_unix(); self.entries .iter() - .filter(|entry| entry.expires > Instant::now()) + .filter(|entry| entry.expires > now) .map(|entry| LocalBanEvent { ip: *entry.key(), - expires_at_unix: std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_secs() - .saturating_add(entry.expires.saturating_duration_since(Instant::now()).as_secs()) + expires_at_unix: now_unix + .saturating_add(entry.expires.saturating_duration_since(now).as_secs()) .min(i64::MAX as u64) as i64, reason: entry._reason.clone(), }) .collect() } + /// Removes a local ban. pub fn remove(&self, ip: IpAddr) { - self.entries.remove(&ip); + self.entries.remove(&normalize(ip)); } + /// Adds a Redis-mirrored ban. pub fn add_redis(&self, ip: IpAddr, duration: Duration, reason: &str) { + let ip = normalize(ip); + let duration = clamp_duration(duration); self.redis_entries.insert( ip, BanEntry { - expires: Instant::now() + duration, + expires: deadline(duration), _reason: reason.to_string(), }, ); } + /// Removes a Redis-mirrored ban. pub fn remove_redis(&self, ip: IpAddr) { - self.redis_entries.remove(&ip); + self.redis_entries.remove(&normalize(ip)); } /// Replaces the Redis-managed ban set, preserving locally detected bans. /// - /// `redis_entries` is rebuilt from `entries` wholesale, so a ban that - /// disappeared upstream stops applying after this call. + /// New entries are inserted before stale ones are removed, so a reconnect + /// never exposes a window without bans. A ban that disappeared upstream + /// stops applying after this call. pub fn replace_redis(&self, entries: I) where I: IntoIterator, { - self.redis_entries.clear(); + let mut fresh = HashSet::new(); for (ip, duration, reason) in entries { + fresh.insert(ip); self.add_redis(ip, duration, &reason); } + self.redis_entries.retain(|ip, _| fresh.contains(ip)); } + /// Total number of tracked bans. pub fn len(&self) -> usize { - self.entries.len() + self.redis_entries.len() + self.entries.len() + self.redis_entries.len() + self.prefixes.len() } + /// Returns whether all ban tables are empty. pub fn is_empty(&self) -> bool { - self.entries.is_empty() && self.redis_entries.is_empty() + self.len() == 0 } + /// Drops expired entries from all tables. pub fn clear_expired(&self) { - self.entries.retain(|_, entry| entry.expires > Instant::now()); - self.redis_entries.retain(|_, entry| entry.expires > Instant::now()); + let now = Instant::now(); + self.entries.retain(|_, entry| entry.expires > now); + self.redis_entries.retain(|_, entry| entry.expires > now); + self.prefixes.clear_expired(now); } + /// Checks `entries`, lazily removing `ip` only if it is still expired. + /// + /// The conditional removal avoids a TOCTOU race in which a concurrent + /// re-ban would be deleted right after the expired entry was observed. fn is_blocked_in(entries: &DashMap, ip: IpAddr) -> bool { - if let Some(entry) = entries.get(&ip) { - if entry.expires > Instant::now() { - return true; - } - drop(entry); - entries.remove(&ip); + let now = Instant::now(); + if let Some(entry) = entries.get(&ip) + && entry.expires > now + { + return true; } + entries.remove_if(&ip, |_, entry| entry.expires <= now); false } } -#[cfg(test)] -mod tests { - use super::*; - use std::net::{IpAddr, Ipv4Addr}; - - fn ip(octets: [u8; 4]) -> IpAddr { - IpAddr::V4(Ipv4Addr::from(octets)) - } - - #[test] - fn test_blacklist_block() { - let bl = Blacklist::new(); - bl.add(ip([1, 2, 3, 4]), Duration::from_secs(60), "test"); - assert!(bl.is_blocked(ip([1, 2, 3, 4]))); - } - - #[test] - fn test_blacklist_not_blocked() { - let bl = Blacklist::new(); - bl.add(ip([1, 2, 3, 4]), Duration::from_secs(60), "test"); - assert!(!bl.is_blocked(ip([5, 6, 7, 8]))); - } - - #[test] - fn test_blacklist_expired() { - let bl = Blacklist::new(); - bl.add(ip([1, 2, 3, 4]), Duration::from_millis(1), "test"); - std::thread::sleep(Duration::from_millis(2)); - assert!(!bl.is_blocked(ip([1, 2, 3, 4]))); - } - - #[test] - fn test_blacklist_remove() { - let bl = Blacklist::new(); - bl.add(ip([1, 2, 3, 4]), Duration::from_secs(60), "test"); - bl.remove(ip([1, 2, 3, 4])); - assert!(!bl.is_blocked(ip([1, 2, 3, 4]))); +/// Collapses IPv4-mapped IPv6 addresses to IPv4, so one client never occupies +/// two different keys in the ban tables. +fn normalize(ip: IpAddr) -> IpAddr { + match ip { + IpAddr::V6(v6) => v6.to_ipv4_mapped().map_or(ip, IpAddr::V4), + IpAddr::V4(_) => ip, } } + +/// Clamps a requested duration to [`MAX_BAN_DURATION_SECS`]. +fn clamp_duration(duration: Duration) -> Duration { + duration.min(Duration::from_secs(MAX_BAN_DURATION_SECS)) +} + +/// Returns the monotonic deadline for an already clamped duration. +/// +/// Overflow cannot occur for clamped values, but is handled defensively so a +/// malformed duration can never panic the data plane. +fn deadline(duration: Duration) -> Instant { + Instant::now() + .checked_add(duration) + .unwrap_or_else(|| Instant::now() + Duration::from_secs(MAX_BAN_DURATION_SECS)) +} + +/// Current wall-clock time as Unix seconds. +fn now_unix() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} diff --git a/src/filter/blacklist/prefix.rs b/src/filter/blacklist/prefix.rs new file mode 100644 index 0000000..a465ac1 --- /dev/null +++ b/src/filter/blacklist/prefix.rs @@ -0,0 +1,119 @@ +//! CIDR ban table. +//! +//! Exact-address bans live in two `DashMap`s in [`super`]; substrate-wide bans +//! (`/24`, `/64`, ...) are stored here as inclusive address ranges. The table is +//! a plain `RwLock>`: entries are few (the subnet monitor bans at most a +//! handful per tick), so a linear scan is cheaper than a trie, and the hard cap +//! bounds the worst case under abuse. + +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; +use std::sync::RwLock; +use std::time::Instant; + +/// Upper bound on distinct CIDR bans kept in memory. +const MAX_PREFIX_BANS: usize = 256; + +/// One CIDR ban, stored as an inclusive address range. +struct PrefixBan { + start: IpAddr, + end: IpAddr, + expires: Instant, +} + +/// Thread-safe set of active CIDR bans. +pub(super) struct PrefixBans { + entries: RwLock>, +} + +impl PrefixBans { + /// Creates an empty table. + pub(super) fn new() -> Self { + Self { + entries: RwLock::new(Vec::new()), + } + } + + /// Inserts `ip/prefix_len` until `expires`. + /// + /// Returns the network address when the ban was inserted, or `None` for an + /// invalid prefix length (`0`, `> 32` for IPv4, `> 128` for IPv6): a `/0` + /// ban would blackhole an entire address family. + pub(super) fn add(&self, ip: IpAddr, prefix_len: u8, expires: Instant) -> Option { + let (start, end) = cidr_range(ip, prefix_len)?; + let now = Instant::now(); + let mut bans = self.entries.write().unwrap_or_else(|poison| poison.into_inner()); + bans.retain(|ban| ban.expires > now); + if bans.len() >= MAX_PREFIX_BANS { + bans.remove(0); + } + bans.push(PrefixBan { start, end, expires }); + Some(start) + } + + /// Returns whether `ip` falls into an active CIDR ban. + pub(super) fn is_blocked(&self, ip: IpAddr, now: Instant) -> bool { + let bans = self.entries.read().unwrap_or_else(|poison| poison.into_inner()); + bans.iter().any(|ban| ban.expires > now && range_contains(ban, ip)) + } + + /// Drops expired CIDR bans. + pub(super) fn clear_expired(&self, now: Instant) { + self.entries + .write() + .unwrap_or_else(|poison| poison.into_inner()) + .retain(|ban| ban.expires > now); + } + + /// Number of tracked CIDR bans (including not-yet-reaped expired ones). + pub(super) fn len(&self) -> usize { + self.entries.read().unwrap_or_else(|poison| poison.into_inner()).len() + } +} + +/// Converts `ip/prefix_len` into an inclusive address range. +/// +/// IPv4-mapped IPv6 addresses are normalised to IPv4 first, matching the prefix +/// keys used by the subnet detector and the XDP prefix stats. +fn cidr_range(ip: IpAddr, prefix_len: u8) -> Option<(IpAddr, IpAddr)> { + let ip = match ip { + IpAddr::V6(v6) => v6.to_ipv4_mapped().map_or(ip, IpAddr::V4), + IpAddr::V4(_) => ip, + }; + match ip { + IpAddr::V4(v4) => { + if prefix_len == 0 || prefix_len > 32 { + return None; + } + let mask = u32::MAX << (32 - prefix_len); + let base = u32::from(v4) & mask; + Some(( + IpAddr::V4(Ipv4Addr::from(base)), + IpAddr::V4(Ipv4Addr::from(base | !mask)), + )) + }, + IpAddr::V6(v6) => { + if prefix_len == 0 || prefix_len > 128 { + return None; + } + let mask = u128::MAX << (128 - prefix_len); + let base = u128::from(v6) & mask; + Some(( + IpAddr::V6(Ipv6Addr::from(base)), + IpAddr::V6(Ipv6Addr::from(base | !mask)), + )) + }, + } +} + +/// Returns whether `ip` is inside `ban`'s inclusive range. +fn range_contains(ban: &PrefixBan, ip: IpAddr) -> bool { + match (ban.start, ban.end, ip) { + (IpAddr::V4(start), IpAddr::V4(end), IpAddr::V4(addr)) => { + u32::from(start) <= u32::from(addr) && u32::from(addr) <= u32::from(end) + }, + (IpAddr::V6(start), IpAddr::V6(end), IpAddr::V6(addr)) => { + u128::from(start) <= u128::from(addr) && u128::from(addr) <= u128::from(end) + }, + _ => false, + } +} diff --git a/src/filter/rate_limit.rs b/src/filter/rate_limit.rs index 76a8305..c857792 100644 --- a/src/filter/rate_limit.rs +++ b/src/filter/rate_limit.rs @@ -1,155 +1,175 @@ -use dashmap::DashMap; +//! Per-source token-bucket rate limiter for the accept path. +//! +//! # Capacity policy +//! +//! The bucket table is a hard-bounded sharded hash map: at most `max_buckets` +//! live buckets (one million for [`RateLimiter::new`]). Admission uses an +//! atomic reservation, so concurrent accept loops cannot overshoot the bound +//! the way a plain `len()` check would. +//! +//! # Saturation policy +//! +//! When the table is full, [`RateLimiter::check`] fails closed for a *new* +//! source: the connection is rejected until [`RateLimiter::sweep`] frees a +//! slot. Existing sources keep their token state and are never refused because +//! of the cap. Failing closed is deliberate for a protection device - the +//! alternative (unbounded buckets, or silently allowing everything once the +//! table is full) turns a memory exhaustion vector into a bypass. +//! +//! # Eviction policy +//! +//! [`RateLimiter::sweep`] visits eight of the sixty-four shards and drops +//! buckets idle for longer than `eviction_idle` (ten minutes by default). Work +//! per call is bounded, so the accept path never stalls on a full-table +//! `retain`, and one full cycle takes eight one-minute ticks. + +use std::collections::HashMap; +use std::collections::hash_map::RandomState; +use std::hash::BuildHasher; use std::net::IpAddr; -use std::sync::Arc; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Mutex, MutexGuard}; use std::time::{Duration, Instant}; +/// Default hard bound of the bucket table (one million sources). const MAX_BUCKETS: usize = 1_000_000; +/// Default idle age after which a bucket may be reclaimed. const EVICTION_IDLE: Duration = Duration::from_secs(600); -const SWEEP_INTERVAL: Duration = Duration::from_secs(60); +/// Number of independently locked map shards. +const SHARDS: usize = 64; +/// Shards examined by one [`RateLimiter::sweep`] call. +const SWEEP_SHARDS_PER_TICK: usize = 8; +/// Token-bucket state of one source address. struct Bucket { tokens: f64, last_refill: Instant, last_access: Instant, } +/// Per-source token buckets with a hard capacity bound. pub struct RateLimiter { - buckets: Arc>, + shards: Box<[Mutex>]>, + hasher: RandomState, + live: AtomicUsize, + max_buckets: usize, max_tokens: f64, refill_rate: f64, - _refill_interval: Duration, - epoch: Instant, - last_sweep_elapsed: AtomicU64, eviction_idle: Duration, + sweep_cursor: AtomicUsize, } impl RateLimiter { + /// Builds a limiter with the production capacity policy. + #[must_use] pub fn new(rate_per_sec: f64, burst: f64) -> Self { - Self::with_eviction(rate_per_sec, burst, EVICTION_IDLE) + Self::with_capacity(rate_per_sec, burst, MAX_BUCKETS, EVICTION_IDLE) } - fn with_eviction(rate_per_sec: f64, burst: f64, eviction_idle: Duration) -> Self { + /// Builds a limiter with an explicit capacity policy. + /// + /// `max_buckets` is the hard bound of the table and `eviction_idle` the + /// idle age after which [`Self::sweep`] may reclaim a bucket. [`Self::new`] + /// supplies the production defaults; this constructor exists for tests and + /// for deployments with a tighter memory budget. + #[must_use] + pub fn with_capacity(rate_per_sec: f64, burst: f64, max_buckets: usize, eviction_idle: Duration) -> Self { Self { - buckets: Arc::new(DashMap::new()), + shards: (0..SHARDS).map(|_| Mutex::new(HashMap::new())).collect(), + hasher: RandomState::new(), + live: AtomicUsize::new(0), + max_buckets, max_tokens: burst, refill_rate: rate_per_sec, - _refill_interval: Duration::from_secs(1), - epoch: Instant::now(), - last_sweep_elapsed: AtomicU64::new(0), eviction_idle, + sweep_cursor: AtomicUsize::new(0), } } + /// Consumes one token for `ip`, returning `false` when the source is over + /// its rate or the table is saturated (see the module documentation). pub fn check(&self, ip: IpAddr) -> bool { let now = Instant::now(); - let mut entry = self.buckets.entry(ip).or_insert_with(|| Bucket { + let index = self.shard_index(ip); + let mut shard = self.lock(index); + if let Some(bucket) = shard.get_mut(&ip) { + return refill_and_take(bucket, now, self.max_tokens, self.refill_rate); + } + if !self.reserve() { + return false; + } + let mut bucket = Bucket { tokens: self.max_tokens, last_refill: now, last_access: now, - }); - - let elapsed = now.duration_since(entry.last_refill); - let refill = elapsed.as_secs_f64() * self.refill_rate; - entry.tokens = (entry.tokens + refill).min(self.max_tokens); - entry.last_refill = now; - entry.last_access = now; - - if entry.tokens >= 1.0 { - entry.tokens -= 1.0; - true - } else { - false - } + }; + let allowed = refill_and_take(&mut bucket, now, self.max_tokens, self.refill_rate); + shard.insert(ip, bucket); + allowed } + /// Number of live buckets. + #[must_use] pub fn len(&self) -> usize { - self.buckets.len() + (0..self.shards.len()).map(|index| self.lock(index).len()).sum() } + /// Whether the table is empty. + #[must_use] pub fn is_empty(&self) -> bool { - self.buckets.is_empty() + self.len() == 0 } - /// Evicts idle buckets. - /// - /// Runs either when the table grows past `MAX_BUCKETS` (cap enforcement) - /// or once per `SWEEP_INTERVAL`. Buckets touched within `eviction_idle` - /// survive, so active sources keep their token state. - pub fn sweep(&self) { + /// Evicts idle buckets from the next shard slice and returns how many were + /// removed. See the eviction policy in the module documentation. + pub fn sweep(&self) -> usize { let now = Instant::now(); - let elapsed_secs = now.duration_since(self.epoch).as_secs(); - let last = self.last_sweep_elapsed.load(Ordering::Relaxed); - let due = last == 0 || elapsed_secs.saturating_sub(last) >= SWEEP_INTERVAL.as_secs(); - let over_cap = self.buckets.len() > MAX_BUCKETS; - if !over_cap && !due { - return; + let shard_count = self.shards.len(); + let start = self.sweep_cursor.fetch_add(SWEEP_SHARDS_PER_TICK, Ordering::Relaxed); + let mut removed = 0; + for offset in 0..SWEEP_SHARDS_PER_TICK.min(shard_count) { + let index = (start + offset) % shard_count; + let mut shard = self.lock(index); + let before = shard.len(); + shard.retain(|_, bucket| now.duration_since(bucket.last_access) < self.eviction_idle); + removed += before - shard.len(); } - self.buckets - .retain(|_, b| now.duration_since(b.last_access) < self.eviction_idle); - self.last_sweep_elapsed.store(elapsed_secs, Ordering::Relaxed); + if removed > 0 { + self.live.fetch_sub(removed, Ordering::AcqRel); + } + removed + } + + /// Reserves one slot in the table, enforcing the hard bound. + fn reserve(&self) -> bool { + let live = self.live.fetch_add(1, Ordering::AcqRel); + if live >= self.max_buckets { + self.live.fetch_sub(1, Ordering::AcqRel); + return false; + } + true + } + + fn shard_index(&self, ip: IpAddr) -> usize { + (self.hasher.hash_one(ip) as usize) % self.shards.len() + } + + fn lock(&self, index: usize) -> MutexGuard<'_, HashMap> { + // A poisoned shard still holds consistent token buckets; keep serving + // instead of taking the whole limiter down. + self.shards[index].lock().unwrap_or_else(|poison| poison.into_inner()) } } -#[cfg(test)] -mod tests { - use super::*; - use std::net::{IpAddr, Ipv4Addr}; - - fn test_ip(octet: u8) -> IpAddr { - IpAddr::V4(Ipv4Addr::new(10, 0, 0, octet)) - } - - #[test] - fn test_rate_limit_under() { - let limiter = RateLimiter::new(10.0, 10.0); - assert!(limiter.check(test_ip(1))); - } - - #[test] - fn test_rate_limit_over() { - let limiter = RateLimiter::new(1.0, 1.0); - assert!(limiter.check(test_ip(1))); - assert!(!limiter.check(test_ip(1))); - } - - #[test] - fn test_rate_limit_burst() { - let limiter = RateLimiter::new(1.0, 5.0); - for _ in 0..5 { - assert!(limiter.check(test_ip(2))); - } - assert!(!limiter.check(test_ip(2))); - } - - #[test] - fn test_rate_limit_refill() { - let limiter = RateLimiter::new(100.0, 1.0); - assert!(limiter.check(test_ip(3))); - assert!(!limiter.check(test_ip(3))); - std::thread::sleep(Duration::from_millis(20)); - assert!(limiter.check(test_ip(3))); - } - - #[test] - fn test_sweep_removes_idle_keeps_active() { - let limiter = RateLimiter::with_eviction(1.0, 10.0, Duration::from_millis(20)); - limiter.check(test_ip(1)); - limiter.check(test_ip(2)); - std::thread::sleep(Duration::from_millis(50)); - limiter.check(test_ip(2)); - limiter.sweep(); - assert_eq!(limiter.len(), 1); - assert!(!limiter.buckets.contains_key(&test_ip(1))); - assert!(limiter.buckets.contains_key(&test_ip(2))); - } - - #[test] - fn test_sweep_does_not_remove_active() { - let limiter = RateLimiter::with_eviction(1.0, 10.0, Duration::from_millis(50)); - limiter.check(test_ip(1)); - limiter.sweep(); - assert_eq!(limiter.len(), 1); - assert!(limiter.buckets.contains_key(&test_ip(1))); +fn refill_and_take(bucket: &mut Bucket, now: Instant, max_tokens: f64, refill_rate: f64) -> bool { + let elapsed = now.duration_since(bucket.last_refill); + bucket.tokens = (bucket.tokens + elapsed.as_secs_f64() * refill_rate).min(max_tokens); + bucket.last_refill = now; + bucket.last_access = now; + if bucket.tokens >= 1.0 { + bucket.tokens -= 1.0; + true + } else { + false } } diff --git a/src/manager/api/auth.rs b/src/manager/api/auth.rs index 8c0e491..4d60333 100644 --- a/src/manager/api/auth.rs +++ b/src/manager/api/auth.rs @@ -1,3 +1,5 @@ +//! `POST /api/v1/auth/login`: password check with a per-IP attempt limiter. + use crate::manager::AppState; use axum::{ Json, @@ -11,7 +13,6 @@ use std::{ sync::Arc, time::{Duration, Instant}, }; -use subtle::ConstantTimeEq; const LOGIN_WINDOW: Duration = Duration::from_secs(60); const LOGIN_MAX_ATTEMPTS: u32 = 5; @@ -33,7 +34,7 @@ pub async fn login( )); } - if !verify_password(&req.password, &state.api_password) { + if !crate::manager::security::verify_password(&req.password, &state.api_password) { return Err(( StatusCode::UNAUTHORIZED, Json(serde_json::json!({"error": "invalid password"})), @@ -49,16 +50,11 @@ pub async fn login( } } -fn verify_password(provided: &str, expected: &str) -> bool { - let provided = provided.as_bytes(); - let expected = expected.as_bytes(); - let len_match = (provided.len() as u64).ct_eq(&(expected.len() as u64)); - let min_len = provided.len().min(expected.len()); - let bytes_match = provided[..min_len].ct_eq(&expected[..min_len]); - bool::from(len_match & bytes_match) -} - -fn allow_login_attempt(limiter: &DashMap, ip: IpAddr) -> bool { +/// Records a login attempt for `ip` and reports whether it is within the window. +/// +/// The limiter allows `LOGIN_MAX_ATTEMPTS` attempts per `LOGIN_WINDOW` and +/// resets the counter when the window elapses. +pub fn allow_login_attempt(limiter: &DashMap, ip: IpAddr) -> bool { let now = Instant::now(); let mut slot = limiter.entry(ip).or_insert((now, 0)); let (last_reset, attempts) = &mut *slot; @@ -72,33 +68,3 @@ fn allow_login_attempt(limiter: &DashMap, ip: IpAddr) -> } true } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn verify_password_matches() { - assert!(verify_password("s3cret-pass", "s3cret-pass")); - } - - #[test] - fn verify_password_rejects_wrong() { - assert!(!verify_password("wrong-pass", "s3cret-pass")); - } - - #[test] - fn verify_password_rejects_different_length() { - assert!(!verify_password("short", "a-longer-password")); - } - - #[test] - fn allow_login_attempt_respects_limit() { - let limiter = DashMap::new(); - let ip = IpAddr::from([127, 0, 0, 1]); - for _ in 0..LOGIN_MAX_ATTEMPTS { - assert!(allow_login_attempt(&limiter, ip)); - } - assert!(!allow_login_attempt(&limiter, ip)); - } -} diff --git a/src/manager/mod.rs b/src/manager/mod.rs index 0f5b051..4ccaf84 100644 --- a/src/manager/mod.rs +++ b/src/manager/mod.rs @@ -3,6 +3,7 @@ //! Owns [`AppState`], the handle shared by every handler through [`SharedState`]. pub mod api; pub mod auth; +pub mod security; pub mod sync; use dashmap::DashMap; diff --git a/src/manager/security.rs b/src/manager/security.rs new file mode 100644 index 0000000..305b45e --- /dev/null +++ b/src/manager/security.rs @@ -0,0 +1,135 @@ +//! Startup security policy of the management plane. +//! +//! Weak credentials turn the control plane API into an authentication bypass: +//! an empty `API_PASSWORD` matches an empty login, and a guessable +//! `JWT_SECRET` lets anyone mint admin tokens. The manager validates both +//! before it binds a socket, and derives its CORS policy from an explicit +//! `CORS_ORIGIN` value instead of silently allowing every origin. + +use anyhow::{Result, bail}; +use subtle::ConstantTimeEq; +use tower_http::cors::{AllowOrigin, Any, CorsLayer}; + +/// Minimum length of `API_PASSWORD` in characters. +const MIN_API_PASSWORD_CHARS: usize = 16; +/// Minimum number of non-whitespace bytes in `JWT_SECRET`. +const MIN_JWT_SECRET_BYTES: usize = 32; +/// Placeholder password shipped in documentation and examples. +const DEFAULT_API_PASSWORD: &str = "changeme"; + +/// Validates `API_PASSWORD` before the manager starts serving. +/// +/// # Errors +/// Rejects an empty or whitespace-only password, a password shorter than +/// `MIN_API_PASSWORD_CHARS` characters, and the `changeme` placeholder. +pub fn validate_api_password(password: &str) -> Result<()> { + let trimmed = password.trim(); + if trimmed.is_empty() { + bail!("API_PASSWORD must not be empty: an empty password would accept an empty login"); + } + if trimmed.chars().count() < MIN_API_PASSWORD_CHARS { + bail!("API_PASSWORD must be at least {MIN_API_PASSWORD_CHARS} characters long"); + } + if trimmed.eq_ignore_ascii_case(DEFAULT_API_PASSWORD) { + bail!("API_PASSWORD must not be the default placeholder '{DEFAULT_API_PASSWORD}'"); + } + Ok(()) +} + +/// Validates `JWT_SECRET` before the manager starts serving. +/// +/// # Errors +/// Rejects a secret with fewer than `MIN_JWT_SECRET_BYTES` non-whitespace +/// bytes, which includes empty and all-whitespace secrets. +pub fn validate_jwt_secret(secret: &str) -> Result<()> { + let significant = secret.bytes().filter(|byte| !byte.is_ascii_whitespace()).count(); + if significant < MIN_JWT_SECRET_BYTES { + bail!("JWT_SECRET must contain at least {MIN_JWT_SECRET_BYTES} non-whitespace bytes, got {significant}"); + } + Ok(()) +} + +/// Compares a login password with the configured one in constant time. +/// +/// An empty stored password never verifies: that is the authentication bypass +/// the startup policy ([`validate_api_password`]) rules out, and the handler +/// keeps the same guarantee for any state built by other code paths. +#[must_use] +pub fn verify_password(provided: &str, expected: &str) -> bool { + if expected.is_empty() { + return false; + } + let provided = provided.as_bytes(); + let expected = expected.as_bytes(); + let len_match = (provided.len() as u64).ct_eq(&(expected.len() as u64)); + let min_len = provided.len().min(expected.len()); + let bytes_match = provided[..min_len].ct_eq(&expected[..min_len]); + bool::from(len_match & bytes_match) +} + +/// Cross-origin resource sharing policy derived from `CORS_ORIGIN`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum CorsPolicy { + /// No cross-origin layer at all: browsers on other origins are denied. + Deny, + /// Every origin is allowed; requires an explicit `CORS_ORIGIN=*`. + Any, + /// Only the listed origins are allowed. + AllowList(Vec), +} + +impl CorsPolicy { + /// Parses the raw `CORS_ORIGIN` value. + /// + /// An unset or empty variable denies cross-origin access, `*` must be set + /// explicitly to allow any origin, and anything else is a comma-separated + /// list of origins. + /// + /// # Errors + /// Fails on an empty entry, on a wildcard mixed with concrete origins, and + /// on an entry that is not a valid HTTP header value. + pub fn parse(raw: Option<&str>) -> Result { + let Some(raw) = raw else { + return Ok(Self::Deny); + }; + let raw = raw.trim(); + if raw.is_empty() { + return Ok(Self::Deny); + } + if raw == "*" { + return Ok(Self::Any); + } + let mut origins = Vec::new(); + for entry in raw.split(',') { + let origin = entry.trim(); + if origin.is_empty() { + bail!("CORS_ORIGIN contains an empty origin entry"); + } + if origin == "*" { + bail!("CORS_ORIGIN wildcard '*' must be used alone, not inside an origin list"); + } + axum::http::HeaderValue::from_str(origin) + .map_err(|_| anyhow::anyhow!("CORS_ORIGIN entry '{origin}' is not a valid HTTP header value"))?; + origins.push(origin.to_owned()); + } + Ok(Self::AllowList(origins)) + } + + /// Builds the axum CORS layer for this policy. + /// + /// Returns `None` for [`CorsPolicy::Deny`] so the router is served without + /// any cross-origin layer. + #[must_use] + pub fn cors_layer(&self) -> Option { + match self { + Self::Deny => None, + Self::Any => Some(CorsLayer::new().allow_origin(Any)), + Self::AllowList(origins) => { + let allowed = origins + .iter() + .filter_map(|origin| axum::http::HeaderValue::from_str(origin).ok()); + Some(CorsLayer::new().allow_origin(AllowOrigin::list(allowed))) + }, + } + } +} diff --git a/src/store/clickhouse.rs b/src/store/clickhouse.rs index fd1530b..7c1a753 100644 --- a/src/store/clickhouse.rs +++ b/src/store/clickhouse.rs @@ -1,13 +1,42 @@ +//! ClickHouse event sink. +//! +//! The data plane must never block on the sink: [`ClickHouseWriter::push`] only +//! touches memory, while the background task drains a batch under the writer +//! lock and performs network I/O outside of it. Every request is bounded by a +//! request timeout and every flush by a global timeout, so an unreachable or +//! hung ClickHouse degrades into dropped events instead of a stalled daemon. + use chrono::{DateTime, Utc}; +use prometheus::IntCounter; use serde::Serialize; -use std::sync::Arc; +use std::sync::{Arc, LazyLock}; use std::time::Duration; use tokio::sync::Mutex; use tokio::sync::watch; -const BATCH_SIZE: usize = 1000; -const FLUSH_INTERVAL: Duration = Duration::from_secs(1); +/// Maximum number of events sent in one INSERT. +pub const BATCH_SIZE: usize = 1000; +/// Hard cap on buffered events; further pushes are dropped and counted. +pub const MAX_BUFFER_EVENTS: usize = 10_000; +const FLUSH_INTERVAL: Duration = Duration::from_secs(1); +const FLUSH_TIMEOUT: Duration = Duration::from_secs(5); +const REQUEST_TIMEOUT: Duration = Duration::from_secs(2); +const CONNECT_TIMEOUT: Duration = Duration::from_secs(1); +const ERROR_BODY_LIMIT: usize = 512; +/// Batches drained per tick; keeps sustained event rates from backing up. +const MAX_BATCHES_PER_FLUSH: usize = 4; + +/// Events discarded because the buffer was full or a retry could not fit. +pub static CLICKHOUSE_DROPPED: LazyLock = LazyLock::new(|| { + prometheus::register_int_counter!( + "rampart_clickhouse_dropped_total", + "ClickHouse events dropped (buffer overflow or failed flush)" + ) + .expect("CLICKHOUSE_DROPPED") +}); + +/// One row of the `rampart_events` table. #[derive(Debug, Clone, Serialize)] pub struct ClickHouseEvent { pub timestamp: DateTime, @@ -18,72 +47,186 @@ pub struct ClickHouseEvent { pub data_string: String, } +/// Buffered ClickHouse writer shared by the data plane and the flush task. pub struct ClickHouseWriter { url: String, client: reqwest::Client, buffer: Vec, + dropped: u64, } impl ClickHouseWriter { + /// Creates a writer for `url` with bounded request and connect timeouts. pub fn new(url: &str) -> Self { + let client = reqwest::Client::builder() + .timeout(REQUEST_TIMEOUT) + .connect_timeout(CONNECT_TIMEOUT) + .build() + .expect("clickhouse http client"); Self { url: url.to_string(), - client: reqwest::Client::new(), + client, buffer: Vec::with_capacity(BATCH_SIZE), + dropped: 0, } } + /// Queues `event` for the next flush. + /// + /// This never performs I/O and never fails; when the buffer is full the + /// event is dropped and counted instead of blocking the data plane. + /// + /// # Errors + /// Currently infallible; the result is kept for API stability. pub async fn push(&mut self, event: ClickHouseEvent) -> anyhow::Result<()> { - self.buffer.push(event); - if self.buffer.len() >= BATCH_SIZE { - self.flush().await?; - } - Ok(()) - } - - pub async fn flush(&mut self) -> anyhow::Result<()> { - if self.buffer.is_empty() { + if self.buffer.len() >= MAX_BUFFER_EVENTS { + self.record_drop(1); return Ok(()); } - let events = std::mem::take(&mut self.buffer); - let json = serde_json::to_string(&events)?; - let response = self - .client - .post(&self.url) - .query(&[("query", "INSERT INTO rampart_events FORMAT JSONEachRow")]) - .header("Content-Type", "application/json") - .body(json) - .send() - .await?; - let status = response.status(); - if !status.is_success() { - let text = response.text().await?; - anyhow::bail!("clickhouse insert failed ({}): {}", status, text); - } - tracing::debug!("flushed {} events to clickhouse", events.len()); + self.buffer.push(event); Ok(()) } + + /// Sends every buffered event, re-queueing the batch on failure. + /// + /// # Errors + /// Returns an error when the insert fails or exceeds `FLUSH_TIMEOUT`; the + /// batch stays buffered for the next attempt. + pub async fn flush(&mut self) -> anyhow::Result<()> { + let batch = std::mem::take(&mut self.buffer); + if batch.is_empty() { + return Ok(()); + } + match tokio::time::timeout(FLUSH_TIMEOUT, send_batch(&self.client, &self.url, &batch)).await { + Ok(Ok(())) => { + tracing::debug!(events = batch.len(), "flushed events to clickhouse"); + Ok(()) + }, + Ok(Err(error)) => { + self.requeue_batch(batch); + Err(error) + }, + Err(_) => { + self.requeue_batch(batch); + anyhow::bail!("clickhouse flush timed out after {FLUSH_TIMEOUT:?}") + }, + } + } + + /// Number of events waiting in the buffer. + pub fn pending(&self) -> usize { + self.buffer.len() + } + + /// Number of events this writer discarded. + pub fn dropped(&self) -> u64 { + self.dropped + } + + fn take_batch(&mut self) -> Vec { + let take = self.buffer.len().min(BATCH_SIZE); + self.buffer.drain(..take).collect() + } + + /// Puts a failed batch back in front of the buffer, keeping it bounded. + fn requeue_batch(&mut self, mut batch: Vec) { + batch.append(&mut self.buffer); + if batch.len() > MAX_BUFFER_EVENTS { + let overflow = batch.len() - MAX_BUFFER_EVENTS; + batch.truncate(MAX_BUFFER_EVENTS); + self.record_drop(overflow); + } + self.buffer = batch; + } + + fn record_drop(&mut self, count: usize) { + let count = u64::try_from(count).unwrap_or(u64::MAX); + let first_drop = self.dropped == 0; + self.dropped = self.dropped.saturating_add(count); + CLICKHOUSE_DROPPED.inc_by(count); + // Log once per writer: further drops are visible in the counter only, + // so a sustained outage cannot flood the log. + if first_drop { + tracing::warn!(count, "clickhouse buffer full; dropping events"); + } + } } +/// Runs the periodic flush loop until shutdown. +/// +/// A closed shutdown channel is treated as a shutdown request: the final +/// flush runs and the task exits instead of busy-looping on `changed()`. pub fn start_flush_task(writer: Arc>, mut shutdown: watch::Receiver) { tokio::spawn(async move { loop { tokio::select! { _ = tokio::time::sleep(FLUSH_INTERVAL) => { - if let Err(e) = writer.lock().await.flush().await { - tracing::error!("clickhouse flush error: {e}"); - } - } - _ = shutdown.changed() => { - if *shutdown.borrow() { - tracing::info!("flushing clickhouse on shutdown"); - if let Err(e) = writer.lock().await.flush().await { - tracing::error!("clickhouse final flush error: {e}"); + for _ in 0..MAX_BATCHES_PER_FLUSH { + if !flush_batch(&writer).await { + break; } + } + }, + changed = shutdown.changed() => { + if changed.is_err() || *shutdown.borrow_and_update() { + tracing::info!("flushing clickhouse on shutdown"); + flush_batch(&writer).await; return; } - } + }, } } }); } + +/// Drains one batch under the lock and sends it without holding the lock. +/// +/// Returns `false` when there was nothing to send or the send failed. +async fn flush_batch(writer: &Arc>) -> bool { + let (client, url, batch) = { + let mut writer = writer.lock().await; + let batch = writer.take_batch(); + (writer.client.clone(), writer.url.clone(), batch) + }; + if batch.is_empty() { + return false; + } + match tokio::time::timeout(FLUSH_TIMEOUT, send_batch(&client, &url, &batch)).await { + Ok(Ok(())) => { + tracing::debug!(events = batch.len(), "flushed events to clickhouse"); + true + }, + Ok(Err(error)) => { + tracing::warn!(%error, events = batch.len(), "clickhouse flush failed; re-queueing"); + requeue_batch(writer, batch).await; + false + }, + Err(_) => { + tracing::warn!(events = batch.len(), "clickhouse flush timed out; re-queueing"); + requeue_batch(writer, batch).await; + false + }, + } +} + +async fn requeue_batch(writer: &Arc>, batch: Vec) { + writer.lock().await.requeue_batch(batch); +} + +async fn send_batch(client: &reqwest::Client, url: &str, batch: &[ClickHouseEvent]) -> anyhow::Result<()> { + let json = serde_json::to_string(batch)?; + let response = client + .post(url) + .query(&[("query", "INSERT INTO rampart_events FORMAT JSONEachRow")]) + .header("Content-Type", "application/json") + .body(json) + .send() + .await?; + let status = response.status(); + if !status.is_success() { + let mut text = response.text().await.unwrap_or_default(); + text.truncate(ERROR_BODY_LIMIT); + anyhow::bail!("clickhouse insert failed ({status}): {text}"); + } + Ok(()) +} diff --git a/src/store/redis.rs b/src/store/redis.rs index 721b912..bd892c2 100644 --- a/src/store/redis.rs +++ b/src/store/redis.rs @@ -1,41 +1,59 @@ +//! Redis-backed [`StateStore`] and blacklist synchronisation. +//! +//! Commands share one reconnecting [`ConnectionManager`] and are bounded by +//! `connection::REDIS_TIMEOUT`; pub/sub connections are acquired and +//! subscribed under the same timeout. + use crate::store::StateStore; use redis::AsyncCommands; +use redis::aio::ConnectionManager; +mod backoff; +mod connection; mod sync; + +pub use backoff::{BACKOFF_MAX, BACKOFF_MIN, next_backoff}; pub use sync::{start_blacklist_sync, start_local_ban_publisher}; +/// Redis-backed state store. pub struct RedisStore { - client: redis::Client, + manager: ConnectionManager, } impl RedisStore { - pub fn new(url: &str) -> anyhow::Result { + /// Connects to `url` and establishes a reconnecting manager. + /// + /// # Errors + /// Returns an error for an invalid URL, an unreachable server or an attempt + /// that exceeds `connection::REDIS_TIMEOUT`. + pub async fn new(url: &str) -> anyhow::Result { let client = redis::Client::open(url)?; - Ok(Self { client }) + let manager = connection::connection_manager(&client).await?; + Ok(Self { manager }) } } impl StateStore for RedisStore { async fn get(&self, key: &str) -> anyhow::Result> { - let mut conn = self.client.get_multiplexed_async_connection().await?; - Ok(conn.get(key).await?) + let mut conn = self.manager.clone(); + connection::command(conn.get(key)).await } async fn set(&self, key: &str, value: &str) -> anyhow::Result<()> { - let mut conn = self.client.get_multiplexed_async_connection().await?; - let _: () = conn.set(key, value).await?; + let mut conn = self.manager.clone(); + let _: () = connection::command(conn.set(key, value)).await?; Ok(()) } async fn del(&self, key: &str) -> anyhow::Result<()> { - let mut conn = self.client.get_multiplexed_async_connection().await?; - let _: () = conn.del(key).await?; + let mut conn = self.manager.clone(); + let _: () = connection::command(conn.del(key)).await?; Ok(()) } async fn publish(&self, channel: &str, message: &str) -> anyhow::Result<()> { - let mut conn = self.client.get_multiplexed_async_connection().await?; - let _: () = conn.publish(channel, message).await?; + let mut conn = self.manager.clone(); + let _: i64 = connection::command(conn.publish(channel, message)).await?; Ok(()) } } diff --git a/src/store/redis/backoff.rs b/src/store/redis/backoff.rs new file mode 100644 index 0000000..7a95c9e --- /dev/null +++ b/src/store/redis/backoff.rs @@ -0,0 +1,14 @@ +//! Exponential reconnect backoff shared by the Redis sync tasks. + +use std::time::Duration; + +/// Initial reconnect delay. +pub const BACKOFF_MIN: Duration = Duration::from_secs(1); +/// Upper bound for the reconnect delay. +pub const BACKOFF_MAX: Duration = Duration::from_secs(30); + +/// Doubles `previous`, capped at `max`; never overflows. +#[must_use] +pub fn next_backoff(previous: Duration, max: Duration) -> Duration { + previous.saturating_mul(2).min(max) +} diff --git a/src/store/redis/connection.rs b/src/store/redis/connection.rs new file mode 100644 index 0000000..bf6089e --- /dev/null +++ b/src/store/redis/connection.rs @@ -0,0 +1,36 @@ +//! Timeout-bounded Redis connection helpers. +//! +//! Every connect and command in the store layer goes through this module so a +//! stalled server can never block a data-plane or shutdown path. + +use redis::aio::ConnectionManager; +use std::future::Future; +use std::time::Duration; + +/// Maximum time spent on one Redis connect or command. +pub const REDIS_TIMEOUT: Duration = Duration::from_secs(2); + +/// Opens a reconnecting connection manager, bounded by `REDIS_TIMEOUT`. +/// +/// # Errors +/// Returns an error for an unreachable server or a timed-out attempt. +pub async fn connection_manager(client: &redis::Client) -> anyhow::Result { + match tokio::time::timeout(REDIS_TIMEOUT, client.get_connection_manager()).await { + Ok(result) => Ok(result?), + Err(_) => anyhow::bail!("redis connection timed out after {REDIS_TIMEOUT:?}"), + } +} + +/// Awaits a Redis command future, bounded by `REDIS_TIMEOUT`. +/// +/// # Errors +/// Returns an error when the command fails or exceeds the timeout. +pub async fn command(future: F) -> anyhow::Result +where + F: Future>, +{ + match tokio::time::timeout(REDIS_TIMEOUT, future).await { + Ok(result) => Ok(result?), + Err(_) => anyhow::bail!("redis command timed out after {REDIS_TIMEOUT:?}"), + } +} diff --git a/src/store/redis/sync.rs b/src/store/redis/sync.rs index 8c25d4d..c4042d9 100644 --- a/src/store/redis/sync.rs +++ b/src/store/redis/sync.rs @@ -1,9 +1,18 @@ +//! Blacklist synchronisation over a Redis pub/sub channel. +//! +//! The subscriber reconciles the durable snapshot on every (re)connect, then +//! applies events. All connects, subscribes and commands are timeout-bounded; +//! a failed attempt backs off exponentially and a panicking handler is logged +//! instead of killing the task. + mod event; mod publisher; mod snapshot; pub use publisher::start_local_ban_publisher; +use super::backoff::{BACKOFF_MAX, BACKOFF_MIN, next_backoff}; +use super::connection::{REDIS_TIMEOUT, connection_manager}; use crate::filter::blacklist::Blacklist; use futures::StreamExt; use std::sync::Arc; @@ -11,19 +20,17 @@ use std::time::Duration; use tokio::sync::watch; const BLACKLIST_CHANNEL: &str = "rampart:blacklist:events"; -const BACKOFF_MIN: Duration = Duration::from_secs(1); -const BACKOFF_MAX: Duration = Duration::from_secs(30); - -fn next_backoff(prev: Duration, max: Duration) -> Duration { - prev.saturating_mul(2).min(max) -} +/// Why a subscription attempt ended. enum Flow { + /// Shutdown was requested. Stop, + /// The attempt failed; reconnect after a backoff delay. Retry, } -async fn wait_for_shutdown(shutdown: &mut watch::Receiver) { +/// Resolves once shutdown is requested or the channel is closed. +pub(super) async fn wait_for_shutdown(shutdown: &mut watch::Receiver) { loop { if shutdown.changed().await.is_err() || *shutdown.borrow_and_update() { return; @@ -31,6 +38,7 @@ async fn wait_for_shutdown(shutdown: &mut watch::Receiver) { } } +/// Runs the blacklist subscriber until shutdown. pub async fn start_blacklist_sync( client: &redis::Client, blacklist: Arc, @@ -63,30 +71,44 @@ async fn subscribe_once( backoff: &mut Duration, ) -> Flow { let mut pubsub = tokio::select! { - connected = client.get_async_pubsub() => match connected { - Ok(pubsub) => pubsub, - Err(e) => { + connected = tokio::time::timeout(REDIS_TIMEOUT, client.get_async_pubsub()) => match connected { + Ok(Ok(pubsub)) => pubsub, + Ok(Err(e)) => { tracing::error!("failed to connect to Redis for blacklist sync: {e}"); return Flow::Retry; }, + Err(_) => { + tracing::error!("timed out connecting to Redis for blacklist sync"); + return Flow::Retry; + }, }, () = wait_for_shutdown(shutdown) => return Flow::Stop, }; tokio::select! { - subscribed = pubsub.subscribe(BLACKLIST_CHANNEL) => match subscribed { - Ok(()) => tracing::info!("subscribed to {BLACKLIST_CHANNEL}"), - Err(e) => { + subscribed = tokio::time::timeout(REDIS_TIMEOUT, pubsub.subscribe(BLACKLIST_CHANNEL)) => match subscribed { + Ok(Ok(())) => { + tracing::info!("subscribed to {BLACKLIST_CHANNEL}"); + *backoff = BACKOFF_MIN; + }, + Ok(Err(e)) => { tracing::error!("failed to subscribe to blacklist events: {e}"); return Flow::Retry; }, + Err(_) => { + tracing::error!("timed out subscribing to blacklist events"); + return Flow::Retry; + }, }, () = wait_for_shutdown(shutdown) => return Flow::Stop, } tokio::select! { - loaded = snapshot::reconcile(client, blacklist) => match loaded { - Ok(()) => tracing::info!("blacklist snapshot reconciled from Redis"), + loaded = reconcile(blacklist, client) => match loaded { + Ok(()) => { + tracing::info!("blacklist snapshot reconciled from Redis"); + *backoff = BACKOFF_MIN; + }, Err(e) => { tracing::error!("failed to reconcile blacklist snapshot: {e}"); return Flow::Retry; @@ -99,13 +121,7 @@ async fn subscribe_once( loop { tokio::select! { maybe_msg = stream.next() => match maybe_msg { - Some(msg) => { - if let Err(e) = event::handle(&msg, blacklist) { - tracing::error!("blacklist event error: {e}"); - } else { - *backoff = BACKOFF_MIN; - } - }, + Some(msg) => handle_message(&msg, blacklist, backoff), None => { tracing::error!("blacklist pubsub stream ended; resubscribing"); return Flow::Retry; @@ -116,22 +132,18 @@ async fn subscribe_once( } } -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn backoff_doubles_until_cap() { - let mut b = BACKOFF_MIN; - for expected_secs in [2, 4, 8, 16, 30, 30] { - b = next_backoff(b, BACKOFF_MAX); - assert_eq!(b, Duration::from_secs(expected_secs)); - } - } - - #[test] - fn backoff_never_exceeds_max_even_with_huge_prev() { - let b = next_backoff(Duration::from_secs(u64::MAX / 2), BACKOFF_MAX); - assert!(b <= BACKOFF_MAX); +/// Handles one event without letting a rejected or panicking payload kill the +/// subscriber task. +fn handle_message(msg: &redis::Msg, blacklist: &Blacklist, backoff: &mut Duration) { + let handled = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| event::handle(msg, blacklist))); + match handled { + Ok(Ok(())) => *backoff = BACKOFF_MIN, + Ok(Err(error)) => tracing::warn!(%error, "rejected blacklist event"), + Err(_) => tracing::error!("blacklist event handler panicked; continuing"), } } + +async fn reconcile(blacklist: &Blacklist, client: &redis::Client) -> anyhow::Result<()> { + let manager = connection_manager(client).await?; + snapshot::reconcile(&manager, blacklist).await +} diff --git a/src/store/redis/sync/event.rs b/src/store/redis/sync/event.rs index 15befe3..80f8dae 100644 --- a/src/store/redis/sync/event.rs +++ b/src/store/redis/sync/event.rs @@ -1,10 +1,14 @@ -use crate::filter::blacklist::Blacklist; +//! Decoding and validation of blacklist events received from Redis. +//! +//! A remote or stale manager must not be able to ban an address for an +//! arbitrary duration, so every ban is range-checked against +//! [`MAX_BAN_DURATION_SECS`] before it reaches the in-memory table. + +use crate::filter::blacklist::{Blacklist, MAX_BAN_DURATION_SECS}; use redis::Msg; use serde::Deserialize; use std::time::Duration; -const MAX_BAN_DURATION_SECS: u64 = 31_536_000; - #[derive(Deserialize)] struct BlacklistEvent { ip: String, @@ -27,31 +31,54 @@ pub(super) fn handle(msg: &Msg, blacklist: &Blacklist) -> anyhow::Result<()> { match event.action.as_str() { "ban" => { let duration = match event.expires_at_unix { - Some(expires_at) => { - if chrono::DateTime::from_timestamp(expires_at, 0).is_none() { - anyhow::bail!("invalid blacklist event expiry timestamp"); - } - let remaining = expires_at.saturating_sub(chrono::Utc::now().timestamp()); - if remaining <= 0 { - blacklist.remove_redis(ip); - return Ok(()); - } - let remaining = u64::try_from(remaining)?; - if remaining > MAX_BAN_DURATION_SECS { - anyhow::bail!("blacklist event expiry exceeds maximum duration"); - } - Duration::from_secs(remaining) - }, - None => Duration::from_secs(event.duration_secs), + Some(expires_at) => absolute_duration(expires_at)?, + None => checked_duration(event.duration_secs)?, }; - blacklist.add_redis(ip, duration, "manager_event"); - tracing::info!("blacklist add via Redis: {}", event.ip); + match duration { + Some(duration) => { + blacklist.add_redis(ip, duration, "manager_event"); + tracing::info!(ip = %event.ip, "blacklist add via Redis"); + }, + None => { + blacklist.remove_redis(ip); + tracing::info!(ip = %event.ip, "blacklist expiry already passed"); + }, + } }, "unban" => { blacklist.remove_redis(ip); - tracing::info!("blacklist remove via Redis: {}", event.ip); + tracing::info!(ip = %event.ip, "blacklist remove via Redis"); }, - action => anyhow::bail!("unknown action: {action}"), + action => anyhow::bail!("unknown blacklist action: {action}"), } Ok(()) } + +/// Converts an absolute expiry into the remaining ban duration. +/// +/// Returns `Ok(None)` when the expiry is already in the past. +fn absolute_duration(expires_at: i64) -> anyhow::Result> { + if chrono::DateTime::from_timestamp(expires_at, 0).is_none() { + anyhow::bail!("invalid blacklist event expiry timestamp"); + } + let remaining = expires_at.saturating_sub(chrono::Utc::now().timestamp()); + if remaining <= 0 { + return Ok(None); + } + let remaining = u64::try_from(remaining)?; + if remaining > MAX_BAN_DURATION_SECS { + anyhow::bail!("blacklist event expiry exceeds {MAX_BAN_DURATION_SECS}s"); + } + Ok(Some(Duration::from_secs(remaining))) +} + +/// Validates a relative ban duration. +fn checked_duration(secs: u64) -> anyhow::Result> { + if secs > MAX_BAN_DURATION_SECS { + anyhow::bail!("blacklist event duration {secs}s exceeds {MAX_BAN_DURATION_SECS}s"); + } + if secs == 0 { + return Ok(None); + } + Ok(Some(Duration::from_secs(secs))) +} diff --git a/src/store/redis/sync/publisher.rs b/src/store/redis/sync/publisher.rs index e1e685c..f88be07 100644 --- a/src/store/redis/sync/publisher.rs +++ b/src/store/redis/sync/publisher.rs @@ -1,6 +1,16 @@ +//! Publishes locally detected bans to Redis. +//! +//! Each ban is stored atomically (set member, metadata hash, event publish) via +//! a Lua script on a shared reconnecting connection. Failed publishes retry +//! with exponential backoff and abandon the event when shutdown is requested. + +use super::wait_for_shutdown; use crate::filter::blacklist::{Blacklist, LocalBanEvent}; +use crate::store::redis::backoff::{BACKOFF_MAX, BACKOFF_MIN, next_backoff}; +use crate::store::redis::connection::{command, connection_manager}; +use redis::aio::ConnectionManager; use serde::Serialize; -use std::time::Duration; +use std::sync::Arc; use tokio::sync::{broadcast, watch}; const BLACKLIST_KEY: &str = "rampart:blacklist"; @@ -24,14 +34,16 @@ struct BanEvent<'a> { reason: &'a str, } +/// Publishes local bans and the current local snapshot until shutdown. pub async fn start_local_ban_publisher( client: redis::Client, - blacklist: std::sync::Arc, + blacklist: Arc, mut events: broadcast::Receiver, mut shutdown: watch::Receiver, ) { + let mut manager: Option = None; for event in blacklist.local_snapshot() { - if !publish_with_retry(&client, event, &mut shutdown).await { + if !publish_with_retry(&client, &mut manager, event, &mut shutdown).await { return; } } @@ -40,14 +52,14 @@ pub async fn start_local_ban_publisher( tokio::select! { received = events.recv() => match received { Ok(event) => { - if !publish_with_retry(&client, event, &mut shutdown).await { + if !publish_with_retry(&client, &mut manager, event, &mut shutdown).await { return; } }, Err(broadcast::error::RecvError::Lagged(skipped)) => { tracing::warn!(skipped, "local ban publisher lagged; reconciling active local bans"); for event in blacklist.local_snapshot() { - if !publish_with_retry(&client, event, &mut shutdown).await { + if !publish_with_retry(&client, &mut manager, event, &mut shutdown).await { return; } } @@ -63,19 +75,29 @@ pub async fn start_local_ban_publisher( } } +/// Publishes one event, retrying until it succeeds or shutdown is requested. async fn publish_with_retry( client: &redis::Client, + manager: &mut Option, event: LocalBanEvent, shutdown: &mut watch::Receiver, ) -> bool { - let mut delay = Duration::from_secs(1); + let mut delay = BACKOFF_MIN; loop { - match publish_ban(client, &event).await { + let result = tokio::select! { + result = publish_ban(client, manager, &event) => result, + () = wait_for_shutdown(shutdown) => return false, + }; + match result { Ok(()) => return true, - Err(error) => tracing::warn!(%error, ip = %event.ip, "failed to persist local ban; retrying"), + Err(error) => { + // Force a reconnect on the next attempt. + *manager = None; + tracing::warn!(%error, ip = %event.ip, "failed to persist local ban; retrying"); + }, } tokio::select! { - _ = tokio::time::sleep(delay) => delay = (delay * 2).min(Duration::from_secs(30)), + _ = tokio::time::sleep(delay) => delay = next_backoff(delay, BACKOFF_MAX), changed = shutdown.changed() => { if changed.is_err() || *shutdown.borrow_and_update() { return false; @@ -85,7 +107,11 @@ async fn publish_with_retry( } } -async fn publish_ban(client: &redis::Client, event: &LocalBanEvent) -> anyhow::Result<()> { +async fn publish_ban( + client: &redis::Client, + manager: &mut Option, + event: &LocalBanEvent, +) -> anyhow::Result<()> { let now = chrono::Utc::now().timestamp(); let remaining = event.expires_at_unix.saturating_sub(now); if remaining <= 0 { @@ -103,17 +129,26 @@ async fn publish_ban(client: &redis::Client, event: &LocalBanEvent) -> anyhow::R expires_at_unix: event.expires_at_unix, reason: &event.reason, })?; - let mut conn = client.get_multiplexed_async_connection().await?; - let _: i64 = redis::cmd("EVAL") - .arg(STORE_BAN_SCRIPT) - .arg(3) - .arg(BLACKLIST_KEY) - .arg(BLACKLIST_METADATA_KEY) - .arg(BLACKLIST_CHANNEL) - .arg(event.ip.to_string()) - .arg(metadata) - .arg(payload) - .query_async(&mut conn) - .await?; + let mut conn = match manager { + Some(conn) => conn.clone(), + None => { + let conn = connection_manager(client).await?; + *manager = Some(conn.clone()); + conn + }, + }; + let _: i64 = command( + redis::cmd("EVAL") + .arg(STORE_BAN_SCRIPT) + .arg(3) + .arg(BLACKLIST_KEY) + .arg(BLACKLIST_METADATA_KEY) + .arg(BLACKLIST_CHANNEL) + .arg(event.ip.to_string()) + .arg(metadata) + .arg(payload) + .query_async(&mut conn), + ) + .await?; Ok(()) } diff --git a/src/store/redis/sync/snapshot.rs b/src/store/redis/sync/snapshot.rs index 70ae343..cfbe9b6 100644 --- a/src/store/redis/sync/snapshot.rs +++ b/src/store/redis/sync/snapshot.rs @@ -1,5 +1,13 @@ -use crate::filter::blacklist::Blacklist; +//! Durable blacklist snapshot reconciliation. +//! +//! The snapshot is rebuilt from `rampart:blacklist` (set) and +//! `rampart:blacklist:metadata` (hash) after every (re)subscribe and then +//! swapped into the mirrored in-memory table. + +use crate::filter::blacklist::{Blacklist, MAX_BAN_DURATION_SECS}; +use crate::store::redis::connection::command; use redis::AsyncCommands; +use redis::aio::ConnectionManager; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::time::Duration; @@ -7,7 +15,6 @@ use std::time::Duration; const BLACKLIST_KEY: &str = "rampart:blacklist"; const BLACKLIST_METADATA_KEY: &str = "rampart:blacklist:metadata"; const LEGACY_BAN_DURATION_SECS: i64 = 300; -const MAX_BAN_DURATION_SECS: i64 = 31_536_000; #[derive(Deserialize, Serialize)] struct BlacklistMetadata { @@ -16,11 +23,15 @@ struct BlacklistMetadata { reason: String, } -/// Restore current durable bans after subscribing to the event channel. -pub(super) async fn reconcile(client: &redis::Client, blacklist: &Blacklist) -> anyhow::Result<()> { - let mut conn = client.get_multiplexed_async_connection().await?; - let members: Vec = redis::cmd("SMEMBERS").arg(BLACKLIST_KEY).query_async(&mut conn).await?; - let metadata: HashMap = conn.hgetall(BLACKLIST_METADATA_KEY).await?; +/// Restores current durable bans after subscribing to the event channel. +/// +/// # Errors +/// Returns an error when Redis is unreachable, a command times out or a +/// recorded timestamp is outside the representable range. +pub(super) async fn reconcile(manager: &ConnectionManager, blacklist: &Blacklist) -> anyhow::Result<()> { + let mut conn = manager.clone(); + let members: Vec = command(redis::cmd("SMEMBERS").arg(BLACKLIST_KEY).query_async(&mut conn)).await?; + let metadata: HashMap = command(conn.hgetall(BLACKLIST_METADATA_KEY)).await?; let now = chrono::Utc::now().timestamp(); let mut active = Vec::with_capacity(members.len()); @@ -39,22 +50,28 @@ pub(super) async fn reconcile(client: &redis::Client, blacklist: &Blacklist) -> None => continue, }, }; - if record.expires_at_unix <= now - || chrono::DateTime::from_timestamp(record.created_at_unix, 0).is_none() + if chrono::DateTime::from_timestamp(record.created_at_unix, 0).is_none() || chrono::DateTime::from_timestamp(record.expires_at_unix, 0).is_none() - || record.expires_at_unix.saturating_sub(now) > MAX_BAN_DURATION_SECS { continue; } - let remaining = u64::try_from(record.expires_at_unix - now)?; + let remaining = record.expires_at_unix.saturating_sub(now); + if remaining <= 0 { + continue; + } + let remaining = u64::try_from(remaining)?; + if remaining > MAX_BAN_DURATION_SECS { + continue; + } active.push((ip, Duration::from_secs(remaining), record.reason)); } blacklist.replace_redis(active); Ok(()) } +/// Backfills metadata for a ban written by an older manager. async fn legacy_record( - conn: &mut redis::aio::MultiplexedConnection, + conn: &mut ConnectionManager, target: &str, now: i64, ) -> anyhow::Result> { @@ -63,15 +80,17 @@ async fn legacy_record( expires_at_unix: now.saturating_add(LEGACY_BAN_DURATION_SECS), reason: "legacy_unverified".to_string(), }; - let inserted: i64 = redis::cmd("HSETNX") - .arg(BLACKLIST_METADATA_KEY) - .arg(target) - .arg(serde_json::to_string(&legacy)?) - .query_async(conn) - .await?; + let inserted: i64 = command( + redis::cmd("HSETNX") + .arg(BLACKLIST_METADATA_KEY) + .arg(target) + .arg(serde_json::to_string(&legacy)?) + .query_async(conn), + ) + .await?; if inserted == 1 { return Ok(Some(legacy)); } - let current: Option = conn.hget(BLACKLIST_METADATA_KEY, target).await?; + let current: Option = command(conn.hget(BLACKLIST_METADATA_KEY, target)).await?; Ok(current.and_then(|value| serde_json::from_str(&value).ok())) } diff --git a/src/traffic/intel/detector.rs b/src/traffic/intel/detector.rs index 011e67a..08afd7e 100644 --- a/src/traffic/intel/detector.rs +++ b/src/traffic/intel/detector.rs @@ -1,16 +1,93 @@ +//! Attack state machine over the hour-of-week traffic profile. +//! +//! [`AttackDetector`] scores every one-second window against the baselines +//! *before* the window is folded into them. Scoring first is what keeps a +//! sustained flood visible: a detector that learns the attack as normal stops +//! escalating after a few ticks, and anything that depends on +//! [`AttackStatus::UnderAttack`] (for example offender bans) stops working for +//! the rest of the attack. +//! +//! Two baselines are combined: +//! +//! - [`TrafficProfiler`]: the hour-of-week shape, updated from clean windows. +//! - A slow seeded EWMA (1% per window): the global traffic level. It covers +//! the cold start and the first tick of every hour slot, where the +//! hour-of-week baseline is still zero or close to it. +//! +//! A window is scored against the higher of the two, and attack-level windows +//! are not recorded at all. A flood therefore never becomes the baseline it is +//! compared to, while ordinary drift (below [`DetectorThresholds::attack_score`]) +//! keeps both baselines current. +//! +//! Leaving [`AttackStatus::UnderAttack`] requires +//! [`DetectorThresholds::recover_after`] consecutive windows below +//! [`DetectorThresholds::attack_score`]. The exit hysteresis keeps a single +//! calm window in the middle of an attack from restarting the escalation +//! count. + +use crate::traffic::ewma::Ewma; use crate::traffic::profiler::TrafficProfiler; +/// Smoothing factor of the slow global baseline: one percent per window, so a +/// legitimate traffic shift is learned in about a hundred seconds. +const SLOW_ALPHA: f64 = 0.01; +/// Upper bound of a reported score; `10` means "at least ten times the +/// baseline" and keeps the attack state machine from chasing huge ratios. +const MAX_ANOMALY_RATIO: f64 = 10.0; + +/// Classification of the current traffic window. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum AttackStatus { + /// Traffic is within the learned baseline. Normal, + /// A ratio above [`DetectorThresholds::suspicious_score`] was observed, + /// but the escalation count is not reached yet. Suspicious, + /// Consecutive anomalous windows were observed; the edge escalates. UnderAttack, } +/// Tunables of the attack state machine. +/// +/// The scores are ratios to the baseline (a value of `3.0` means "three times +/// the usual rate"). The defaults match the v0.3 policy; a caller that needs a +/// different sensitivity passes its own values to +/// [`AttackDetector::with_thresholds`]. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct DetectorThresholds { + /// Ratio that marks a window as suspicious. + pub suspicious_score: f64, + /// Ratio that counts a window as anomalous for escalation. + pub attack_score: f64, + /// Consecutive anomalous windows required to enter `UnderAttack`. + pub escalate_after: u32, + /// Consecutive sub-attack windows required to leave `UnderAttack`. + pub recover_after: u32, +} + +impl Default for DetectorThresholds { + fn default() -> Self { + Self { + suspicious_score: 2.0, + attack_score: 3.0, + escalate_after: 3, + recover_after: 5, + } + } +} + +/// Detects floods from the packets-per-second and connections-per-second +/// profiles. pub struct AttackDetector { pps_profiler: TrafficProfiler, cps_profiler: TrafficProfiler, + pps_slow: Ewma, + cps_slow: Ewma, + primed: bool, + thresholds: DetectorThresholds, consecutive_anomalies: u32, + recovery_windows: u32, + under_attack: bool, } impl Default for AttackDetector { @@ -20,95 +97,99 @@ impl Default for AttackDetector { } impl AttackDetector { + /// Builds a detector with the default [`DetectorThresholds`]. + #[must_use] pub fn new() -> Self { + Self::with_thresholds(DetectorThresholds::default()) + } + + /// Builds a detector with explicit thresholds. + #[must_use] + pub fn with_thresholds(thresholds: DetectorThresholds) -> Self { Self { pps_profiler: TrafficProfiler::new(), cps_profiler: TrafficProfiler::new(), + pps_slow: Ewma::new(SLOW_ALPHA), + cps_slow: Ewma::new(SLOW_ALPHA), + primed: false, + thresholds, consecutive_anomalies: 0, + recovery_windows: 0, + under_attack: false, } } - /// Analyses one metrics window (packets/s and new connections/s) against - /// the accumulated profiles. + /// Analyses one metrics window (packets/s and new connections/s) and + /// returns the attack classification. /// - /// A spike in either metric counts as an anomaly. Three consecutive - /// anomalous windows escalate to [`AttackStatus::UnderAttack`]; a single - /// one above the suspicious score yields [`AttackStatus::Suspicious`]. + /// A window is anomalous when its ratio to the baseline reaches + /// [`DetectorThresholds::attack_score`]. + /// [`DetectorThresholds::escalate_after`] consecutive anomalous windows + /// enter `UnderAttack`; the status is left only after + /// [`DetectorThresholds::recover_after`] consecutive windows below the + /// attack score. The first window only primes the baselines. pub fn analyze(&mut self, pps: f64, cps: f64) -> AttackStatus { - self.pps_profiler.record(pps); - self.cps_profiler.record(cps); - let score = self - .pps_profiler - .anomaly_score(pps) - .max(self.cps_profiler.anomaly_score(cps)); - - if score > 3.0 { - self.consecutive_anomalies += 1; - } else { - self.consecutive_anomalies = 0; + if !self.primed { + self.primed = true; + self.pps_slow.seed(pps); + self.cps_slow.seed(cps); + self.pps_profiler.record(pps); + self.cps_profiler.record(cps); + return AttackStatus::Normal; } - if self.consecutive_anomalies >= 3 { + // Score against the baselines first: recording first would fold the + // spike into the baseline it is scored against. + let pps_baseline = self.pps_profiler.baseline().max(self.pps_slow.value()); + let cps_baseline = self.cps_profiler.baseline().max(self.cps_slow.value()); + let score = anomaly_ratio(pps, pps_baseline).max(anomaly_ratio(cps, cps_baseline)); + + self.update_state(score); + + // Attack-level windows never enter the baselines (see the module + // documentation); everything else keeps both of them current. + if score < self.thresholds.attack_score { + self.pps_profiler.record(pps); + self.cps_profiler.record(cps); + self.pps_slow.update(pps); + self.cps_slow.update(cps); + } + + if self.under_attack { AttackStatus::UnderAttack - } else if score > 2.0 { + } else if score >= self.thresholds.suspicious_score { AttackStatus::Suspicious } else { AttackStatus::Normal } } -} -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_attack_status_normal() { - let mut d = AttackDetector::new(); - for _ in 0..10 { - d.analyze(100.0, 10.0); + fn update_state(&mut self, score: f64) { + if score >= self.thresholds.attack_score { + self.consecutive_anomalies = self.consecutive_anomalies.saturating_add(1); + self.recovery_windows = 0; + } else { + self.consecutive_anomalies = 0; + self.recovery_windows = self.recovery_windows.saturating_add(1); } - assert_eq!(d.analyze(100.0, 10.0), AttackStatus::Normal); - } - #[test] - fn test_attack_status_suspicious_pps_spike() { - let mut d = AttackDetector::new(); - for _ in 0..10 { - d.analyze(1.0, 1.0); + if self.under_attack { + if self.recovery_windows >= self.thresholds.recover_after { + self.under_attack = false; + } + } else if self.consecutive_anomalies >= self.thresholds.escalate_after { + self.under_attack = true; } - assert_eq!(d.analyze(5.0, 1.0), AttackStatus::Suspicious); - } - - #[test] - fn test_attack_status_suspicious_cps_spike() { - let mut d = AttackDetector::new(); - for _ in 0..10 { - d.analyze(1.0, 1.0); - } - assert_eq!(d.analyze(1.0, 5.0), AttackStatus::Suspicious); - } - - #[test] - fn test_attack_status_under_attack() { - let mut d = AttackDetector::new(); - for _ in 0..10 { - d.analyze(1.0, 1.0); - } - d.analyze(10_000.0, 100.0); - d.analyze(10_000.0, 100.0); - assert_eq!(d.analyze(10_000.0, 100.0), AttackStatus::UnderAttack); - } - - #[test] - fn test_consecutive_resets_on_normal() { - let mut d = AttackDetector::new(); - for _ in 0..10 { - d.analyze(1.0, 1.0); - } - d.analyze(100.0, 10.0); - d.analyze(100.0, 10.0); - d.analyze(1.0, 1.0); - assert_eq!(d.analyze(1.0, 1.0), AttackStatus::Normal); } } + +/// Ratio of a window to its baseline, capped at [`MAX_ANOMALY_RATIO`]. +/// +/// Returns zero for an unset baseline and for non-finite windows, so a broken +/// counter cannot pin the detector in the attack state. +fn anomaly_ratio(sample: f64, baseline: f64) -> f64 { + if !sample.is_finite() || baseline <= 0.0 { + return 0.0; + } + (sample.max(0.0) / baseline).min(MAX_ANOMALY_RATIO) +} diff --git a/src/traffic/intel/ewma.rs b/src/traffic/intel/ewma.rs index 74a1b8c..754edb5 100644 --- a/src/traffic/intel/ewma.rs +++ b/src/traffic/intel/ewma.rs @@ -1,5 +1,29 @@ +//! Time-aware exponential moving average used by the traffic profilers. +//! +//! The average is expressed in samples per unit time and is fed once per +//! metrics window by [`crate::traffic::profiler::TrafficProfiler`]. + use std::time::Instant; +/// Smoothing factor used for `alpha` values outside the valid `(0, 1]` range +/// and for `NaN`, so a malformed configuration cannot freeze the baseline. +const DEFAULT_ALPHA: f64 = 0.125; + +/// Exponential moving average with a wall-clock aware step count. +/// +/// # Formula +/// +/// For a sample `x` that arrives `dt` seconds after the previous update, +/// `steps = max(dt, 1)` and `weight = (1 - alpha)^steps`. The average is then +/// +/// ```text +/// value = value * weight + x * (1 - weight) +/// ``` +/// +/// A larger `alpha` gives more weight to the newest sample; `alpha = 1` makes +/// the average equal to the last finite sample. The step count keeps the +/// smoothing independent of the tick rate: one update after a two-second gap +/// is worth two one-second updates. pub struct Ewma { value: f64, last: Instant, @@ -7,51 +31,59 @@ pub struct Ewma { } impl Ewma { + /// Builds a smoothed average that starts at zero. + /// + /// `alpha` is clamped to `(0, 1]`; `NaN`, infinity and non-positive values + /// fall back to the default 0.125 instead of silently disabling the + /// smoothing or producing a non-finite baseline. + #[must_use] pub fn new(alpha: f64) -> Self { Self { value: 0.0, last: Instant::now(), - alpha, + alpha: sanitize_alpha(alpha), } } + /// Folds `sample` into the average, following the formula in the type + /// documentation. + /// + /// Non-finite samples are ignored (the update timestamp is not advanced + /// either), so one broken counter reading cannot poison the baseline. pub fn update(&mut self, sample: f64) { + if !sample.is_finite() { + return; + } let now = Instant::now(); - let elapsed = now.duration_since(self.last).as_secs_f64(); - let steps = elapsed.max(1.0); + let steps = now.duration_since(self.last).as_secs_f64().max(1.0); let weight = (1.0 - self.alpha).powf(steps); self.value = self.value * weight + sample * (1.0 - weight); self.last = now; } + /// Replaces the average with `value` and restarts the step timer. + /// + /// Used to prime a baseline from the first observed window instead of + /// climbing from zero, which would otherwise read as an anomaly. + pub fn seed(&mut self, value: f64) { + if !value.is_finite() { + return; + } + self.value = value; + self.last = Instant::now(); + } + + /// Current smoothed value. + #[must_use] pub fn value(&self) -> f64 { self.value } } -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_ewma_initial() { - let ewma = Ewma::new(0.125); - assert_eq!(ewma.value(), 0.0); - } - - #[test] - fn test_ewma_update() { - let mut ewma = Ewma::new(1.0); - ewma.update(100.0); - assert!((ewma.value() - 100.0).abs() < 1.0); - } - - #[test] - fn test_ewma_convergence() { - let mut ewma = Ewma::new(0.5); - for _ in 0..10 { - ewma.update(50.0); - } - assert!((ewma.value() - 50.0).abs() < 1.0); +fn sanitize_alpha(alpha: f64) -> f64 { + if alpha.is_finite() && alpha > 0.0 { + alpha.min(1.0) + } else { + DEFAULT_ALPHA } } diff --git a/src/traffic/intel/reputation.rs b/src/traffic/intel/reputation.rs index 554040c..570e5e4 100644 --- a/src/traffic/intel/reputation.rs +++ b/src/traffic/intel/reputation.rs @@ -1,9 +1,79 @@ -use dashmap::DashMap; -use std::net::IpAddr; -use std::sync::Arc; +//! Per-source reputation with time decay and idle-TTL eviction. +//! +//! A source starts neutral (`0`); [`IpReputation::record_bad`] subtracts ten +//! points and [`IpReputation::record_good`] adds one, clamped to `[-100, 100]`. +//! Scores decay toward neutral on read (half of the magnitude per thirty +//! minutes of inactivity), so one old success cannot keep a source trusted +//! forever and one old failure cannot keep it toxic. +//! +//! State lives in sixty-four locked shards. [`IpReputation::sweep`] visits +//! eight of them per call and drops entries idle for longer than ten minutes; +//! recording advances one step per minute, so the table is reclaimed without +//! an external scheduler and no full-map `retain`. +//! +//! [`IpReputation::offenders_below`] returns at most 64 of the worst sources, +//! so one escalation cannot flood the ban list or the log. +use std::collections::hash_map::RandomState; +use std::collections::{BinaryHeap, HashMap}; +use std::hash::BuildHasher; +use std::net::IpAddr; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Mutex, MutexGuard}; +use std::time::{Duration, Instant}; + +/// Idle age after which a source is forgotten. +const IDLE_TTL: Duration = Duration::from_secs(600); +/// Time after which a dead source's score is halved. +const DECAY_HALF_LIFE: Duration = Duration::from_secs(1800); +/// Minimum interval between two automatic sweep steps. +const SWEEP_INTERVAL: Duration = Duration::from_secs(60); +/// Number of independently locked score shards. +const SHARDS: usize = 64; +/// Shards examined by one [`IpReputation::sweep`] call. +const SWEEP_SHARDS_PER_TICK: usize = 8; +/// Upper bound of a single offender batch. +const MAX_OFFENDERS_PER_CYCLE: usize = 64; +const GOOD_STEP: i32 = 1; +const BAD_STEP: i32 = -10; +const MAX_SCORE: i32 = 100; +const MIN_SCORE: i32 = -100; + +/// Tunables of [`IpReputation`]. +#[derive(Debug, Clone, Copy)] +pub struct ReputationPolicy { + /// Idle age after which a source is forgotten. + pub idle_ttl: Duration, + /// Time after which a dead source's score is halved. + pub decay_half_life: Duration, + /// Upper bound of a single [`IpReputation::offenders_below`] batch. + pub max_offenders: usize, +} + +impl Default for ReputationPolicy { + fn default() -> Self { + Self { + idle_ttl: IDLE_TTL, + decay_half_life: DECAY_HALF_LIFE, + max_offenders: MAX_OFFENDERS_PER_CYCLE, + } + } +} + +/// Reputation entry: the raw score and when it was last touched. +struct Entry { + score: i32, + last_update: Instant, +} + +/// Reputation scores of the sources seen by the edge node. pub struct IpReputation { - scores: Arc>, + shards: Box<[Mutex>]>, + hasher: RandomState, + policy: ReputationPolicy, + epoch: Instant, + last_sweep: AtomicU64, + sweep_cursor: AtomicUsize, } impl Default for IpReputation { @@ -13,129 +83,166 @@ impl Default for IpReputation { } impl IpReputation { + /// Builds a reputation table with the production policy. + #[must_use] pub fn new() -> Self { + Self::with_policy(ReputationPolicy::default()) + } + + /// Builds a reputation table with an explicit policy. + pub fn with_policy(policy: ReputationPolicy) -> Self { Self { - scores: Arc::new(DashMap::new()), + shards: (0..SHARDS).map(|_| Mutex::new(HashMap::new())).collect(), + hasher: RandomState::new(), + policy, + epoch: Instant::now(), + last_sweep: AtomicU64::new(0), + sweep_cursor: AtomicUsize::new(0), } } + /// Records a successful connection. pub fn record_good(&self, ip: IpAddr) { - let mut entry = self.scores.entry(ip).or_insert(0); - *entry = (*entry + 1).min(100); + self.sweep_if_due(); + self.adjust(ip, GOOD_STEP); } + /// Records a failed or abusive connection. pub fn record_bad(&self, ip: IpAddr) { - let mut entry = self.scores.entry(ip).or_insert(0); - *entry = (*entry - 10).max(-100); + self.sweep_if_due(); + self.adjust(ip, BAD_STEP); } + /// Current decayed score of `ip`; `0` for unknown or idle sources. pub fn score(&self, ip: IpAddr) -> i32 { - self.scores.get(&ip).map(|v| *v).unwrap_or(0) + let now = Instant::now(); + let index = self.shard_index(ip); + let shard = self.lock(index); + let Some(entry) = shard.get(&ip) else { return 0 }; + let idle = now.duration_since(entry.last_update); + if idle > self.policy.idle_ttl { + return 0; + } + decayed_score(entry.score, idle, self.policy.decay_half_life) } - /// Returns the sources scored at or below `threshold` (the offenders). + /// Returns up to [`ReputationPolicy::max_offenders`] of the worst sources + /// scored at or below `threshold`. /// - /// The result is a snapshot copy, so it is safe to ban from it while other - /// threads keep recording events. + /// The snapshot is safe to ban from while other threads keep recording + /// events. When more offenders exist than the batch cap, exactly one + /// aggregated warning carries the total count. pub fn offenders_below(&self, threshold: i32) -> Vec { - self.scores - .iter() - .filter(|entry| *entry.value() <= threshold) - .map(|entry| *entry.key()) - .collect() + let now = Instant::now(); + let limit = self.policy.max_offenders; + let mut worst: BinaryHeap<(i32, IpAddr)> = BinaryHeap::new(); + let mut found = 0usize; + for index in 0..self.shards.len() { + let shard = self.lock(index); + for (ip, entry) in shard.iter() { + let idle = now.duration_since(entry.last_update); + if idle > self.policy.idle_ttl { + continue; + } + let score = decayed_score(entry.score, idle, self.policy.decay_half_life); + if score > threshold { + continue; + } + found += 1; + worst.push((score, *ip)); + if worst.len() > limit { + worst.pop(); + } + } + } + let offenders: Vec = worst.into_iter().map(|(_, ip)| ip).collect(); + if found > offenders.len() { + tracing::warn!( + found, + returned = offenders.len(), + "reputation offender batch truncated for this cycle" + ); + } + offenders } + /// Whether the source is above the trusted score. pub fn is_trusted(&self, ip: IpAddr) -> bool { self.score(ip) > 50 } + + /// Number of live entries. + pub fn len(&self) -> usize { + (0..self.shards.len()).map(|index| self.lock(index).len()).sum() + } + + /// Whether the table is empty. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Evicts idle entries from the next shard slice; returns the count. + pub fn sweep(&self) -> usize { + let now = Instant::now(); + let shard_count = self.shards.len(); + let start = self.sweep_cursor.fetch_add(SWEEP_SHARDS_PER_TICK, Ordering::Relaxed); + let mut removed = 0; + for offset in 0..SWEEP_SHARDS_PER_TICK.min(shard_count) { + let index = (start + offset) % shard_count; + let mut shard = self.lock(index); + let before = shard.len(); + shard.retain(|_, entry| now.duration_since(entry.last_update) <= self.policy.idle_ttl); + removed += before - shard.len(); + } + removed + } + + fn adjust(&self, ip: IpAddr, delta: i32) { + let now = Instant::now(); + let index = self.shard_index(ip); + let mut shard = self.lock(index); + let entry = shard.entry(ip).or_insert(Entry { + score: 0, + last_update: now, + }); + let idle = now.duration_since(entry.last_update); + let decayed = decayed_score(entry.score, idle, self.policy.decay_half_life); + entry.score = (decayed + delta).clamp(MIN_SCORE, MAX_SCORE); + entry.last_update = now; + } + + /// Advances one sweep step when [`SWEEP_INTERVAL`] has elapsed, so eviction + /// does not depend on an external timer. + fn sweep_if_due(&self) { + let elapsed = self.epoch.elapsed().as_secs(); + let last = self.last_sweep.load(Ordering::Relaxed); + if elapsed.saturating_sub(last) < SWEEP_INTERVAL.as_secs() { + return; + } + let advanced = self + .last_sweep + .compare_exchange(last, elapsed, Ordering::AcqRel, Ordering::Relaxed); + if advanced.is_ok() { + self.sweep(); + } + } + + fn shard_index(&self, ip: IpAddr) -> usize { + (self.hasher.hash_one(ip) as usize) % self.shards.len() + } + + fn lock(&self, index: usize) -> MutexGuard<'_, HashMap> { + // A poisoned shard still holds consistent scores; keep serving. + self.shards[index].lock().unwrap_or_else(|poison| poison.into_inner()) + } } -#[cfg(test)] -mod tests { - use super::*; - use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; - - fn v4(octets: [u8; 4]) -> IpAddr { - IpAddr::V4(Ipv4Addr::from(octets)) - } - - #[test] - fn test_reputation_initial_score() { - let rep = IpReputation::new(); - assert_eq!(rep.score(v4([192, 168, 1, 1])), 0); - } - - #[test] - fn test_reputation_good() { - let rep = IpReputation::new(); - let ip = v4([10, 0, 0, 1]); - rep.record_good(ip); - assert_eq!(rep.score(ip), 1); - } - - #[test] - fn test_reputation_bad() { - let rep = IpReputation::new(); - let ip = v4([10, 0, 0, 2]); - rep.record_bad(ip); - assert_eq!(rep.score(ip), -10); - } - - #[test] - fn test_reputation_cap_positive() { - let rep = IpReputation::new(); - let ip = v4([10, 0, 0, 3]); - for _ in 0..200 { - rep.record_good(ip); - } - assert_eq!(rep.score(ip), 100); - } - - #[test] - fn test_reputation_cap_negative() { - let rep = IpReputation::new(); - let ip = v4([10, 0, 0, 4]); - for _ in 0..20 { - rep.record_bad(ip); - } - assert_eq!(rep.score(ip), -100); - } - - #[test] - fn test_is_trusted() { - let rep = IpReputation::new(); - let ip = v4([10, 0, 0, 5]); - assert!(!rep.is_trusted(ip)); - for _ in 0..51 { - rep.record_good(ip); - } - assert!(rep.is_trusted(ip)); - } - - #[test] - fn test_reputation_v6() { - let rep = IpReputation::new(); - let ip = IpAddr::V6(Ipv6Addr::LOCALHOST); - rep.record_bad(ip); - assert_eq!(rep.score(ip), -10); - } - - #[test] - fn test_offenders_below_threshold() { - let rep = IpReputation::new(); - let bad = v4([10, 0, 0, 1]); - let good = v4([10, 0, 0, 2]); - for _ in 0..6 { - rep.record_bad(bad); - } - rep.record_good(good); - let offenders = rep.offenders_below(-50); - assert_eq!(offenders, vec![bad]); - } - - #[test] - fn test_offenders_empty_when_none_below() { - let rep = IpReputation::new(); - assert!(rep.offenders_below(-50).is_empty()); +fn decayed_score(score: i32, idle: Duration, half_life: Duration) -> i32 { + if score == 0 || half_life.is_zero() { + return 0; } + let factor = 0.5f64.powf(idle.as_secs_f64() / half_life.as_secs_f64()); + // Rounding (instead of truncation) keeps an immediate second update from + // decaying a score of `1` back to zero on the microsecond scale. + (f64::from(score) * factor).round() as i32 } diff --git a/src/traffic/prefix/key.rs b/src/traffic/prefix/key.rs index 13dc9eb..f8055df 100644 --- a/src/traffic/prefix/key.rs +++ b/src/traffic/prefix/key.rs @@ -17,17 +17,22 @@ pub struct PrefixKey { impl PrefixKey { /// Builds the prefix key for an address (v4 maps to /24 in the first four /// bytes, v6 maps to /64). + /// + /// The address is normalised through [`prefix_of`] first, so an + /// IPv4-mapped IPv6 source (`::ffff:a.b.c.d`) produces exactly the same key + /// as the plain IPv4 address. XDP `prefix_stats` entries and the user-space + /// tracker therefore agree on the bucket of a dual-stack client. #[must_use] pub fn new(ip: IpAddr) -> Self { let mut key = Self::default(); - match ip { + match prefix_of(ip) { IpAddr::V4(v4) => { key.family = PREFIX_FAMILY_V4; - key.addr[..4].copy_from_slice(&masked_v4(v4).octets()); + key.addr[..4].copy_from_slice(&v4.octets()); }, IpAddr::V6(v6) => { key.family = PREFIX_FAMILY_V6; - key.addr.copy_from_slice(&masked_v6_segments(v6)); + key.addr.copy_from_slice(&v6.octets()); }, } key diff --git a/tests/app_shutdown.rs b/tests/app_shutdown.rs new file mode 100644 index 0000000..97b54cb --- /dev/null +++ b/tests/app_shutdown.rs @@ -0,0 +1,15 @@ +//! Edge daemon exit codes: a forced drain timeout must not look like success. + +use rampart::app::RunExit; + +#[test] +fn clean_shutdown_exits_zero() { + assert_eq!(RunExit::Ok.code(), 0); +} + +#[test] +fn forced_drain_timeout_exits_nonzero() { + // Connections were cut before finishing: supervisors and scripts must see a + // failure, not a clean exit. + assert_eq!(RunExit::DrainTimeout.code(), 1); +} diff --git a/tests/clickhouse_writer.rs b/tests/clickhouse_writer.rs new file mode 100644 index 0000000..0ce466b --- /dev/null +++ b/tests/clickhouse_writer.rs @@ -0,0 +1,81 @@ +//! Integration tests for the ClickHouse sink: buffering, bounded flush and +//! retry on failure. + +use std::time::{Duration, Instant}; + +use rampart::store::clickhouse::{ClickHouseEvent, ClickHouseWriter, MAX_BUFFER_EVENTS}; + +fn event(seq: usize) -> ClickHouseEvent { + ClickHouseEvent { + timestamp: chrono::Utc::now(), + event_type: "test".to_string(), + ip: format!("192.0.2.{}", seq % 250), + data_float: 0.0, + data_int: 0, + data_string: String::new(), + } +} + +#[tokio::test] +async fn push_only_buffers_events() { + let mut writer = ClickHouseWriter::new("http://127.0.0.1:1"); + writer.push(event(0)).await.expect("push must succeed"); + + assert_eq!(writer.pending(), 1, "push must not flush eagerly"); + assert_eq!(writer.dropped(), 0); +} + +#[tokio::test] +async fn empty_flush_is_a_noop() { + let mut writer = ClickHouseWriter::new("http://127.0.0.1:1"); + writer.flush().await.expect("empty flush must not touch the network"); +} + +#[tokio::test] +async fn failed_flush_requeues_the_batch() { + let mut writer = ClickHouseWriter::new("http://127.0.0.1:1"); + writer.push(event(1)).await.expect("push must succeed"); + + let result = writer.flush().await; + + assert!(result.is_err(), "connection refused must surface as an error"); + assert_eq!(writer.pending(), 1, "failed batch must be re-queued"); + assert_eq!(writer.dropped(), 0, "a single failed batch is not a drop"); +} + +#[tokio::test] +async fn stalled_server_flush_is_bounded() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let addr = listener.local_addr().expect("local addr"); + let server = tokio::spawn(async move { + let (_stream, _) = listener.accept().await.expect("accept"); + // Keep the connection open without ever answering. + std::future::pending::<()>().await; + }); + + let mut writer = ClickHouseWriter::new(&format!("http://{addr}/")); + writer.push(event(2)).await.expect("push must succeed"); + + let started = Instant::now(); + let result = writer.flush().await; + let elapsed = started.elapsed(); + server.abort(); + + assert!(result.is_err(), "a stalled server must fail the flush"); + assert!( + elapsed < Duration::from_secs(5), + "flush must be bounded, took {elapsed:?}" + ); + assert_eq!(writer.pending(), 1, "timed-out batch must be re-queued"); +} + +#[tokio::test] +async fn buffer_is_bounded_and_overflow_is_counted() { + let mut writer = ClickHouseWriter::new("http://127.0.0.1:1"); + for seq in 0..MAX_BUFFER_EVENTS + 7 { + writer.push(event(seq)).await.expect("push must succeed"); + } + + assert_eq!(writer.pending(), MAX_BUFFER_EVENTS, "buffer must stay bounded"); + assert_eq!(writer.dropped(), 7, "overflow must be counted"); +} diff --git a/tests/config_parse.rs b/tests/config_parse.rs index 230c25d..74b2b4a 100644 --- a/tests/config_parse.rs +++ b/tests/config_parse.rs @@ -142,3 +142,127 @@ events_enabled = false assert!(!xdp.throttle_enabled); assert!(!xdp.events_enabled); } + +#[test] +fn prefix_detect_defaults_disabled() { + let config = Config::parse_str("").expect("empty config must parse"); + assert!(!config.detect.prefix.enabled); + assert_eq!(config.detect.prefix.syn_threshold, 500); + assert_eq!(config.detect.prefix.window_secs, 10); + assert_eq!(config.detect.prefix.min_unique_sources, 16); +} + +#[test] +fn prefix_detect_section_parses() { + let config = Config::parse_str( + r#" +[detect.prefix] +enabled = true +syn_threshold = 1000 +window_secs = 5 +min_unique_sources = 8 +"#, + ) + .expect("detect.prefix section must parse"); + assert!(config.detect.prefix.enabled); + assert_eq!(config.detect.prefix.syn_threshold, 1000); + assert_eq!(config.detect.prefix.window_secs, 5); + assert_eq!(config.detect.prefix.min_unique_sources, 8); +} + +#[test] +fn prefix_detect_zero_threshold_rejected_when_enabled() { + let err = Config::parse_str("[detect.prefix]\nenabled = true\nsyn_threshold = 0\n") + .expect_err("zero threshold must be rejected"); + assert!(err.to_string().contains("detect.prefix.syn_threshold")); +} + +#[test] +fn syn_challenge_requires_a_secret() { + let err = Config::parse_str("[xdp]\nsyn_challenge_enabled = true\n") + .expect_err("missing challenge secret must be rejected"); + let message = err.to_string(); + assert!(message.contains("challenge_secret_hex")); + assert!(message.contains("openssl rand -hex 8")); +} + +#[test] +fn syn_challenge_rejects_every_placeholder_spelling() { + for secret in [ + "a5a5a5a5a5a5a5a5", + "A5A5A5A5A5A5A5A5", + "0xA5A5A5A5A5A5A5A5", + "a5a5-5a5a-5a5a-a5a5", + "a5a55a5a5a5aa5a5", + "A5A5_5A5A_5A5A_A5A5", + "0xa5a55a5a5a5aa5a5", + ] { + let toml = format!("[xdp]\nsyn_challenge_enabled = true\nchallenge_secret_hex = \"{secret}\"\n"); + let err = Config::parse_str(&toml).expect_err("placeholder secret must be rejected"); + assert!( + err.to_string().contains("placeholder"), + "secret '{secret}' must be rejected" + ); + } +} + +#[test] +fn syn_challenge_accepts_a_generated_secret() { + let config = + Config::parse_str("[xdp]\nsyn_challenge_enabled = true\nchallenge_secret_hex = \"9f3c71a0b2d84e65\"\n") + .expect("random challenge secret must be accepted"); + assert_eq!(config.xdp.challenge_secret_hex.as_deref(), Some("9f3c71a0b2d84e65")); +} + +#[test] +fn rejects_non_positive_rate_limit() { + for pps in ["0.0", "-1.0", "nan", "inf"] { + let toml = format!("[limits]\nrate_limit_pps = {pps}\n"); + let err = Config::parse_str(&toml).expect_err("rate_limit_pps must be positive and finite"); + assert!( + err.to_string().contains("limits.rate_limit_pps"), + "pps '{pps}' must be rejected" + ); + } +} + +#[test] +fn rejects_rate_limit_burst_below_one() { + for burst in ["0.0", "0.5", "nan"] { + let toml = format!("[limits]\nrate_limit_burst = {burst}\n"); + let err = Config::parse_str(&toml).expect_err("rate_limit_burst must be at least 1"); + assert!( + err.to_string().contains("limits.rate_limit_burst"), + "burst '{burst}' must be rejected" + ); + } +} + +#[test] +fn rejects_out_of_range_ban_duration() { + let err = Config::parse_str("[ban]\nban_duration_secs = 0\n").expect_err("zero ban duration must be rejected"); + assert!(err.to_string().contains("ban.ban_duration_secs")); + let err = Config::parse_str("[ban]\nban_duration_secs = 31536001\n") + .expect_err("ban duration above one year must be rejected"); + assert!(err.to_string().contains("ban.ban_duration_secs")); + assert!(Config::parse_str("[ban]\nban_duration_secs = 31536000\n").is_ok()); +} + +#[test] +fn rejects_out_of_range_pow_difficulty() { + for difficulty in ["0", "11"] { + let toml = format!("[pow]\nenabled = true\ndifficulty = {difficulty}\n"); + let err = Config::parse_str(&toml).expect_err("pow.difficulty must stay in 1..=10"); + assert!( + err.to_string().contains("pow.difficulty"), + "difficulty '{difficulty}' must be rejected" + ); + } + assert!(Config::parse_str("[pow]\ndifficulty = 10\n").is_ok()); +} + +#[test] +fn rejects_zero_workers() { + let err = Config::parse_str("[workers]\ncount = 0\n").expect_err("zero workers must be rejected"); + assert!(err.to_string().contains("workers.count")); +} diff --git a/tests/connection_limits.rs b/tests/connection_limits.rs new file mode 100644 index 0000000..33601dd --- /dev/null +++ b/tests/connection_limits.rs @@ -0,0 +1,313 @@ +//! Connection-path limits: filter order around the PoW gate, the whitelist +//! exemption from bans, accept-loop backoff and the idle relay timeout. + +use rampart::config::Config; +use rampart::engine::challenge::DifficultyAdjuster; +use rampart::engine::listener::AcceptBackoff; +use rampart::engine::tunnel::Gateway; +use rampart::filter::blacklist::Blacklist; +use rampart::filter::rate_limit::RateLimiter; +use rampart::protocol::ProtocolRegistry; +use rampart::traffic::reputation::IpReputation; +use std::collections::HashSet; +use std::io; +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::atomic::AtomicU64; +use std::time::Duration; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::{TcpListener, TcpStream}; + +/// Linux `EMFILE` (too many open files), the errno behind the accept spin. +#[cfg(target_os = "linux")] +const EMFILE: i32 = 24; + +/// Builds a gateway with the cheap collaborators and no XDP/ClickHouse/protocol +/// handler, so every stage past the filters fails fast. +fn test_gateway(config: Config, whitelist: HashSet) -> Arc { + Arc::new(Gateway::new( + Arc::new(config.clone()), + Arc::new(RateLimiter::new( + config.limits.rate_limit_pps, + config.limits.rate_limit_burst, + )), + Arc::new(Blacklist::new()), + Arc::new(Mutex::new(DifficultyAdjuster::default())), + Arc::new(whitelist), + Arc::new(IpReputation::new()), + None, + None, + Arc::new(AtomicU64::new(0)), + Arc::new(ProtocolRegistry::new()), + None, + )) +} + +/// Returns a connected client/server pair over loopback. +async fn loopback() -> (TcpStream, TcpStream) { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind loopback"); + let addr = listener.local_addr().expect("local addr"); + let (client, accepted) = tokio::join!(TcpStream::connect(addr), listener.accept()); + (client.expect("connect"), accepted.expect("accept").0) +} + +/// Reads one newline-terminated line without blocking past `limit`. +async fn read_line(stream: &mut TcpStream, limit: Duration) -> io::Result> { + let mut line = Vec::new(); + let mut byte = [0u8; 1]; + loop { + let read = tokio::time::timeout(limit, stream.read(&mut byte)).await??; + if read == 0 || byte[0] == b'\n' { + return Ok(line); + } + line.push(byte[0]); + } +} + +/// A rate-limited client must be rejected before the PoW exchange starts, so +/// no challenge is written and the task never waits for a solver. +#[tokio::test] +async fn rate_limit_runs_before_pow() { + let mut config = Config::default(); + config.limits.rate_limit_pps = 0.0; + config.limits.rate_limit_burst = 0.0; + config.pow.enabled = true; + config.pow.difficulty = 1; + let gateway = test_gateway(config, HashSet::new()); + + let (mut client, server) = loopback().await; + let peer = SocketAddr::from(([203, 0, 113, 7], 51_000)); + let started = tokio::time::Instant::now(); + let outcome = gateway.handle(server, peer).await; + assert!(outcome.result.is_ok(), "a rate-limited connection is dropped silently"); + assert!( + started.elapsed() < Duration::from_millis(500), + "rate limit must short-circuit the PoW gate" + ); + + let mut scratch = [0u8; 64]; + let read = tokio::time::timeout(Duration::from_secs(1), client.read(&mut scratch)) + .await + .expect("read must not hang") + .expect("read must succeed"); + assert_eq!(read, 0, "a rate-limited client must not receive a PoW challenge"); +} + +/// A client that never answers PoW must still burn its own token, so the next +/// attempt from the same source is rate limited instead of parked in PoW. +#[tokio::test] +async fn silent_pow_client_exhausts_budget_for_next_attempt() { + let mut config = Config::default(); + config.limits.rate_limit_pps = 0.0; + config.limits.rate_limit_burst = 1.0; + config.pow.enabled = true; + config.pow.difficulty = 1; + let gateway = test_gateway(config, HashSet::new()); + let peer = SocketAddr::from(([203, 0, 113, 8], 51_001)); + + let (mut first_client, first_server) = loopback().await; + let first_gateway = Arc::clone(&gateway); + let first = tokio::spawn(async move { first_gateway.handle(first_server, peer).await.result }); + + let challenge = read_line(&mut first_client, Duration::from_secs(2)) + .await + .expect("first attempt must reach the PoW gate"); + assert!(!challenge.is_empty(), "PoW gate issues a challenge line"); + + let (mut second_client, second_server) = loopback().await; + let started = tokio::time::Instant::now(); + let outcome = gateway.handle(second_server, peer).await; + assert!(outcome.result.is_ok()); + assert!( + started.elapsed() < Duration::from_millis(500), + "second attempt must be rejected by the rate limiter, not queued behind PoW" + ); + let mut scratch = [0u8; 1]; + let read = second_client.read(&mut scratch).await.expect("read must succeed"); + assert_eq!(read, 0, "second attempt is closed without a challenge"); + + first.abort(); +} + +/// A whitelisted source stays exempt from the rate limiter and is never +/// auto-banned, even after enough abuse to drive reputation below the ban +/// threshold, and an existing ban entry is ignored for it. +#[tokio::test] +async fn whitelisted_peer_is_exempt_from_rate_limit_and_ban() { + let mut config = Config::default(); + config.limits.rate_limit_pps = 0.0; + config.limits.rate_limit_burst = 0.0; + let trusted = IpAddr::V4(Ipv4Addr::new(198, 51, 100, 10)); + let gateway = test_gateway(config, HashSet::from([trusted])); + let peer = SocketAddr::new(trusted, 51_002); + + for _ in 0..8 { + let (_client, server) = loopback().await; + let outcome = gateway.handle(server, peer).await; + assert!( + outcome.result.is_err(), + "a trusted peer must pass every filter and reach the (empty) registry" + ); + } + assert!( + !gateway.blacklist.is_blocked(trusted), + "a trusted peer must never be banned" + ); + assert_eq!( + gateway.reputation.score(trusted), + 0, + "trusted traffic is not scored as abusive" + ); + + gateway + .blacklist + .add(trusted, Duration::from_secs(60), "stale-test-ban"); + let (_client, server) = loopback().await; + assert!( + gateway.handle(server, peer).await.result.is_err(), + "the whitelist overrides an existing blacklist entry" + ); +} + +/// The untrusted counterpart: repeated rate-limit hits drive reputation below +/// the threshold and the peer is banned. +#[tokio::test] +async fn repeated_rate_limit_hits_auto_ban_untrusted_peer() { + let mut config = Config::default(); + config.limits.rate_limit_pps = 0.0; + config.limits.rate_limit_burst = 0.0; + let gateway = test_gateway(config, HashSet::new()); + let peer_ip = IpAddr::V4(Ipv4Addr::new(203, 0, 113, 9)); + let peer = SocketAddr::new(peer_ip, 51_003); + + for _ in 0..5 { + let (_client, server) = loopback().await; + assert!( + gateway.handle(server, peer).await.result.is_ok(), + "a rate-limited connection is dropped silently" + ); + } + assert!( + gateway.blacklist.is_blocked(peer_ip), + "reputation below -40 must ban the peer" + ); +} + +/// Backoff doubles from 50ms to a 500ms cap and restarts after a successful +/// accept. +#[cfg(target_os = "linux")] +#[test] +fn accept_backoff_grows_caps_and_resets() { + let mut backoff = AcceptBackoff::new(); + let error = io::Error::from_raw_os_error(EMFILE); + let expected = [50u64, 100, 200, 400, 500, 500]; + for milliseconds in expected { + assert_eq!(backoff.delay(&error), Some(Duration::from_millis(milliseconds))); + } + backoff.reset(); + assert_eq!( + backoff.delay(&error), + Some(Duration::from_millis(50)), + "reset after a successful accept restarts at the minimum" + ); +} + +/// Peer-abort errors need no delay: retrying immediately is correct there. +#[test] +fn accept_backoff_skips_non_transient_errors() { + let mut backoff = AcceptBackoff::new(); + assert_eq!(backoff.delay(&io::Error::from(io::ErrorKind::ConnectionAborted)), None); + assert_eq!(backoff.delay(&io::Error::from(io::ErrorKind::InvalidInput)), None); +} + +/// Error logging is rate limited: first failure and then one per window. +#[cfg(target_os = "linux")] +#[test] +fn accept_backoff_rate_limits_error_logs() { + let mut backoff = AcceptBackoff::new(); + let error = io::Error::from_raw_os_error(EMFILE); + assert!(backoff.delay(&error).is_some()); + assert!( + backoff.should_log(), + "the first failure is always logged at error level" + ); + assert!(backoff.delay(&error).is_some()); + assert!(!backoff.should_log(), "repeated failures stay at debug level"); + for _ in 2..32 { + assert!(backoff.delay(&error).is_some()); + } + assert!(backoff.should_log(), "a periodic reminder keeps the condition visible"); +} + +/// The loop must spend real time sleeping instead of spinning. `tokio`'s +/// `test-util` feature is not compiled in, so this uses the real minimum +/// backoff (50ms) with a short iteration count. +#[cfg(target_os = "linux")] +#[tokio::test] +async fn accept_backoff_sleeps_instead_of_spinning() { + let mut backoff = AcceptBackoff::new(); + let error = io::Error::from_raw_os_error(EMFILE); + let started = tokio::time::Instant::now(); + let mut slept = Duration::ZERO; + for _ in 0..3 { + let delay = backoff.delay(&error).expect("transient error must back off"); + assert!(!delay.is_zero(), "a failed accept must yield a non-zero delay"); + tokio::time::sleep(delay).await; + slept += delay; + } + assert!(slept >= Duration::from_millis(350), "50+100+200ms of mandatory backoff"); + assert!( + started.elapsed() >= slept, + "the loop cannot advance faster than its backoff" + ); +} + +/// A silent pair is closed by the relay idle timeout instead of pinning two +/// descriptors forever. +#[tokio::test] +async fn relay_idle_timeout_closes_silent_pair() { + let gateway = test_gateway(Config::default(), HashSet::new()); + let (_client_peer, gateway_side) = loopback().await; + let (backend_gateway_side, _backend_peer) = loopback().await; + + let idle = Duration::from_millis(60); + let started = tokio::time::Instant::now(); + let result = tokio::time::timeout( + Duration::from_secs(2), + gateway.relay_with_idle(gateway_side, backend_gateway_side, idle), + ) + .await + .expect("relay must be cut by the idle timeout, not hang"); + assert!(result.is_err(), "silent relay must be cut by the idle timeout"); + assert!(started.elapsed() >= idle, "the idle budget must actually elapse"); +} + +/// The idle budget must not break normal forwarding in either direction. +#[tokio::test] +async fn relay_forwards_bytes_in_both_directions() { + let gateway = test_gateway(Config::default(), HashSet::new()); + let (mut client_peer, client_side) = loopback().await; + let (backend_side, mut backend_peer) = loopback().await; + + let relay = tokio::spawn(async move { + gateway + .relay_with_idle(client_side, backend_side, Duration::from_secs(5)) + .await + }); + + client_peer.write_all(b"ping").await.expect("write to relay"); + let mut buffer = [0u8; 4]; + backend_peer.read_exact(&mut buffer).await.expect("backend read"); + assert_eq!(&buffer, b"ping"); + + backend_peer.write_all(b"pong").await.expect("write from backend"); + client_peer.read_exact(&mut buffer).await.expect("client read"); + assert_eq!(&buffer, b"pong"); + + drop(client_peer); + assert!( + relay.await.expect("relay task").is_ok(), + "clean half-close is not an error" + ); +} diff --git a/tests/filter_logic.rs b/tests/filter_logic.rs index 59f6df3..2180439 100644 --- a/tests/filter_logic.rs +++ b/tests/filter_logic.rs @@ -57,3 +57,201 @@ fn blacklist_remove_clears_ban() { blacklist.remove(ip(7)); assert!(!blacklist.is_blocked(ip(7))); } + +#[test] +fn blacklist_expired_cleanup_keeps_fresh_reban() { + let blacklist = Blacklist::new(); + blacklist.add(ip(8), Duration::ZERO, "stale"); + assert!(!blacklist.is_blocked(ip(8)), "zero-duration ban is already expired"); + + blacklist.add(ip(8), Duration::from_secs(60), "fresh"); + assert!( + blacklist.is_blocked(ip(8)), + "expired-entry cleanup must not delete a fresh ban" + ); + assert_eq!(blacklist.len(), 1); +} + +#[test] +fn blacklist_expired_cleanup_does_not_race_with_reban() { + let blacklist = std::sync::Arc::new(Blacklist::new()); + let reader = blacklist.clone(); + let handle = std::thread::spawn(move || { + for _ in 0..2000 { + let _ = reader.is_blocked(ip(9)); + std::thread::yield_now(); + } + }); + for _ in 0..2000 { + blacklist.add(ip(9), Duration::ZERO, "stale"); + blacklist.add(ip(9), Duration::from_secs(60), "fresh"); + } + handle.join().expect("reader thread must not panic"); + assert!( + blacklist.is_blocked(ip(9)), + "a concurrently refreshed ban must survive stale cleanup" + ); +} + +#[test] +fn blacklist_clamps_huge_durations_without_panicking() { + let blacklist = Blacklist::new(); + blacklist.add(ip(10), Duration::from_secs(u64::MAX), "hostile"); + blacklist.add_redis(ip(11), Duration::from_secs(u64::MAX), "hostile"); + + assert!(blacklist.is_blocked(ip(10)), "local ban must be clamped, not dropped"); + assert!(blacklist.is_blocked(ip(11)), "redis ban must be clamped, not dropped"); +} + +#[test] +fn replace_redis_keeps_new_bans_and_local_entries() { + let blacklist = Blacklist::new(); + blacklist.add(ip(12), Duration::from_secs(60), "local"); + blacklist.add_redis(ip(13), Duration::from_secs(60), "stale-upstream"); + + blacklist.replace_redis([ + (ip(14), Duration::from_secs(60), "upstream".to_string()), + (ip(15), Duration::from_secs(60), "upstream".to_string()), + ]); + + assert!(blacklist.is_blocked(ip(14)), "new upstream ban must apply immediately"); + assert!(blacklist.is_blocked(ip(15)), "new upstream ban must apply immediately"); + assert!(!blacklist.is_blocked(ip(13)), "ban removed upstream must stop applying"); + assert!( + blacklist.is_blocked(ip(12)), + "local bans must survive a Redis reconcile" + ); +} + +#[test] +fn replace_redis_can_clear_every_mirrored_ban() { + let blacklist = Blacklist::new(); + blacklist.add_redis(ip(16), Duration::from_secs(60), "stale-upstream"); + + blacklist.replace_redis(std::iter::empty()); + + assert!(!blacklist.is_blocked(ip(16))); + assert!(blacklist.is_empty()); +} + +fn numbered(index: u32) -> IpAddr { + IpAddr::V4(Ipv4Addr::from(0x0a00_0000 + index)) +} + +#[test] +fn rate_limiter_enforces_a_hard_bucket_cap() { + let limiter = RateLimiter::with_capacity(1.0, 1.0, 4, Duration::from_secs(600)); + let admitted = (0..100).filter(|index| limiter.check(numbered(*index))).count(); + assert_eq!(admitted, 4, "only max_buckets new sources may be admitted"); + assert_eq!(limiter.len(), 4); + assert!( + !limiter.check(numbered(500)), + "a new source fails closed once the table is saturated" + ); + assert_eq!(limiter.len(), 4, "a refused source must not allocate a bucket"); +} + +#[test] +fn rate_limiter_sweep_frees_capacity_and_keeps_active() { + let limiter = RateLimiter::with_capacity(1.0, 10.0, 2, Duration::from_millis(100)); + assert!(limiter.check(ip(1))); + assert!(limiter.check(ip(2))); + std::thread::sleep(Duration::from_millis(200)); + assert!(limiter.check(ip(2)), "the active source still has tokens"); + + for _ in 0..8 { + limiter.sweep(); + } + assert_eq!(limiter.len(), 1, "idle bucket evicted, active bucket kept"); + assert!(limiter.check(ip(3)), "eviction must free a slot for a new source"); +} + +#[test] +fn rate_limiter_sweep_is_incremental() { + let limiter = RateLimiter::with_capacity(1000.0, 1.0, 4096, Duration::from_millis(10)); + for index in 0..4096 { + assert!(limiter.check(numbered(index))); + } + std::thread::sleep(Duration::from_millis(20)); + + let first = limiter.sweep(); + assert!(first > 0, "the first sweep must reclaim idle buckets"); + assert!(first < 4096, "one sweep must not scan the whole table"); + assert!(!limiter.is_empty(), "a single sweep leaves work for the next tick"); + + for _ in 0..8 { + limiter.sweep(); + } + assert!(limiter.is_empty(), "a full shard cycle reclaims every idle bucket"); +} + +#[test] +fn cidr_ban_covers_the_whole_range() { + let blacklist = Blacklist::new(); + blacklist.add_cidr( + IpAddr::V4(Ipv4Addr::new(10, 1, 2, 3)), + 24, + Duration::from_secs(60), + "subnet", + ); + assert!(blacklist.is_blocked(IpAddr::V4(Ipv4Addr::new(10, 1, 2, 1)))); + assert!(blacklist.is_blocked(IpAddr::V4(Ipv4Addr::new(10, 1, 2, 255)))); + assert!(!blacklist.is_blocked(IpAddr::V4(Ipv4Addr::new(10, 1, 3, 1)))); +} + +#[test] +fn cidr_ban_normalises_host_bits_and_v4_mapped_addresses() { + let blacklist = Blacklist::new(); + blacklist.add_cidr( + IpAddr::V4(Ipv4Addr::new(10, 7, 9, 200)), + 24, + Duration::from_secs(60), + "subnet", + ); + let mapped: IpAddr = "::ffff:10.7.9.1".parse().expect("mapped ipv4"); + assert!(blacklist.is_blocked(mapped), "/24 ban must cover the mapped form too"); +} + +#[test] +fn cidr_ban_expires() { + let blacklist = Blacklist::new(); + blacklist.add_cidr( + IpAddr::V4(Ipv4Addr::new(10, 8, 0, 0)), + 16, + Duration::from_millis(1), + "subnet", + ); + std::thread::sleep(Duration::from_millis(5)); + assert!(!blacklist.is_blocked(IpAddr::V4(Ipv4Addr::new(10, 8, 0, 1)))); + blacklist.clear_expired(); + assert!(blacklist.is_empty()); +} + +#[test] +fn invalid_cidr_prefix_is_ignored() { + let blacklist = Blacklist::new(); + blacklist.add_cidr( + IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1)), + 0, + Duration::from_secs(60), + "subnet", + ); + blacklist.add_cidr( + IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1)), + 33, + Duration::from_secs(60), + "subnet", + ); + assert!(blacklist.is_empty(), "a /0 or /33 ban must not be installed"); +} + +#[test] +fn ipv6_cidr_ban_covers_the_range() { + let blacklist = Blacklist::new(); + let prefix: IpAddr = "fd00:1234:5678:9abc::".parse().expect("ipv6 prefix"); + blacklist.add_cidr(prefix, 64, Duration::from_secs(60), "subnet"); + let inside: IpAddr = "fd00:1234:5678:9abc::42".parse().expect("ipv6 inside"); + let outside: IpAddr = "fd00:1234:5678:9abd::1".parse().expect("ipv6 outside"); + assert!(blacklist.is_blocked(inside)); + assert!(!blacklist.is_blocked(outside)); +} diff --git a/tests/manager_security.rs b/tests/manager_security.rs new file mode 100644 index 0000000..2d9d3e6 --- /dev/null +++ b/tests/manager_security.rs @@ -0,0 +1,124 @@ +//! Management-plane security: API password/JWT policy and the CORS downgrade. +//! +//! Regression tests for the authentication bypass where an empty +//! `API_PASSWORD` matched an empty login (`verify_password("", "")`). + +use dashmap::DashMap; +use rampart::manager::api::auth::allow_login_attempt; +use rampart::manager::security::{CorsPolicy, validate_api_password, validate_jwt_secret, verify_password}; +use std::net::IpAddr; + +const STRONG_PASSWORD: &str = "correct-horse-battery-staple"; +const STRONG_SECRET: &str = "0123456789abcdef0123456789abcdef"; + +#[test] +fn rejects_empty_api_password() { + let err = validate_api_password("").expect_err("empty password must be rejected"); + assert!(err.to_string().contains("must not be empty")); +} + +#[test] +fn rejects_whitespace_only_api_password() { + assert!(validate_api_password(" ").is_err()); +} + +#[test] +fn rejects_short_api_password() { + let short = "a".repeat(15); + let err = validate_api_password(&short).expect_err("15 characters must be rejected"); + assert!(err.to_string().contains("at least 16")); + assert!(validate_api_password(&"a".repeat(16)).is_ok()); +} + +#[test] +fn rejects_placeholder_api_password() { + assert!(validate_api_password("changeme").is_err()); + assert!(validate_api_password("ChangeMe").is_err()); + assert!(validate_api_password(" changeme ").is_err()); +} + +#[test] +fn accepts_strong_api_password() { + assert!(validate_api_password(STRONG_PASSWORD).is_ok()); +} + +#[test] +fn empty_stored_password_never_verifies() { + assert!(!verify_password("", "")); + assert!(!verify_password("anything", "")); +} + +#[test] +fn verify_password_matches_exact_value_only() { + assert!(verify_password(STRONG_PASSWORD, STRONG_PASSWORD)); + assert!(!verify_password("wrong-password", STRONG_PASSWORD)); + assert!(!verify_password("short", STRONG_PASSWORD)); +} + +#[test] +fn rejects_empty_or_short_jwt_secret() { + assert!(validate_jwt_secret("").is_err()); + assert!(validate_jwt_secret(&" ".repeat(64)).is_err()); + assert!(validate_jwt_secret(&"a".repeat(31)).is_err()); + let err = validate_jwt_secret(" \t\n").expect_err("whitespace must be rejected"); + assert!(err.to_string().contains("non-whitespace")); +} + +#[test] +fn accepts_strong_jwt_secret() { + assert!(validate_jwt_secret(STRONG_SECRET).is_ok()); + assert!(validate_jwt_secret(&format!(" {STRONG_SECRET} ")).is_ok()); +} + +#[test] +fn cors_without_explicit_origin_is_denied() { + assert_eq!(CorsPolicy::parse(None).expect("unset is valid"), CorsPolicy::Deny); + assert_eq!(CorsPolicy::parse(Some("")).expect("empty is valid"), CorsPolicy::Deny); + assert_eq!( + CorsPolicy::parse(Some(" ")).expect("blank is valid"), + CorsPolicy::Deny + ); + assert!(CorsPolicy::Deny.cors_layer().is_none()); +} + +#[test] +fn cors_wildcard_requires_explicit_flag() { + assert_eq!( + CorsPolicy::parse(Some("*")).expect("wildcard is valid"), + CorsPolicy::Any + ); + assert!(CorsPolicy::Any.cors_layer().is_some()); +} + +#[test] +fn cors_origin_list_is_trimmed() { + let policy = CorsPolicy::parse(Some("https://a.example, https://b.example")).expect("list must parse"); + assert_eq!( + policy, + CorsPolicy::AllowList(vec!["https://a.example".to_owned(), "https://b.example".to_owned()]) + ); + assert!(policy.cors_layer().is_some()); +} + +#[test] +fn cors_rejects_empty_entries_and_mixed_wildcard() { + assert!(CorsPolicy::parse(Some("https://a.example,")).is_err()); + assert!(CorsPolicy::parse(Some("https://a.example,*")).is_err()); +} + +#[test] +fn cors_rejects_invalid_header_value() { + // A newline inside an origin is a header-injection attempt, not a valid value. + assert!(CorsPolicy::parse(Some("https://evil.example\nSet-Cookie: x")).is_err()); + assert!(CorsPolicy::parse(Some("https://evil.example/\u{7f}")).is_err()); +} + +#[test] +fn login_limiter_blocks_after_five_attempts() { + let limiter = DashMap::new(); + let ip = IpAddr::from([127, 0, 0, 1]); + for _ in 0..5 { + assert!(allow_login_attempt(&limiter, ip)); + } + assert!(!allow_login_attempt(&limiter, ip)); +} diff --git a/tests/pow_difficulty.rs b/tests/pow_difficulty.rs new file mode 100644 index 0000000..78d06db --- /dev/null +++ b/tests/pow_difficulty.rs @@ -0,0 +1,80 @@ +//! Adaptive PoW difficulty: configured floor, range clamping and the rate +//! ladder. The adjuster lives in `src/engine/challenge/difficulty.rs`; these +//! tests are the only consumer of the range constants. + +use rampart::engine::challenge::{DifficultyAdjuster, MAX_DIFFICULTY, MIN_DIFFICULTY}; + +#[test] +fn idle_uses_the_configured_floor() { + let adjuster = DifficultyAdjuster::new(4, 10); + assert_eq!(adjuster.current_difficulty(), 4); +} + +#[test] +fn configured_pow_difficulty_is_the_starting_point() { + // `[pow].difficulty = 7` must not be ignored until the first connection. + assert_eq!(DifficultyAdjuster::with_floor(7).current_difficulty(), 7); +} + +#[test] +fn configured_floor_can_be_lower_than_the_old_hardcoded_minimum() { + assert_eq!(DifficultyAdjuster::with_floor(2).current_difficulty(), 2); +} + +#[test] +fn configured_floor_is_clamped_to_the_supported_range() { + assert_eq!(DifficultyAdjuster::with_floor(0).current_difficulty(), MIN_DIFFICULTY); + assert_eq!(DifficultyAdjuster::with_floor(250).current_difficulty(), MAX_DIFFICULTY); +} + +#[test] +fn inverted_range_is_swapped_instead_of_breaking_the_clamp() { + let mut adjuster = DifficultyAdjuster::new(9, 5); + assert_eq!(adjuster.current_difficulty(), 5, "lower end is the starting point"); + for _ in 0..600 { + adjuster.record_connection(); + } + assert_eq!( + adjuster.current_difficulty(), + 9, + "under load the swapped upper end applies" + ); +} + +#[test] +fn pinned_range_never_moves() { + let mut adjuster = DifficultyAdjuster::new(6, 6); + for _ in 0..600 { + adjuster.record_connection(); + } + assert_eq!(adjuster.current_difficulty(), 6); +} + +#[test] +fn rate_ladder_is_clamped_to_the_configured_range() { + let mut adjuster = DifficultyAdjuster::new(7, 8); + for _ in 0..60 { + adjuster.record_connection(); + } + // Raw ladder step is 6, below the configured floor. + assert_eq!(adjuster.current_difficulty(), 7); + for _ in 60..201 { + adjuster.record_connection(); + } + // Raw ladder step is 8 and fits the range. + assert_eq!(adjuster.current_difficulty(), 8); + for _ in 201..600 { + adjuster.record_connection(); + } + // Raw ladder step is MAX_DIFFICULTY, above the configured ceiling. + assert_eq!(adjuster.current_difficulty(), 8); +} + +#[test] +fn default_range_rises_to_ten_under_load() { + let mut adjuster = DifficultyAdjuster::default(); + for _ in 0..600 { + adjuster.record_connection(); + } + assert_eq!(adjuster.current_difficulty(), 10); +} diff --git a/tests/pow_roundtrip.rs b/tests/pow_roundtrip.rs index ba7a660..7c3e9d0 100644 --- a/tests/pow_roundtrip.rs +++ b/tests/pow_roundtrip.rs @@ -1,7 +1,11 @@ //! PoW roundtrip tests: solver output must satisfy the verifier, replays and -//! garbage must not. +//! garbage must not, and the nonce exchange must survive TCP framing quirks. -use rampart::engine::challenge::{Challenge, solve}; +use rampart::engine::challenge::{Challenge, enforce, solve}; +use std::net::IpAddr; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::{TcpListener, TcpStream}; +use tokio::time::{Duration, sleep}; /// Fixed 32-byte token: sha256(hex(token) || "garbage") starts with "ebc1", /// so "garbage" provably fails the difficulty-2 check (first two hex chars @@ -9,6 +13,8 @@ use rampart::engine::challenge::{Challenge, solve}; /// per run, since any digest lands in the accepted prefix with p=(4/16)^2. const FIXED_TOKEN: [u8; 32] = *b"rampart-pow-fixed-token-test-v1!"; +const PEER: IpAddr = IpAddr::V4(std::net::Ipv4Addr::new(127, 0, 0, 1)); + #[test] fn solver_output_passes_verifier() { let mut challenge = Challenge::generate(3); @@ -25,3 +31,104 @@ fn verifier_rejects_garbage_and_replay() { assert!(challenge.verify(&nonce), "first use passes"); assert!(!challenge.verify(&nonce), "replay is rejected"); } + +#[test] +fn verifier_rejects_oversized_nonce() { + let mut challenge = Challenge::with_token(FIXED_TOKEN, 2); + let big_nonce = "0".repeat(65); + assert!(!challenge.verify(&big_nonce)); +} + +/// Connects a server/client loopback pair; the first value is the server side. +async fn pair() -> (TcpStream, TcpStream) { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind loopback"); + let addr = listener.local_addr().expect("local address"); + let accept = tokio::spawn(async move { listener.accept().await.expect("accept").0 }); + let client = TcpStream::connect(addr).await.expect("connect loopback"); + let server = accept.await.expect("accept task"); + (server, client) +} + +/// Reads one newline-terminated server line byte by byte. +async fn read_line(stream: &mut TcpStream) -> String { + let mut line = Vec::new(); + loop { + let mut byte = [0u8; 1]; + let n = stream.read(&mut byte).await.expect("read line"); + if n == 0 || byte[0] == b'\n' { + break; + } + line.push(byte[0]); + } + String::from_utf8(line).expect("challenge is hex") +} + +#[tokio::test] +async fn nonce_coalesced_with_handshake_is_preserved() { + let (mut server, mut client) = pair().await; + let server_task = tokio::spawn(async move { + let passed = enforce(&mut server, PEER, 2).await.expect("enforce"); + let mut tail = [0u8; 5]; + server + .read_exact(&mut tail) + .await + .expect("handshake bytes after the nonce"); + (passed, tail) + }); + + let challenge = read_line(&mut client).await; + let nonce = solve(&challenge, 2).expect("solved"); + // One write: nonce delimiter and the first handshake bytes in one segment. + client + .write_all(format!("{nonce}\nHELLO").as_bytes()) + .await + .expect("write nonce and handshake"); + let (passed, tail) = server_task.await.expect("server task"); + assert!(passed); + assert_eq!(&tail, b"HELLO", "bytes after the delimiter must reach the handshake"); +} + +#[tokio::test] +async fn nonce_split_across_segments_is_accepted() { + let (mut server, mut client) = pair().await; + let server_task = tokio::spawn(async move { enforce(&mut server, PEER, 2).await.expect("enforce") }); + + let challenge = read_line(&mut client).await; + let nonce = solve(&challenge, 2).expect("solved"); + let split = nonce.len() / 2; + client + .write_all(&nonce.as_bytes()[..split]) + .await + .expect("first segment"); + sleep(Duration::from_millis(20)).await; + client + .write_all(&nonce.as_bytes()[split..]) + .await + .expect("second segment"); + client.write_all(b"\n").await.expect("delimiter"); + + assert!(server_task.await.expect("server task")); +} + +#[tokio::test] +async fn eof_terminated_nonce_is_still_accepted() { + let (mut server, mut client) = pair().await; + let server_task = tokio::spawn(async move { enforce(&mut server, PEER, 2).await.expect("enforce") }); + + let challenge = read_line(&mut client).await; + let nonce = solve(&challenge, 2).expect("solved"); + client.write_all(nonce.as_bytes()).await.expect("write nonce"); + client.shutdown().await.expect("close write half"); + + assert!(server_task.await.expect("server task")); +} + +#[tokio::test] +async fn oversized_nonce_without_delimiter_is_rejected() { + let (mut server, mut client) = pair().await; + let server_task = tokio::spawn(async move { enforce(&mut server, PEER, 2).await.expect("enforce") }); + + let _challenge = read_line(&mut client).await; + client.write_all(&[b'0'; 65]).await.expect("oversized nonce"); + assert!(!server_task.await.expect("server task")); +} diff --git a/tests/redis_backoff.rs b/tests/redis_backoff.rs new file mode 100644 index 0000000..5e03f77 --- /dev/null +++ b/tests/redis_backoff.rs @@ -0,0 +1,25 @@ +//! Integration tests for the Redis reconnect backoff policy. + +// The backoff helpers live behind the optional `store-redis` feature, so this +// test target must be skipped when that feature is off (`cargo check +// --no-default-features --all-targets`, CI protocol-http job). +#![cfg(feature = "store-redis")] + +use std::time::Duration; + +use rampart::store::redis::{BACKOFF_MAX, BACKOFF_MIN, next_backoff}; + +#[test] +fn backoff_doubles_until_cap() { + let mut backoff = BACKOFF_MIN; + for expected_secs in [2, 4, 8, 16, 30, 30] { + backoff = next_backoff(backoff, BACKOFF_MAX); + assert_eq!(backoff, Duration::from_secs(expected_secs)); + } +} + +#[test] +fn backoff_never_exceeds_max() { + let backoff = next_backoff(Duration::from_secs(u64::MAX / 2), BACKOFF_MAX); + assert!(backoff <= BACKOFF_MAX); +} diff --git a/tests/subnet_detection.rs b/tests/subnet_detection.rs index 64b373a..e635118 100644 --- a/tests/subnet_detection.rs +++ b/tests/subnet_detection.rs @@ -1,12 +1,16 @@ //! Integration tests for subnet-level detection: tracker, detector and verdicts. use rampart::config::Config; +use rampart::engine::subnet::monitor::enforce_tracker_verdicts; use rampart::engine::subnet_tracker::SubnetTracker; +use rampart::filter::blacklist::Blacklist; use rampart::traffic::prefix::{ - PrefixKey, PrefixSnapshot, PrefixStatsSource, PrefixStatsVal, SubnetDetector, SubnetVerdict, prefix_of, + PrefixKey, PrefixSnapshot, PrefixStatsSource, PrefixStatsVal, PrefixVerdict, SubnetDetector, SubnetVerdict, + prefix_of, }; use std::collections::HashMap; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; +use std::time::Duration; fn v4(octets: [u8; 4]) -> IpAddr { IpAddr::V4(Ipv4Addr::from(octets)) @@ -194,6 +198,145 @@ fn spoof_gate_downgrades_block_to_challenge() { assert_eq!(verdict_for(&detector, &unknown), SubnetVerdict::Challenge); } +#[test] +fn tracker_groups_connections_by_prefix() { + let tracker = SubnetTracker::new(); + tracker.record_connection(v4([203, 0, 113, 1])); + tracker.record_connection(v4([203, 0, 113, 2])); + tracker.record_connection(v4([198, 51, 100, 9])); + let by_prefix: HashMap = tracker.take_snapshot().into_iter().collect(); + assert_eq!(by_prefix.len(), 2); + assert_eq!(by_prefix[&v4([203, 0, 113, 0])].syn_count, 2); + assert_eq!(by_prefix[&v4([203, 0, 113, 0])].unique_sources, 2); + assert_eq!(by_prefix[&v4([198, 51, 100, 0])].syn_count, 1); +} + +#[test] +fn tracker_snapshot_drains_window() { + let tracker = SubnetTracker::new(); + tracker.record_connection(v4([10, 0, 0, 1])); + assert_eq!(tracker.take_snapshot().len(), 1); + assert!(tracker.take_snapshot().is_empty()); +} + +#[test] +fn tracker_mapped_addresses_share_v4_prefix() { + let tracker = SubnetTracker::new(); + tracker.record_connection(v4([203, 0, 113, 5])); + // ::ffff:203.0.113.6 + let mapped = Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xcb00, 0x7106); + tracker.record_connection(IpAddr::V6(mapped)); + let snaps = tracker.take_snapshot(); + assert_eq!(snaps.len(), 1); + assert_eq!(snaps[0].0, v4([203, 0, 113, 0])); + assert_eq!(snaps[0].1.unique_sources, 2); +} + +#[test] +fn tracker_prefix_cap_drops_new_prefixes_and_counts_them() { + let tracker = SubnetTracker::with_limits(2, 8); + tracker.record_connection(v4([10, 0, 0, 1])); + tracker.record_connection(v4([10, 0, 1, 1])); + tracker.record_connection(v4([10, 0, 2, 1])); + assert_eq!(tracker.dropped_prefixes(), 1); + // Existing prefixes keep accumulating even while the table is full. + tracker.record_connection(v4([10, 0, 0, 2])); + + let snaps: HashMap = tracker.take_snapshot().into_iter().collect(); + assert_eq!(snaps.len(), 2); + assert_eq!(snaps[&v4([10, 0, 0, 0])].syn_count, 2); + assert!( + !snaps.contains_key(&v4([10, 0, 2, 0])), + "over-cap prefix was not stored" + ); +} + +#[test] +fn tracker_source_cap_bounds_per_prefix_memory() { + let tracker = SubnetTracker::with_limits(8, 3); + for last in 1..=100u8 { + tracker.record_connection(v4([203, 0, 113, last])); + } + assert_eq!( + tracker.dropped_sources(), + 97, + "only the first three distinct sources are stored" + ); + + let snaps = tracker.take_snapshot(); + assert_eq!(snaps.len(), 1); + assert_eq!(snaps[0].1.syn_count, 100, "the aggregate counter is not capped"); + assert_eq!(snaps[0].1.unique_sources, 3); +} + +#[test] +fn tracker_source_cap_counts_only_new_addresses() { + let tracker = SubnetTracker::with_limits(1, 2); + tracker.record_connection(v4([198, 51, 100, 1])); + tracker.record_connection(v4([198, 51, 100, 2])); + tracker.record_connection(v4([198, 51, 100, 3])); + assert_eq!(tracker.dropped_sources(), 1); + // Repeated traffic from a remembered source is not a dropped new source. + tracker.record_connection(v4([198, 51, 100, 1])); + assert_eq!(tracker.dropped_sources(), 1); + tracker.record_connection(v4([198, 51, 100, 4])); + assert_eq!(tracker.dropped_sources(), 2); +} + +fn block_verdict(prefix: IpAddr) -> PrefixVerdict { + PrefixVerdict { + prefix, + verdict: SubnetVerdict::Block, + snapshot: PrefixSnapshot { + syn_count: 4000, + unique_sources: 64, + ..PrefixSnapshot::default() + }, + } +} + +#[test] +fn trusted_tracker_block_reaches_the_blacklist_with_the_configured_ttl() { + let blacklist = Blacklist::new(); + let verdicts = [block_verdict(v4([203, 0, 113, 0]))]; + let banned = enforce_tracker_verdicts(&verdicts, &blacklist, Duration::from_secs(120), true); + assert_eq!(banned, 1); + assert!(blacklist.is_blocked(v4([203, 0, 113, 0]))); +} + +#[test] +fn untrusted_tracker_block_only_warns() { + let blacklist = Blacklist::new(); + let verdicts = [block_verdict(v4([198, 51, 100, 0]))]; + let banned = enforce_tracker_verdicts(&verdicts, &blacklist, Duration::from_secs(120), false); + assert_eq!(banned, 0); + assert!(blacklist.is_empty()); +} + +#[test] +fn tracker_ban_uses_the_given_duration() { + let blacklist = Blacklist::new(); + let verdicts = [block_verdict(v4([192, 0, 2, 0]))]; + enforce_tracker_verdicts(&verdicts, &blacklist, Duration::from_millis(1), true); + std::thread::sleep(Duration::from_millis(2)); + assert!( + !blacklist.is_blocked(v4([192, 0, 2, 0])), + "ban must expire with its TTL" + ); +} + +#[test] +fn non_block_verdicts_do_not_ban() { + let blacklist = Blacklist::new(); + let mut verdict = block_verdict(v4([203, 0, 113, 0])); + verdict.verdict = SubnetVerdict::Challenge; + assert_eq!( + enforce_tracker_verdicts(&[verdict], &blacklist, Duration::from_secs(60), true), + 0 + ); + assert!(blacklist.is_empty()); +} + #[test] fn evaluate_skips_empty_prefixes() { let detector = SubnetDetector::new(100, 0); @@ -213,3 +356,22 @@ fn evaluate_skips_empty_prefixes() { assert_eq!(verdicts[0].prefix, v4([10, 0, 1, 0])); assert_eq!(verdicts[0].verdict, SubnetVerdict::Block); } + +#[test] +fn prefix_key_normalises_ipv4_mapped_addresses() { + // ::ffff:1.2.3.4 + let mapped = IpAddr::V6(Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0x0102, 0x0304)); + let plain = v4([1, 2, 3, 4]); + + assert_eq!( + PrefixKey::new(mapped), + PrefixKey::new(plain), + "a mapped client must produce one key in XDP prefix stats and the user-space tracker" + ); + assert_eq!( + PrefixKey::new(mapped).family, + rampart::traffic::prefix::PREFIX_FAMILY_V4 + ); + assert_eq!(PrefixKey::new(mapped).to_prefix(), Some(prefix_of(mapped))); + assert_eq!(PrefixKey::new(mapped).to_prefix(), Some(v4([1, 2, 3, 0]))); +} diff --git a/tests/traffic_intel.rs b/tests/traffic_intel.rs index 1156bcb..7f73248 100644 --- a/tests/traffic_intel.rs +++ b/tests/traffic_intel.rs @@ -1,14 +1,15 @@ -//! Integration tests for Traffic Intelligence on the hot path: auto-ban driven -//! by reputation, attack detection on a synthetic spike and alerts on state -//! transitions. +//! Integration tests for Traffic Intelligence on the hot path: EWMA sanity, +//! attack detection across a sustained flood, reputation decay/eviction and +//! auto-ban driven by reputation. use rampart::config::DetectAutobanConfig; use rampart::filter::blacklist::Blacklist; use rampart::traffic::alert::AlertDispatcher; -use rampart::traffic::detector::{AttackDetector, AttackStatus}; +use rampart::traffic::detector::{AttackDetector, AttackStatus, DetectorThresholds}; +use rampart::traffic::ewma::Ewma; use rampart::traffic::hook::TrafficHook; -use rampart::traffic::reputation::IpReputation; -use std::net::{IpAddr, Ipv4Addr}; +use rampart::traffic::reputation::{IpReputation, ReputationPolicy}; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use std::sync::Arc; use std::time::Duration; @@ -23,6 +24,363 @@ fn autoban_cfg(enabled: bool) -> DetectAutobanConfig { } } +fn quiet(detector: &mut AttackDetector) { + for _ in 0..12 { + detector.analyze(500.0, 50.0); + } +} + +// EWMA + +#[test] +fn ewma_starts_at_zero() { + assert_eq!(Ewma::new(0.125).value(), 0.0); +} + +#[test] +fn ewma_alpha_one_follows_the_sample() { + let mut ewma = Ewma::new(1.0); + ewma.update(100.0); + assert!((ewma.value() - 100.0).abs() < 1.0); +} + +#[test] +fn ewma_converges_to_a_constant_sample() { + let mut ewma = Ewma::new(0.5); + for _ in 0..10 { + ewma.update(50.0); + } + assert!((ewma.value() - 50.0).abs() < 1.0); +} + +#[test] +fn ewma_falls_back_on_invalid_alpha() { + for alpha in [f64::NAN, f64::INFINITY, 0.0, -0.5] { + let mut ewma = Ewma::new(alpha); + ewma.update(100.0); + assert!(ewma.value().is_finite(), "alpha={alpha} must keep the average finite"); + assert!( + ewma.value() > 0.0 && ewma.value() < 100.0, + "alpha={alpha} must fall back to the default smoothing" + ); + } +} + +#[test] +fn ewma_seed_primes_the_baseline() { + let mut ewma = Ewma::new(0.01); + ewma.seed(500.0); + assert_eq!(ewma.value(), 500.0, "a seeded baseline starts at the observed value"); + ewma.update(500.0); + assert_eq!(ewma.value(), 500.0); + assert!(!ewma.value().is_nan()); +} + +#[test] +fn ewma_ignores_non_finite_samples() { + let mut ewma = Ewma::new(0.5); + ewma.update(10.0); + let before = ewma.value(); + ewma.update(f64::NAN); + ewma.update(f64::INFINITY); + assert_eq!(ewma.value(), before, "non-finite samples must not move the average"); +} + +// Attack detector + +#[test] +fn attack_detector_reports_normal_traffic() { + let mut detector = AttackDetector::new(); + quiet(&mut detector); + assert_eq!(detector.analyze(500.0, 50.0), AttackStatus::Normal); +} + +#[test] +fn attack_detector_flags_pps_spike_as_suspicious() { + let mut detector = AttackDetector::new(); + for _ in 0..10 { + detector.analyze(1.0, 1.0); + } + assert_eq!(detector.analyze(5.0, 1.0), AttackStatus::Suspicious); +} + +#[test] +fn attack_detector_flags_cps_spike_as_suspicious() { + let mut detector = AttackDetector::new(); + for _ in 0..10 { + detector.analyze(1.0, 1.0); + } + assert_eq!(detector.analyze(1.0, 5.0), AttackStatus::Suspicious); +} + +#[test] +fn attack_detector_escalates_after_three_anomalies() { + let mut detector = AttackDetector::new(); + for _ in 0..10 { + detector.analyze(1.0, 1.0); + } + detector.analyze(10_000.0, 100.0); + detector.analyze(10_000.0, 100.0); + assert_eq!(detector.analyze(10_000.0, 100.0), AttackStatus::UnderAttack); +} + +#[test] +fn attack_detector_resets_on_a_normal_window() { + let mut detector = AttackDetector::new(); + for _ in 0..10 { + detector.analyze(1.0, 1.0); + } + detector.analyze(100.0, 10.0); + detector.analyze(100.0, 10.0); + detector.analyze(1.0, 1.0); + assert_eq!(detector.analyze(1.0, 1.0), AttackStatus::Normal); +} + +#[test] +fn detector_flags_synthetic_traffic_spike() { + let mut detector = AttackDetector::new(); + + // Baseline: a quiet profile. + for _ in 0..12 { + detector.analyze(500.0, 50.0); + } + + // Synthetic spike: 200x pps and cps for three consecutive windows. + assert_eq!( + detector.analyze(100_000.0, 10_000.0), + AttackStatus::Suspicious, + "first anomalous window is suspicious" + ); + assert_eq!(detector.analyze(100_000.0, 10_000.0), AttackStatus::Suspicious); + assert_eq!( + detector.analyze(100_000.0, 10_000.0), + AttackStatus::UnderAttack, + "three anomalous windows in a row must escalate to UnderAttack" + ); +} + +#[test] +fn sustained_flood_stays_under_attack_and_keeps_banning_offenders() { + let reputation = Arc::new(IpReputation::new()); + let blacklist = Arc::new(Blacklist::new()); + let hook = TrafficHook::new(Arc::clone(&reputation), Arc::clone(&blacklist), autoban_cfg(true), 60); + let mut detector = AttackDetector::new(); + quiet(&mut detector); + + for tick in 0..10u8 { + let status = detector.analyze(100_000.0, 10_000.0); + if tick < 2 { + assert_eq!(status, AttackStatus::Suspicious, "flood tick {tick}"); + } else { + assert_eq!( + status, + AttackStatus::UnderAttack, + "flood tick {tick}: the detector must not learn the attack as normal" + ); + } + + // A fresh offender joins on every tick of the attack. The edge keeps + // banning them only while the detector stays in UnderAttack. + let offender = ip([203, 0, 113, tick + 1]); + for _ in 0..6 { + reputation.record_bad(offender); + } + assert!(hook.ban_offenders() >= 1, "flood tick {tick} must produce a ban"); + assert!( + blacklist.is_blocked(offender), + "flood tick {tick}: offender must be banned" + ); + } +} + +#[test] +fn under_attack_exits_only_after_the_recovery_window() { + let mut detector = AttackDetector::new(); + quiet(&mut detector); + for _ in 0..3 { + detector.analyze(100_000.0, 10_000.0); + } + assert_eq!(detector.analyze(100_000.0, 10_000.0), AttackStatus::UnderAttack); + + // Four calm windows are not enough: the exit hysteresis is five ticks. + for tick in 0..4 { + assert_eq!( + detector.analyze(500.0, 50.0), + AttackStatus::UnderAttack, + "calm tick {tick} must not end the attack" + ); + } + assert_eq!( + detector.analyze(500.0, 50.0), + AttackStatus::Normal, + "the fifth calm tick ends the attack" + ); +} + +#[test] +fn detector_thresholds_are_configurable() { + let thresholds = DetectorThresholds { + suspicious_score: 1.5, + attack_score: 2.0, + escalate_after: 1, + recover_after: 2, + }; + let mut detector = AttackDetector::with_thresholds(thresholds); + for _ in 0..10 { + detector.analyze(100.0, 10.0); + } + assert_eq!( + detector.analyze(1_000.0, 100.0), + AttackStatus::UnderAttack, + "escalate_after = 1 must escalate on the first anomalous window" + ); + assert_eq!(detector.analyze(100.0, 10.0), AttackStatus::UnderAttack); + assert_ne!( + detector.analyze(100.0, 10.0), + AttackStatus::UnderAttack, + "recover_after = 2 must end the attack" + ); +} + +// Reputation + +#[test] +fn reputation_initial_score_is_neutral() { + let rep = IpReputation::new(); + assert_eq!(rep.score(ip([192, 168, 1, 1])), 0); +} + +#[test] +fn reputation_good_and_bad_steps() { + let rep = IpReputation::new(); + let good = ip([10, 0, 0, 1]); + let bad = ip([10, 0, 0, 2]); + rep.record_good(good); + rep.record_bad(bad); + assert_eq!(rep.score(good), 1); + assert_eq!(rep.score(bad), -10); +} + +#[test] +fn reputation_scores_are_clamped() { + let rep = IpReputation::new(); + let good = ip([10, 0, 0, 3]); + let bad = ip([10, 0, 0, 4]); + for _ in 0..200 { + rep.record_good(good); + } + for _ in 0..20 { + rep.record_bad(bad); + } + assert_eq!(rep.score(good), 100); + assert_eq!(rep.score(bad), -100); +} + +#[test] +fn reputation_trusts_after_enough_good_connections() { + let rep = IpReputation::new(); + let ip = ip([10, 0, 0, 5]); + assert!(!rep.is_trusted(ip)); + for _ in 0..51 { + rep.record_good(ip); + } + assert!(rep.is_trusted(ip)); +} + +#[test] +fn reputation_tracks_ipv6() { + let rep = IpReputation::new(); + let ip = IpAddr::V6(Ipv6Addr::LOCALHOST); + rep.record_bad(ip); + assert_eq!(rep.score(ip), -10); +} + +#[test] +fn reputation_decays_toward_neutral() { + let policy = ReputationPolicy { + idle_ttl: Duration::from_secs(10), + decay_half_life: Duration::from_millis(20), + max_offenders: 64, + }; + let rep = IpReputation::with_policy(policy); + let bad = ip([10, 0, 0, 6]); + for _ in 0..20 { + rep.record_bad(bad); + } + assert_eq!(rep.score(bad), -100); + std::thread::sleep(Duration::from_millis(40)); + let decayed = rep.score(bad); + assert!( + decayed > -100 && decayed < 0, + "two half-lives must move the score toward neutral, got {decayed}" + ); +} + +#[test] +fn reputation_sweep_evicts_idle_and_keeps_active() { + let policy = ReputationPolicy { + idle_ttl: Duration::from_millis(20), + decay_half_life: Duration::from_secs(1800), + max_offenders: 64, + }; + let rep = IpReputation::with_policy(policy); + let idle = ip([10, 0, 0, 7]); + let active = ip([10, 0, 0, 8]); + rep.record_bad(idle); + rep.record_bad(active); + + std::thread::sleep(Duration::from_millis(40)); + rep.record_bad(active); + for _ in 0..8 { + rep.sweep(); + } + + assert_eq!(rep.len(), 1, "only the idle source may be evicted"); + assert_eq!(rep.score(idle), 0, "evicted source is neutral again"); + assert!(rep.score(active) < 0, "active source keeps its score"); +} + +#[test] +fn offenders_are_capped_per_cycle() { + let policy = ReputationPolicy { + idle_ttl: Duration::from_secs(600), + decay_half_life: Duration::from_secs(1800), + max_offenders: 3, + }; + let rep = IpReputation::with_policy(policy); + for last in 1..=20u8 { + let source = ip([198, 51, 100, last]); + for _ in 0..6 { + rep.record_bad(source); + } + } + assert_eq!( + rep.offenders_below(-50).len(), + 3, + "one cycle must not return the whole offender table" + ); +} + +#[test] +fn offenders_below_returns_worst_sources() { + let rep = IpReputation::new(); + let bad = ip([10, 0, 0, 1]); + let good = ip([10, 0, 0, 2]); + for _ in 0..6 { + rep.record_bad(bad); + } + rep.record_good(good); + assert_eq!(rep.offenders_below(-50), vec![bad]); +} + +#[test] +fn offenders_below_is_empty_without_offenders() { + let rep = IpReputation::new(); + assert!(rep.offenders_below(-50).is_empty()); +} + +// Traffic hook auto-ban + #[test] fn reputation_fall_after_drops_triggers_autoban() { let reputation = Arc::new(IpReputation::new()); @@ -69,29 +427,6 @@ fn autoban_ttl_expires_ban() { ); } -#[test] -fn detector_flags_synthetic_traffic_spike() { - let mut detector = AttackDetector::new(); - - // Baseline: a quiet profile. - for _ in 0..12 { - detector.analyze(500.0, 50.0); - } - - // Synthetic spike: 200x pps and cps for three consecutive windows. - assert_eq!( - detector.analyze(100_000.0, 10_000.0), - AttackStatus::Suspicious, - "first anomalous window is suspicious" - ); - assert_eq!(detector.analyze(100_000.0, 10_000.0), AttackStatus::Suspicious); - assert_eq!( - detector.analyze(100_000.0, 10_000.0), - AttackStatus::UnderAttack, - "three anomalous windows in a row must escalate to UnderAttack" - ); -} - #[test] fn alerts_fire_only_on_state_transitions() { let dispatcher = AlertDispatcher::new();