perf: replace regex swear filter with Aho-Corasick O(n) trie + add FilterResult unit tests

This commit is contained in:
loki5512344 2026-07-05 16:08:32 +02:00
parent 42ebac890c
commit 5eae09e5bb
12 changed files with 394 additions and 73 deletions

View file

@ -1,6 +1,7 @@
package com.loki.lochat.core.filter.filters; package com.loki.lochat.core.filter.filters;
import com.loki.lochat.core.filter.FilterResult; import com.loki.lochat.core.filter.FilterResult;
import com.loki.lochat.utils.text.AhoCorasick;
import org.bukkit.configuration.file.FileConfiguration; import org.bukkit.configuration.file.FileConfiguration;
import org.bukkit.entity.Player; import org.bukkit.entity.Player;
@ -10,32 +11,14 @@ import java.io.BufferedReader;
import java.io.InputStream; import java.io.InputStream;
import java.io.InputStreamReader; import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.HashSet; import java.util.ArrayList;
import java.util.Map; import java.util.List;
import java.util.Set;
import java.util.regex.Pattern;
public class SwearFilter { public class SwearFilter {
private final boolean enabled; private final boolean enabled;
private final boolean replaceMode; private final boolean replaceMode;
private final String replacement; private final String replacement;
private final Set<Pattern> swearPatterns = new HashSet<>(); private AhoCorasick ahoCorasick;
// Карта замены латиницы на кириллицу для обхода фильтра
private static final Map<Character, Character> LATIN_TO_CYRILLIC = Map.ofEntries(
Map.entry('a', 'а'), Map.entry('A', 'А'),
Map.entry('e', 'е'), Map.entry('E', 'Е'),
Map.entry('o', 'о'), Map.entry('O', 'О'),
Map.entry('p', 'р'), Map.entry('P', 'Р'),
Map.entry('c', 'с'), Map.entry('C', 'С'),
Map.entry('y', 'у'), Map.entry('Y', 'У'),
Map.entry('x', 'х'), Map.entry('X', 'Х'),
Map.entry('k', 'к'), Map.entry('K', 'К'),
Map.entry('m', 'м'), Map.entry('M', 'М'),
Map.entry('h', 'н'), Map.entry('H', 'Н'),
Map.entry('t', 'т'), Map.entry('T', 'Т'),
Map.entry('b', 'в'), Map.entry('B', 'В')
);
public SwearFilter(FileConfiguration config, JavaPlugin plugin) { public SwearFilter(FileConfiguration config, JavaPlugin plugin) {
this.enabled = config.getBoolean("filters.swear.enabled", true); this.enabled = config.getBoolean("filters.swear.enabled", true);
@ -48,51 +31,17 @@ public class SwearFilter {
} }
private void loadWordList(JavaPlugin plugin) { private void loadWordList(JavaPlugin plugin) {
List<String> words = new ArrayList<>();
try (InputStream is = plugin.getResource("badwords.txt"); try (InputStream is = plugin.getResource("badwords.txt");
BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) { BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) {
reader.lines() reader.lines()
.map(String::trim) .map(String::trim)
.filter(line -> !line.isEmpty() && !line.startsWith("#")) .filter(line -> !line.isEmpty() && !line.startsWith("#"))
.forEach(word -> { .forEach(words::add);
// Создаем паттерн с учетом возможной замены букв
String regex = buildFlexiblePattern(word);
swearPatterns.add(Pattern.compile(regex, Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE));
});
} catch (Exception e) { } catch (Exception e) {
plugin.getLogger().warning("Failed to load badwords.txt: " + e.getMessage()); plugin.getLogger().warning("Failed to load badwords.txt: " + e.getMessage());
} }
} this.ahoCorasick = new AhoCorasick(words);
/**
* Создает гибкий паттерн, который ловит замены букв (х->x, о->o и т.д.)
*/
private String buildFlexiblePattern(String word) {
StringBuilder pattern = new StringBuilder();
for (char c : word.toLowerCase().toCharArray()) {
// Для каждой буквы создаем группу с возможными заменами
switch (c) {
case 'а' -> pattern.append("[аaА@]");
case 'е' -> pattern.append("[еeЕ3]");
case 'о' -> pattern.append("[оoО0]");
case 'р' -> pattern.append("[рpР]");
case 'с' -> pattern.append("[сcС]");
case 'у' -> pattern.append("[уyУ]");
case 'х' -> pattern.append("[хxХ]");
case 'к' -> pattern.append("[кkК]");
case 'м' -> pattern.append("[мmМ]");
case 'н' -> pattern.append("[нhНN]");
case 'т' -> pattern.append("[тtТ]");
case 'в' -> pattern.append("[вbВ]");
case 'и' -> pattern.append("[иiИ1]");
case 'з' -> pattern.append("[з3З]");
case 'б' -> pattern.append("[б6Б]");
default -> pattern.append(Pattern.quote(String.valueOf(c)));
}
}
return pattern.toString();
} }
public FilterResult filter(Player player, String message) { public FilterResult filter(Player player, String message) {
@ -100,23 +49,13 @@ public class SwearFilter {
return FilterResult.ok(message); return FilterResult.ok(message);
} }
for (Pattern pattern : swearPatterns) { if (ahoCorasick.matches(message)) {
if (pattern.matcher(message).find()) {
if (replaceMode) { if (replaceMode) {
return FilterResult.ok(replaceSwear(message)); return FilterResult.ok(ahoCorasick.replaceAll(message, replacement));
} }
return FilterResult.blocked("Сообщение содержит запрещенные слова"); return FilterResult.blocked("Сообщение содержит запрещенные слова");
} }
}
return FilterResult.ok(message); return FilterResult.ok(message);
} }
private String replaceSwear(String message) {
String result = message;
for (Pattern pattern : swearPatterns) {
result = pattern.matcher(result).replaceAll(replacement);
}
return result;
}
} }

View file

@ -0,0 +1,172 @@
package com.loki.lochat.utils.text;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Deque;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public final class AhoCorasick {
private static final Map<Character, Character> NORMALIZE = new HashMap<>();
static {
NORMALIZE.put('a', 'а');
NORMALIZE.put('A', 'А');
NORMALIZE.put('e', 'е');
NORMALIZE.put('E', 'Е');
NORMALIZE.put('o', 'о');
NORMALIZE.put('O', 'О');
NORMALIZE.put('p', 'р');
NORMALIZE.put('P', 'Р');
NORMALIZE.put('c', 'с');
NORMALIZE.put('C', 'С');
NORMALIZE.put('y', 'у');
NORMALIZE.put('Y', 'У');
NORMALIZE.put('x', 'х');
NORMALIZE.put('X', 'Х');
NORMALIZE.put('k', 'к');
NORMALIZE.put('K', 'К');
NORMALIZE.put('m', 'м');
NORMALIZE.put('M', 'М');
NORMALIZE.put('h', 'н');
NORMALIZE.put('H', 'Н');
NORMALIZE.put('n', 'н');
NORMALIZE.put('N', 'Н');
NORMALIZE.put('t', 'т');
NORMALIZE.put('T', 'Т');
NORMALIZE.put('b', 'в');
NORMALIZE.put('B', 'В');
NORMALIZE.put('@', 'а');
NORMALIZE.put('0', 'о');
NORMALIZE.put('3', 'е');
NORMALIZE.put('1', 'и');
NORMALIZE.put('6', 'б');
}
private final Node root;
public AhoCorasick(List<String> words) {
this.root = new Node((char) 0);
buildTrie(words);
buildFailLinks();
}
private void buildTrie(List<String> words) {
for (String word : words) {
if (word.isEmpty()) {
continue;
}
Node node = root;
for (char c : word.toCharArray()) {
char normalized = normalize(c);
Node current = node;
node = node.children.computeIfAbsent(normalized, ch -> new Node(ch, current));
}
node.output = true;
node.word = word;
}
}
private void buildFailLinks() {
Deque<Node> queue = new ArrayDeque<>();
for (Node node : root.children.values()) {
node.fail = root;
queue.addLast(node);
}
while (!queue.isEmpty()) {
Node current = queue.pollFirst();
for (Node child : current.children.values()) {
Node fail = current.fail;
while (fail != root && !fail.children.containsKey(child.c)) {
fail = fail.fail;
}
if (fail != root || fail.children.containsKey(child.c)) {
Node f = fail.children.get(child.c);
child.fail = f != null ? f : root;
} else {
child.fail = root;
}
child.output = child.output || child.fail.output;
if (child.fail.output && child.word == null) {
child.word = child.fail.word;
}
queue.addLast(child);
}
}
}
public boolean matches(String text) {
Node node = root;
for (int i = 0; i < text.length(); i++) {
char c = normalize(text.charAt(i));
while (node != root && !node.children.containsKey(c)) {
node = node.fail;
}
node = node.children.getOrDefault(c, root);
if (node.output) {
return true;
}
}
return false;
}
public String replaceAll(String text, String replacement) {
Node node = root;
int lastEnd = 0;
StringBuilder result = new StringBuilder();
for (int i = 0; i < text.length(); i++) {
char c = normalize(text.charAt(i));
while (node != root && !node.children.containsKey(c)) {
node = node.fail;
}
node = node.children.getOrDefault(c, root);
if (node.output && node.word != null) {
result.append(text, lastEnd, i - node.word.length() + 1);
result.append(replacement);
lastEnd = i + 1;
}
}
result.append(text, lastEnd, text.length());
return result.toString();
}
public List<String> findMatches(String text) {
List<String> found = new ArrayList<>();
Node node = root;
for (int i = 0; i < text.length(); i++) {
char c = normalize(text.charAt(i));
while (node != root && !node.children.containsKey(c)) {
node = node.fail;
}
node = node.children.getOrDefault(c, root);
if (node.output && node.word != null) {
found.add(node.word);
}
}
return found;
}
private static char normalize(char c) {
return NORMALIZE.getOrDefault(c, Character.toLowerCase(c));
}
private static final class Node {
final char c;
final Map<Character, Node> children = new HashMap<>();
Node fail;
boolean output;
String word;
Node(char c) {
this.c = c;
this.fail = this;
}
Node(char c, Node parent) {
this.c = c;
}
}
}

View file

@ -0,0 +1,65 @@
package com.loki.lochat.core.filter;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.*;
class FilterResultTest {
@Test
void ok_allowed() {
FilterResult r = FilterResult.ok("hello");
assertTrue(r.allowed());
assertNull(r.blockReason());
assertEquals("hello", r.filteredMessage());
}
@Test
void ok_getMessage() {
FilterResult r = FilterResult.ok("hello");
assertEquals("hello", r.getMessage());
}
@Test
void blocked_notAllowed() {
FilterResult r = FilterResult.blocked("bad word");
assertFalse(r.allowed());
assertEquals("bad word", r.blockReason());
assertNull(r.filteredMessage());
}
@Test
void ok_nullMessage() {
FilterResult r = FilterResult.ok(null);
assertTrue(r.allowed());
assertNull(r.filteredMessage());
}
@Test
void blocked_nullReason() {
FilterResult r = FilterResult.blocked(null);
assertFalse(r.allowed());
assertNull(r.blockReason());
}
@Test
void record_equals() {
FilterResult a = FilterResult.ok("hello");
FilterResult b = FilterResult.ok("hello");
assertEquals(a, b);
assertEquals(a.hashCode(), b.hashCode());
}
@Test
void record_notEquals() {
FilterResult a = FilterResult.ok("hello");
FilterResult b = FilterResult.blocked("bad");
assertNotEquals(a, b);
}
@Test
void record_toString() {
FilterResult r = FilterResult.ok("test");
assertTrue(r.toString().contains("test"));
}
}

View file

@ -0,0 +1,145 @@
package com.loki.lochat.utils.text;
import org.junit.jupiter.api.Test;
import java.util.List;
import static org.junit.jupiter.api.Assertions.*;
class AhoCorasickTest {
@Test
void matches_simpleWord() {
AhoCorasick ac = new AhoCorasick(List.of("хуй"));
assertTrue(ac.matches("это хуй"));
assertTrue(ac.matches("хуй"));
assertFalse(ac.matches("привет"));
}
@Test
void matches_multipleWords() {
AhoCorasick ac = new AhoCorasick(List.of("хуй", "пизда", "ебать"));
assertTrue(ac.matches("старый хуй"));
assertTrue(ac.matches("полная пизда"));
assertTrue(ac.matches("ебать копать"));
assertFalse(ac.matches("нормальный текст"));
}
@Test
void matches_overlappingWords() {
AhoCorasick ac = new AhoCorasick(List.of("хуй", "хуе"));
assertTrue(ac.matches("хуе"));
assertTrue(ac.matches("хуй"));
}
@Test
void matches_subword() {
AhoCorasick ac = new AhoCorasick(List.of("хуй"));
assertFalse(ac.matches("ху"));
assertFalse(ac.matches(""));
}
@Test
void matches_latinToCyrillicSubstitution() {
AhoCorasick ac = new AhoCorasick(List.of("хуй", "пизда"));
assertTrue(ac.matches("xуй"));
assertTrue(ac.matches("xyй"));
assertTrue(ac.matches("хyй"));
}
@Test
void matches_caseInsensitive() {
AhoCorasick ac = new AhoCorasick(List.of("хуй"));
assertTrue(ac.matches("ХУЙ"));
assertTrue(ac.matches("Хуй"));
assertTrue(ac.matches("хуй"));
}
@Test
void matches_mixedSubstitutions() {
AhoCorasick ac = new AhoCorasick(List.of("ебать"));
assertTrue(ac.matches("3бать"));
assertTrue(ac.matches("ебaть"));
}
@Test
void replaceAll_replacesMatches() {
AhoCorasick ac = new AhoCorasick(List.of("хуй", "пизда"));
assertEquals("*** текст", ac.replaceAll("хуй текст", "***"));
assertEquals("*** и ***", ac.replaceAll("хуй и пизда", "***"));
}
@Test
void replaceAll_noMatch() {
AhoCorasick ac = new AhoCorasick(List.of("хуй"));
assertEquals("чистый текст", ac.replaceAll("чистый текст", "***"));
}
@Test
void replaceAll_nonOverlapping() {
AhoCorasick ac = new AhoCorasick(List.of("хуй", "пизда"));
assertEquals("*** ***", ac.replaceAll("хуй пизда", "***"));
}
@Test
void findMatches_returnsAll() {
AhoCorasick ac = new AhoCorasick(List.of("хуй", "пизда", "ебать"));
List<String> found = ac.findMatches("хуй и пизда и ебать");
assertTrue(found.contains("хуй"));
assertTrue(found.contains("пизда"));
assertTrue(found.contains("ебать"));
}
@Test
void findMatches_empty() {
AhoCorasick ac = new AhoCorasick(List.of("хуй"));
assertTrue(ac.findMatches("чистый текст").isEmpty());
}
@Test
void emptyWordList() {
AhoCorasick ac = new AhoCorasick(List.of());
assertFalse(ac.matches("любой текст"));
assertEquals("любой текст", ac.replaceAll("любой текст", "***"));
}
@Test
void emptyStringInWordList() {
AhoCorasick ac = new AhoCorasick(List.of(""));
assertFalse(ac.matches("тест"));
}
@Test
void matches_longText() {
AhoCorasick ac = new AhoCorasick(List.of("хуй", "пизда"));
assertTrue(ac.matches("a".repeat(1000) + "хуй" + "b".repeat(1000)));
}
@Test
void matches_specialCharacters() {
AhoCorasick ac = new AhoCorasick(List.of("хуй"));
assertTrue(ac.matches(",хуй."));
assertTrue(ac.matches("текст с xуй внутри"));
}
@Test
void matches_insideWord() {
AhoCorasick ac = new AhoCorasick(List.of("хуй"));
assertTrue(ac.matches("хуйтекст"));
assertTrue(ac.matches("текстхуйтекст"));
}
@Test
void matches_punctuationAround() {
AhoCorasick ac = new AhoCorasick(List.of("хуй"));
assertTrue(ac.matches("хуй,"));
assertTrue(ac.matches("(хуй)"));
assertTrue(ac.matches("\"хуй\""));
}
@Test
void replaceAll_partialWord() {
AhoCorasick ac = new AhoCorasick(List.of("хуй"));
assertEquals("***ня", ac.replaceAll("хуйня", "***"));
}
}