perf: replace regex swear filter with Aho-Corasick O(n) trie + add FilterResult unit tests
This commit is contained in:
parent
42ebac890c
commit
5eae09e5bb
12 changed files with 394 additions and 73 deletions
|
|
@ -1,6 +1,7 @@
|
|||
package com.loki.lochat.core.filter.filters;
|
||||
|
||||
import com.loki.lochat.core.filter.FilterResult;
|
||||
import com.loki.lochat.utils.text.AhoCorasick;
|
||||
|
||||
import org.bukkit.configuration.file.FileConfiguration;
|
||||
import org.bukkit.entity.Player;
|
||||
|
|
@ -10,89 +11,37 @@ import java.io.BufferedReader;
|
|||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class SwearFilter {
|
||||
private final boolean enabled;
|
||||
private final boolean replaceMode;
|
||||
private final String replacement;
|
||||
private final Set<Pattern> swearPatterns = new HashSet<>();
|
||||
|
||||
// Карта замены латиницы на кириллицу для обхода фильтра
|
||||
private static final Map<Character, Character> LATIN_TO_CYRILLIC = Map.ofEntries(
|
||||
Map.entry('a', 'а'), Map.entry('A', 'А'),
|
||||
Map.entry('e', 'е'), Map.entry('E', 'Е'),
|
||||
Map.entry('o', 'о'), Map.entry('O', 'О'),
|
||||
Map.entry('p', 'р'), Map.entry('P', 'Р'),
|
||||
Map.entry('c', 'с'), Map.entry('C', 'С'),
|
||||
Map.entry('y', 'у'), Map.entry('Y', 'У'),
|
||||
Map.entry('x', 'х'), Map.entry('X', 'Х'),
|
||||
Map.entry('k', 'к'), Map.entry('K', 'К'),
|
||||
Map.entry('m', 'м'), Map.entry('M', 'М'),
|
||||
Map.entry('h', 'н'), Map.entry('H', 'Н'),
|
||||
Map.entry('t', 'т'), Map.entry('T', 'Т'),
|
||||
Map.entry('b', 'в'), Map.entry('B', 'В')
|
||||
);
|
||||
private AhoCorasick ahoCorasick;
|
||||
|
||||
public SwearFilter(FileConfiguration config, JavaPlugin plugin) {
|
||||
this.enabled = config.getBoolean("filters.swear.enabled", true);
|
||||
this.replaceMode = config.getBoolean("filters.swear.replace", false);
|
||||
this.replacement = config.getString("filters.swear.replacement", "***");
|
||||
|
||||
|
||||
if (enabled) {
|
||||
loadWordList(plugin);
|
||||
}
|
||||
}
|
||||
|
||||
private void loadWordList(JavaPlugin plugin) {
|
||||
List<String> words = new ArrayList<>();
|
||||
try (InputStream is = plugin.getResource("badwords.txt");
|
||||
BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) {
|
||||
|
||||
reader.lines()
|
||||
.map(String::trim)
|
||||
.filter(line -> !line.isEmpty() && !line.startsWith("#"))
|
||||
.forEach(word -> {
|
||||
// Создаем паттерн с учетом возможной замены букв
|
||||
String regex = buildFlexiblePattern(word);
|
||||
swearPatterns.add(Pattern.compile(regex, Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE));
|
||||
});
|
||||
.forEach(words::add);
|
||||
} catch (Exception e) {
|
||||
plugin.getLogger().warning("Failed to load badwords.txt: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Создает гибкий паттерн, который ловит замены букв (х->x, о->o и т.д.)
|
||||
*/
|
||||
private String buildFlexiblePattern(String word) {
|
||||
StringBuilder pattern = new StringBuilder();
|
||||
|
||||
for (char c : word.toLowerCase().toCharArray()) {
|
||||
// Для каждой буквы создаем группу с возможными заменами
|
||||
switch (c) {
|
||||
case 'а' -> pattern.append("[аaА@]");
|
||||
case 'е' -> pattern.append("[еeЕ3]");
|
||||
case 'о' -> pattern.append("[оoО0]");
|
||||
case 'р' -> pattern.append("[рpР]");
|
||||
case 'с' -> pattern.append("[сcС]");
|
||||
case 'у' -> pattern.append("[уyУ]");
|
||||
case 'х' -> pattern.append("[хxХ]");
|
||||
case 'к' -> pattern.append("[кkК]");
|
||||
case 'м' -> pattern.append("[мmМ]");
|
||||
case 'н' -> pattern.append("[нhНN]");
|
||||
case 'т' -> pattern.append("[тtТ]");
|
||||
case 'в' -> pattern.append("[вbВ]");
|
||||
case 'и' -> pattern.append("[иiИ1]");
|
||||
case 'з' -> pattern.append("[з3З]");
|
||||
case 'б' -> pattern.append("[б6Б]");
|
||||
default -> pattern.append(Pattern.quote(String.valueOf(c)));
|
||||
}
|
||||
}
|
||||
|
||||
return pattern.toString();
|
||||
this.ahoCorasick = new AhoCorasick(words);
|
||||
}
|
||||
|
||||
public FilterResult filter(Player player, String message) {
|
||||
|
|
@ -100,23 +49,13 @@ public class SwearFilter {
|
|||
return FilterResult.ok(message);
|
||||
}
|
||||
|
||||
for (Pattern pattern : swearPatterns) {
|
||||
if (pattern.matcher(message).find()) {
|
||||
if (replaceMode) {
|
||||
return FilterResult.ok(replaceSwear(message));
|
||||
}
|
||||
return FilterResult.blocked("Сообщение содержит запрещенные слова");
|
||||
if (ahoCorasick.matches(message)) {
|
||||
if (replaceMode) {
|
||||
return FilterResult.ok(ahoCorasick.replaceAll(message, replacement));
|
||||
}
|
||||
return FilterResult.blocked("Сообщение содержит запрещенные слова");
|
||||
}
|
||||
|
||||
return FilterResult.ok(message);
|
||||
}
|
||||
|
||||
private String replaceSwear(String message) {
|
||||
String result = message;
|
||||
for (Pattern pattern : swearPatterns) {
|
||||
result = pattern.matcher(result).replaceAll(replacement);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
172
src/main/java/com/loki/lochat/utils/text/AhoCorasick.java
Normal file
172
src/main/java/com/loki/lochat/utils/text/AhoCorasick.java
Normal file
|
|
@ -0,0 +1,172 @@
|
|||
package com.loki.lochat.utils.text;
|
||||
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Deque;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public final class AhoCorasick {
|
||||
|
||||
private static final Map<Character, Character> NORMALIZE = new HashMap<>();
|
||||
|
||||
static {
|
||||
NORMALIZE.put('a', 'а');
|
||||
NORMALIZE.put('A', 'А');
|
||||
NORMALIZE.put('e', 'е');
|
||||
NORMALIZE.put('E', 'Е');
|
||||
NORMALIZE.put('o', 'о');
|
||||
NORMALIZE.put('O', 'О');
|
||||
NORMALIZE.put('p', 'р');
|
||||
NORMALIZE.put('P', 'Р');
|
||||
NORMALIZE.put('c', 'с');
|
||||
NORMALIZE.put('C', 'С');
|
||||
NORMALIZE.put('y', 'у');
|
||||
NORMALIZE.put('Y', 'У');
|
||||
NORMALIZE.put('x', 'х');
|
||||
NORMALIZE.put('X', 'Х');
|
||||
NORMALIZE.put('k', 'к');
|
||||
NORMALIZE.put('K', 'К');
|
||||
NORMALIZE.put('m', 'м');
|
||||
NORMALIZE.put('M', 'М');
|
||||
NORMALIZE.put('h', 'н');
|
||||
NORMALIZE.put('H', 'Н');
|
||||
NORMALIZE.put('n', 'н');
|
||||
NORMALIZE.put('N', 'Н');
|
||||
NORMALIZE.put('t', 'т');
|
||||
NORMALIZE.put('T', 'Т');
|
||||
NORMALIZE.put('b', 'в');
|
||||
NORMALIZE.put('B', 'В');
|
||||
|
||||
NORMALIZE.put('@', 'а');
|
||||
NORMALIZE.put('0', 'о');
|
||||
NORMALIZE.put('3', 'е');
|
||||
NORMALIZE.put('1', 'и');
|
||||
NORMALIZE.put('6', 'б');
|
||||
}
|
||||
|
||||
private final Node root;
|
||||
|
||||
public AhoCorasick(List<String> words) {
|
||||
this.root = new Node((char) 0);
|
||||
buildTrie(words);
|
||||
buildFailLinks();
|
||||
}
|
||||
|
||||
private void buildTrie(List<String> words) {
|
||||
for (String word : words) {
|
||||
if (word.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
Node node = root;
|
||||
for (char c : word.toCharArray()) {
|
||||
char normalized = normalize(c);
|
||||
Node current = node;
|
||||
node = node.children.computeIfAbsent(normalized, ch -> new Node(ch, current));
|
||||
}
|
||||
node.output = true;
|
||||
node.word = word;
|
||||
}
|
||||
}
|
||||
|
||||
private void buildFailLinks() {
|
||||
Deque<Node> queue = new ArrayDeque<>();
|
||||
for (Node node : root.children.values()) {
|
||||
node.fail = root;
|
||||
queue.addLast(node);
|
||||
}
|
||||
while (!queue.isEmpty()) {
|
||||
Node current = queue.pollFirst();
|
||||
for (Node child : current.children.values()) {
|
||||
Node fail = current.fail;
|
||||
while (fail != root && !fail.children.containsKey(child.c)) {
|
||||
fail = fail.fail;
|
||||
}
|
||||
if (fail != root || fail.children.containsKey(child.c)) {
|
||||
Node f = fail.children.get(child.c);
|
||||
child.fail = f != null ? f : root;
|
||||
} else {
|
||||
child.fail = root;
|
||||
}
|
||||
child.output = child.output || child.fail.output;
|
||||
if (child.fail.output && child.word == null) {
|
||||
child.word = child.fail.word;
|
||||
}
|
||||
queue.addLast(child);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public boolean matches(String text) {
|
||||
Node node = root;
|
||||
for (int i = 0; i < text.length(); i++) {
|
||||
char c = normalize(text.charAt(i));
|
||||
while (node != root && !node.children.containsKey(c)) {
|
||||
node = node.fail;
|
||||
}
|
||||
node = node.children.getOrDefault(c, root);
|
||||
if (node.output) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public String replaceAll(String text, String replacement) {
|
||||
Node node = root;
|
||||
int lastEnd = 0;
|
||||
StringBuilder result = new StringBuilder();
|
||||
for (int i = 0; i < text.length(); i++) {
|
||||
char c = normalize(text.charAt(i));
|
||||
while (node != root && !node.children.containsKey(c)) {
|
||||
node = node.fail;
|
||||
}
|
||||
node = node.children.getOrDefault(c, root);
|
||||
if (node.output && node.word != null) {
|
||||
result.append(text, lastEnd, i - node.word.length() + 1);
|
||||
result.append(replacement);
|
||||
lastEnd = i + 1;
|
||||
}
|
||||
}
|
||||
result.append(text, lastEnd, text.length());
|
||||
return result.toString();
|
||||
}
|
||||
|
||||
public List<String> findMatches(String text) {
|
||||
List<String> found = new ArrayList<>();
|
||||
Node node = root;
|
||||
for (int i = 0; i < text.length(); i++) {
|
||||
char c = normalize(text.charAt(i));
|
||||
while (node != root && !node.children.containsKey(c)) {
|
||||
node = node.fail;
|
||||
}
|
||||
node = node.children.getOrDefault(c, root);
|
||||
if (node.output && node.word != null) {
|
||||
found.add(node.word);
|
||||
}
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
private static char normalize(char c) {
|
||||
return NORMALIZE.getOrDefault(c, Character.toLowerCase(c));
|
||||
}
|
||||
|
||||
private static final class Node {
|
||||
final char c;
|
||||
final Map<Character, Node> children = new HashMap<>();
|
||||
Node fail;
|
||||
boolean output;
|
||||
String word;
|
||||
|
||||
Node(char c) {
|
||||
this.c = c;
|
||||
this.fail = this;
|
||||
}
|
||||
|
||||
Node(char c, Node parent) {
|
||||
this.c = c;
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue