这是文本自动审核服务，包含敏感词检测、规则识别、风险打分和人工复审逻辑。

```python

-- coding: utf-8 --

"""

文本自动审核引擎

敏感词 DFA 多模匹配(分级加权)
正则规则(联系方式/广告/灌水)
抗绕过:全角、零宽字符、干扰符
输出:pass / review / reject + 命中明细 + 打码文本

"""

from future import annotations

import re

import unicodedata

from dataclasses import dataclass, field, asdict

from typing import Iterable

---------------- 常量 ----------------

PASS = "pass" # 通过

REVIEW = "review" # 转人工复审

REJECT = "reject" # 拒绝

MAX_SCORE = 100

DEFAULT_REJECT_THRESHOLD = 80

DEFAULT_REVIEW_THRESHOLD = 30

敏感词等级 -> 风险分

SCORE_BY_LEVEL = {1: 15, 2: 45, 3: 100}

---------------- 词库 ----------------

⚠️ 生产环境请替换成你自己的词库,并从 DB / Redis 加载、支持热更新

DEFAULT_WORDS: dict $str, int$ = {

level 3 ------ 严重违规,命中即拒绝

"赌博": 3, "博彩": 3, "私彩": 3, "六合彩": 3,

"毒品": 3, "冰毒": 3, "大麻": 3, "迷药": 3,

"枪支": 3, "弹药": 3, "假币": 3, "假钞": 3,

"代开发票": 3, "办证": 3, "假证": 3,

"色情": 3, "成人网站": 3, "一夜情": 3, "约炮": 3,

"黑客": 3, "代考": 3, "包过": 3,

level 2 ------ 广告 / 引流

"刷单": 2, "兼职刷单": 2, "刷信誉": 2,

"加微信": 2, "私聊我": 2, "私信我": 2,

"贷款": 2, "套现": 2, "花呗套现": 2, "无抵押": 2,

"高薪招聘": 2, "日结": 2, "返利": 2,

level 1 ------ 轻度

"傻逼": 1, "去死": 1, "滚蛋": 1,

}

---------------- 正则规则 ----------------

@dataclass(frozen=True)

class RegexRule:

name: str

pattern: re.Pattern

score: int

max_hits: int = 3 # 同一规则最多计分次数,避免刷分

DEFAULT_REGEX_RULES: list $RegexRule$ = [

RegexRule("手机号", re.compile(r"(?<!\d)1 $3-9$ \d{9}(?!\d)"), 40),

RegexRule("QQ号", re.compile(r"(?:qq|扣扣|企鹅)\s* $:：$ ?\s*\d{5,12}", re.I), 35),

RegexRule("微信号", re.compile(r"(?:微信|weixin|vx|v信|薇信|威信)\s* $:：$ ?\s* $a-zA-Z$ $a-zA-Z0-9_-$ {4,19}"), 35),

RegexRule("网址", re.compile(r"(?:https?://|www\.) $\\w\\-.$ +\. $a-zA-Z$ {2,}(?:/\S*)?", re.I), 30),

RegexRule("邮箱", re.compile(r" $\\w.+-$ +@ $\\w-$ +\. $a-zA-Z$ {2,}"), 25),

RegexRule("银行卡号", re.compile(r"(?<!\d)\d{16,19}(?!\d)"), 50),

RegexRule("灌水重复", re.compile(r"(.)\1{7,}"), 20, 1),

]

---------------- 文本归一化 ----------------

_ZERO_WIDTH = re.compile(r" $\\u200b-\\u200f\\u202a-\\u202e\\u2060\\ufeff$ ")

常见用于绕过检测的干扰符号

_INTERFERENCE = re.compile(

r" $\\s\\\*\\.\\-_\~·、,，。!！?？@#￥$%\^\&+\|=／/\\\\\\\[\\$ 【】()（）<>《》\"'""'':：;；]+"

)

def normalize(text: str) -> str:

"""全角转半角 + 去掉零宽字符。注意:可能改变字符串长度。"""

text = unicodedata.normalize("NFKC", text)

return _ZERO_WIDTH.sub("", text)

def deobfuscate(text: str) -> str:

"""去掉空格、星号等干扰符,用于检测 '赌博' / '赌*博' 这类变体。"""

return _INTERFERENCE.sub("", text)

---------------- 结果模型 ----------------

@dataclass

class Hit:

kind: str # 命中类型:sensitive_word / rule:xxx / remote:xxx

detail: str # 命中的具体内容

score: int # 风险分

start: int = -1 # 在归一化文本中的位置(仅本地规则有效)

end: int = -1

@dataclass

class AuditResult:

action: str

score: int

hits: list $Hit$ = field(default_factory=list)

normalized_text: str = ""

filtered_text: str = "" # 命中部分打码后的文本

def to_dict(self) -> dict:

return {

"action": self.action,

"score": self.score,

"hits": $asdict(h) for h in self.hits$ ,

"filtered_text": self.filtered_text,

}

def mask_spans(text: str, spans: Iterable $tuple\[int, int$ ]) -> str:

"""把命中的位置替换成 *"""

chars = list(text)

n = len(chars)

for s, e in spans:

for k in range(max(0, s), min(n, e)):

chars $k$ = "*"

return "".join(chars)

---------------- DFA 敏感词匹配 ----------------

class DFAFilter:

"""基于 Trie 的多模匹配,最长匹配优先。时间复杂度 O(n * 最长词长)。"""

slots = ("root",)

def init(self) -> None:

self.root: dict = {}

def add(self, word: str) -> None:

node = self.root

for ch in word:

node = node.setdefault(ch, {})

node $"\\x00"$ = True # 结束标记

def build(self, words: Iterable $str$ ) -> "DFAFilter":

for w in words:

w = normalize(w).strip()

if w:

self.add(w)

return self

def match(self, text: str) -> list $tuple\[int, int, str$ ]:

"""返回 $(start, end, word)$ """

hits: list $tuple\[int, int, str$ ] = \[\]

n = len(text)

i = 0

while i < n:

node = self.root

j = i

last_end = -1

while j < n:

nxt = node.get(text $j$ )

if nxt is None:

break

node = nxt

j += 1

if "\x00" in node:

last_end = j # 记录最近一次完整匹配

if last_end > i:

hits.append((i, last_end, text $i:last_end$ ))

i = last_end # 跳过已匹配部分

else:

i += 1

return hits

---------------- 审核引擎 ----------------

class TextModerator:

def init(

self,

words: dict $str, int$ | None = None,

regex_rules: list $RegexRule$ | None = None,

reject_threshold: int = DEFAULT_REJECT_THRESHOLD,

review_threshold: int = DEFAULT_REVIEW_THRESHOLD,

) -> None:

self.words = self._normalize_words(DEFAULT_WORDS if words is None else words)

self.dfa = DFAFilter().build(self.words)

self.regex_rules = DEFAULT_REGEX_RULES if regex_rules is None else regex_rules

self.reject_threshold = reject_threshold

self.review_threshold = review_threshold

@staticmethod

def _normalize_words(words: dict $str, int$ ) -> dict $str, int$ :

out: dict $str, int$ = {}

for k, v in words.items():

k = normalize(k).strip()

if k:

out $k$ = v

return out

def reload_words(self, words: dict $str, int$ ) -> None:

"""热更新词库(可对接 Redis 订阅 / 定时拉取)"""

self.words = self._normalize_words(words)

self.dfa = DFAFilter().build(self.words)

---------- 决策 ----------

def decide(self, score: int, hits: list $Hit$ ) -> str:

if any(h.score >= MAX_SCORE for h in hits):

return REJECT

if score >= self.reject_threshold:

return REJECT

if score >= self.review_threshold:

return REVIEW

return PASS

---------- 主流程 ----------

def audit(self, text: str) -> AuditResult:

if not text or not text.strip():

return AuditResult(action=PASS, score=0)

norm = normalize(text)

hits: list $Hit$ = \[\]

spans: list $tuple\[int, int$ ] = \[\]

score = 0

1) 敏感词

for start, end, word in self.dfa.match(norm):

level = self.words.get(word, 2)

s = SCORE_BY_LEVEL.get(level, 45)

hits.append(Hit("sensitive_word", word, s, start, end))

spans.append((start, end))

score += s

2) 抗绕过:去掉干扰符后再匹配一次(位置无法映射回原文,故不打码)

stripped = deobfuscate(norm)

if stripped != norm:

for _, _, word in self.dfa.match(stripped):

hits.append(Hit("sensitive_word_obfuscated", word, 60))

score += 60

3) 正则规则

for rule in self.regex_rules:

count = 0

for m in rule.pattern.finditer(norm):

if count >= rule.max_hits:

break

count += 1

hits.append(Hit(f"rule:{rule.name}", m.group(), rule.score, m.start(), m.end()))

spans.append((m.start(), m.end()))

score += rule.score

score = min(score, MAX_SCORE)

return AuditResult(

action=self.decide(score, hits),

score=score,

hits=hits,

normalized_text=norm,

filtered_text=mask_spans(norm, spans),

)

单例,供接口层复用

moderator = TextModerator()

```

审核流程与操作说明

这块代码把自动审核拆成了三道关卡，下面具体说说每一步在干什么。

· 敏感词分级检测：命中敏感词时按等级加权扣分，3 级词汇直接触发拒绝，2 级和 1 级累积风险分。

· 格式规则识别：内置手机号、QQ号、微信号、网址等正则，同一规则最多计分 3 次，避免刷分。

· 抗绕过处理：先做全角转半角、去掉零宽字符，再把文字里的空格、星号等干扰符去掉后二次匹配，防止"赌博""赌*博"这类变体。

· 自动决策：根据最终得分落在 $0,30)、\[30,80)、\[80,100$ 区间，分别给出通过、人工复审、拒绝三种结果，并输出命中明细和打码后的文本。

优化建议：建议您将 DEFAULT_WORDS 里的示例词替换为自己的词库，并接入数据库或 Redis 支持热更新；也可按需调整拒绝和复审的分数阈值。

文章仅供参考用。

Python文字审核模型

-*- coding: utf-8 -*-