93 lines
3.2 KiB
Python
93 lines
3.2 KiB
Python
|
"""
|
|||
|
|
口述归一：在进入叙事与忠实度校验前，对同一段文本做可控预处理（规则 / 可选 LLM）。
|
|||
|
|
|
|||
|
|
不改变 segment 落库原文；仅作为 memoir story 生成路径的派生输入。
|
|||
|
|
|
|||
|
|
规则层与聊天侧共用 `apply_conversation_input_rules`（见 conversation.input_normalize）。
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
from __future__ import annotations
|
|||
|
|
|
|||
|
|
import json
|
|||
|
|
from typing import Any
|
|||
|
|
|
|||
|
|
from app.core.config import settings
|
|||
|
|
from app.core.langchain_llm import invoke_json_object
|
|||
|
|
from app.core.logging import get_logger
|
|||
|
|
from app.features.conversation.input_normalize import apply_conversation_input_rules
|
|||
|
|
from app.features.memoir.memoir_images.json_payload import extract_json_payload
|
|||
|
|
|
|||
|
|
logger = get_logger(__name__)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def apply_oral_normalization_rules(text: str) -> str:
|
|||
|
|
"""确定性规则；与 `apply_conversation_input_rules` 等价（memoir 历史名保留）。"""
|
|||
|
|
return apply_conversation_input_rules(text)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _llm_normalize_oral(text: str, llm: Any) -> str | None:
|
|||
|
|
"""仅修正明显错字与同音字，不增事实；失败返回 None。"""
|
|||
|
|
if not llm or not (text or "").strip():
|
|||
|
|
return None
|
|||
|
|
max_in = int(settings.memoir_oral_normalize_llm_max_input_chars)
|
|||
|
|
t = (text or "").strip()
|
|||
|
|
if len(t) > max_in:
|
|||
|
|
logger.debug(
|
|||
|
|
"event=oral_normalize_llm_skip reason=input_too_long len={} max={}",
|
|||
|
|
len(t),
|
|||
|
|
max_in,
|
|||
|
|
)
|
|||
|
|
return None
|
|||
|
|
prompt = f"""你是口述转写纠错助手。只修正明显的同音错别字、别字与标点，使句子通顺可读。
|
|||
|
|
禁止增加事实、不补充细节、不摘要、不改写句式风格；不得新增人名、地名、数字、事件。
|
|||
|
|
若原文已通顺或无法确定错误，则照抄输入。
|
|||
|
|
|
|||
|
|
【用户口述】
|
|||
|
|
{t}
|
|||
|
|
|
|||
|
|
**JSON 输出**：只输出一个合法 JSON 对象。
|
|||
|
|
{{"normalized_text": "纠错后的完整文本（与输入等意，仅修错字与标点）"}}
|
|||
|
|
|
|||
|
|
只输出 JSON，不要其它文字。"""
|
|||
|
|
try:
|
|||
|
|
raw = invoke_json_object(
|
|||
|
|
llm,
|
|||
|
|
prompt,
|
|||
|
|
max_tokens=int(settings.memoir_oral_normalize_llm_max_tokens),
|
|||
|
|
agent="oral_normalize.llm",
|
|||
|
|
)
|
|||
|
|
data = json.loads(extract_json_payload(raw))
|
|||
|
|
if not isinstance(data, dict):
|
|||
|
|
return None
|
|||
|
|
out = (data.get("normalized_text") or "").strip()
|
|||
|
|
if not out:
|
|||
|
|
return None
|
|||
|
|
return out
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.warning("oral_normalize LLM 失败，回退规则结果: {}", e)
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
|
|||
|
|
def normalize_oral_for_memoir(text: str, *, llm: Any | None = None) -> str:
|
|||
|
|
"""
|
|||
|
|
供 story pipeline 单一出口：叙事与忠实度使用同一返回值。
|
|||
|
|
|
|||
|
|
- off / 全局关闭：原文
|
|||
|
|
- rules：仅规则
|
|||
|
|
- rules + LLM 分支：先规则，再（可选）LLM；LLM 失败则保留规则结果
|
|||
|
|
"""
|
|||
|
|
if not settings.memoir_oral_normalize_enabled:
|
|||
|
|
return text or ""
|
|||
|
|
mode = (settings.memoir_oral_normalize_mode or "rules").strip().lower()
|
|||
|
|
if mode == "off":
|
|||
|
|
return text or ""
|
|||
|
|
|
|||
|
|
base = apply_oral_normalization_rules(text or "")
|
|||
|
|
if mode != "llm":
|
|||
|
|
return base
|
|||
|
|
|
|||
|
|
refined = _llm_normalize_oral(base, llm)
|
|||
|
|
if refined is not None:
|
|||
|
|
return refined
|
|||
|
|
return base
|