373 lines
13 KiB
Python
373 lines
13 KiB
Python
"""手动触发 GLM 评审（不写 eval_runs）。"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import re
|
||
from collections.abc import AsyncIterator
|
||
from typing import Any
|
||
|
||
from sqlalchemy.ext.asyncio import AsyncSession
|
||
|
||
from app.core.dependencies import get_eval_judge_langchain_llm
|
||
from app.core.logging import get_logger
|
||
from app.features.evaluation.errors import (
|
||
EvaluationBadRequestError,
|
||
EvaluationNotFoundError,
|
||
)
|
||
from app.features.evaluation.execution_service import _assistant_text_for_eval_display
|
||
from app.features.evaluation.judge_service import EvalJudgeService
|
||
from app.features.evaluation.schemas import MemoirSectionBaselineOut
|
||
from app.features.evaluation.session_catalog_service import SessionCatalogService
|
||
from app.features.evaluation.user_export_fixtures import read_user_export_fixture
|
||
from app.features.memoir.repo import get_chapters_for_memoir_list
|
||
from app.features.story.repo import get_stories_for_user
|
||
|
||
logger = get_logger(__name__)
|
||
|
||
_MAX_JUDGE_MARKDOWN_CHARS = 20_000
|
||
_MAX_EVAL_CHAPTERS = 30
|
||
_MAX_EVAL_STORIES = 40
|
||
|
||
|
||
def _clip_md_for_judge(text: str, max_chars: int = _MAX_JUDGE_MARKDOWN_CHARS) -> str:
|
||
s = (text or "").strip()
|
||
if len(s) <= max_chars:
|
||
return s
|
||
return f"{s[:max_chars]}\n\n…（已截断供评审）"
|
||
|
||
|
||
def _transcript_from_export_turns(turns: list[tuple[str, str]]) -> str:
|
||
parts: list[str] = []
|
||
for u, ai in turns:
|
||
u = (u or "").strip()
|
||
ai = (ai or "").strip()
|
||
if u:
|
||
parts.append(f"用户: {u}")
|
||
if ai:
|
||
parts.append(f"AI: {_assistant_text_for_eval_display(ai)}")
|
||
return "\n\n".join(parts)
|
||
|
||
|
||
def _normalize_title_key(title: str) -> str:
|
||
t = (title or "").strip().lower()
|
||
t = re.sub(r"^#+\s*", "", t)
|
||
return re.sub(r"\s+", " ", t)
|
||
|
||
|
||
def _baseline_for_chapter_title(
|
||
baselines: list[MemoirSectionBaselineOut],
|
||
chapter_title: str,
|
||
index: int,
|
||
) -> MemoirSectionBaselineOut | None:
|
||
if baselines:
|
||
key = _normalize_title_key(chapter_title)
|
||
for b in baselines:
|
||
if _normalize_title_key(b.title) == key:
|
||
return b
|
||
if 0 <= index < len(baselines):
|
||
return baselines[index]
|
||
return None
|
||
|
||
|
||
class EvalJudgeManualService:
|
||
def __init__(self, db: AsyncSession) -> None:
|
||
self._db = db
|
||
|
||
async def judge_conversation(
|
||
self,
|
||
conversation_id: str,
|
||
fixture_filename: str | None,
|
||
) -> dict[str, Any]:
|
||
cid = (conversation_id or "").strip()
|
||
if not cid:
|
||
raise EvaluationBadRequestError("conversation_id is required")
|
||
|
||
catalog = SessionCatalogService(self._db)
|
||
dialogue = await catalog.get_session_dialogue(cid)
|
||
if not dialogue:
|
||
raise EvaluationNotFoundError("conversation not found")
|
||
|
||
parts: list[str] = []
|
||
for m in dialogue.messages:
|
||
r = (m.role or "").lower()
|
||
label = "用户" if r == "human" else "AI"
|
||
raw = m.content or ""
|
||
out = _assistant_text_for_eval_display(raw) if r != "human" else raw
|
||
parts.append(f"{label}: {out}")
|
||
replay_transcript = "\n\n".join(parts)
|
||
if not replay_transcript.strip():
|
||
raise EvaluationBadRequestError("no messages to judge")
|
||
|
||
fn = (fixture_filename or "").strip() or None
|
||
baseline_transcript = ""
|
||
if fn:
|
||
try:
|
||
turns, _ = read_user_export_fixture(fn)
|
||
baseline_transcript = _transcript_from_export_turns(turns)
|
||
except ValueError as e:
|
||
raise EvaluationBadRequestError(str(e)) from e
|
||
except FileNotFoundError as e:
|
||
raise EvaluationNotFoundError("fixture not found") from e
|
||
|
||
errors: list[str] = []
|
||
judge_llm = get_eval_judge_langchain_llm()
|
||
judge = EvalJudgeService(judge_llm)
|
||
baseline_judge_dict: dict[str, Any] | None = None
|
||
if baseline_transcript.strip():
|
||
bj = await judge.judge_conversation(full_transcript=baseline_transcript)
|
||
if bj:
|
||
baseline_judge_dict = bj.model_dump()
|
||
else:
|
||
errors.append("baseline_glm_failed")
|
||
elif fn:
|
||
errors.append("baseline_transcript_empty")
|
||
|
||
rj = await judge.judge_conversation(full_transcript=replay_transcript)
|
||
replay_judge_dict = rj.model_dump() if rj else None
|
||
if not rj:
|
||
errors.append("replay_glm_failed")
|
||
|
||
return {
|
||
"conversation_id": cid,
|
||
"fixture_filename": fn,
|
||
"baseline_transcript": baseline_transcript,
|
||
"replay_transcript": replay_transcript,
|
||
"baseline_judge": baseline_judge_dict,
|
||
"replay_judge": replay_judge_dict,
|
||
"errors": errors,
|
||
}
|
||
|
||
async def iter_conversation_judge_sse(
|
||
self,
|
||
conversation_id: str,
|
||
fixture_filename: str | None,
|
||
) -> AsyncIterator[dict[str, Any]]:
|
||
"""供 SSE：先整体基准分、再整体回放分，再流式对比与建议。"""
|
||
cid = (conversation_id or "").strip()
|
||
if not cid:
|
||
yield {
|
||
"event": "error",
|
||
"phase": "validate",
|
||
"message": "conversation_id is required",
|
||
}
|
||
return
|
||
|
||
catalog = SessionCatalogService(self._db)
|
||
dialogue = await catalog.get_session_dialogue(cid)
|
||
if not dialogue:
|
||
yield {
|
||
"event": "error",
|
||
"phase": "load",
|
||
"message": "conversation not found",
|
||
}
|
||
return
|
||
|
||
parts: list[str] = []
|
||
for m in dialogue.messages:
|
||
r = (m.role or "").lower()
|
||
label = "用户" if r == "human" else "AI"
|
||
raw = m.content or ""
|
||
out = _assistant_text_for_eval_display(raw) if r != "human" else raw
|
||
parts.append(f"{label}: {out}")
|
||
replay_transcript = "\n\n".join(parts)
|
||
if not replay_transcript.strip():
|
||
yield {"event": "error", "phase": "load", "message": "no messages to judge"}
|
||
return
|
||
|
||
fn = (fixture_filename or "").strip() or None
|
||
baseline_transcript = ""
|
||
if fn:
|
||
try:
|
||
turns, _ = read_user_export_fixture(fn)
|
||
baseline_transcript = _transcript_from_export_turns(turns)
|
||
except ValueError as e:
|
||
yield {"event": "error", "phase": "fixture", "message": str(e)}
|
||
return
|
||
except FileNotFoundError:
|
||
yield {
|
||
"event": "error",
|
||
"phase": "fixture",
|
||
"message": "fixture not found",
|
||
}
|
||
return
|
||
|
||
judge_llm = get_eval_judge_langchain_llm()
|
||
if not judge_llm:
|
||
yield {
|
||
"event": "error",
|
||
"phase": "config",
|
||
"message": "评审 LLM 未配置（eval_judge_api_key / zhipu_api_key）",
|
||
}
|
||
return
|
||
|
||
judge = EvalJudgeService(judge_llm)
|
||
yield {"event": "meta", "conversation_id": cid, "fixture_filename": fn}
|
||
|
||
if not baseline_transcript.strip():
|
||
yield {
|
||
"event": "warning",
|
||
"message": "未提供基准 MD 或基准无文本：仅对回放对话打分并输出单侧改进建议",
|
||
}
|
||
|
||
baseline_judge = None
|
||
if baseline_transcript.strip():
|
||
baseline_judge = await judge.judge_conversation(
|
||
full_transcript=baseline_transcript
|
||
)
|
||
yield {
|
||
"event": "baseline_judge",
|
||
"ok": baseline_judge is not None,
|
||
"judge": baseline_judge.model_dump() if baseline_judge else None,
|
||
}
|
||
if not baseline_judge:
|
||
yield {
|
||
"event": "error",
|
||
"phase": "baseline_glm",
|
||
"message": "基准整体打分失败（密钥、限流或 JSON 解析失败，见服务端日志）",
|
||
}
|
||
else:
|
||
yield {
|
||
"event": "baseline_judge",
|
||
"ok": False,
|
||
"skipped": True,
|
||
"judge": None,
|
||
}
|
||
|
||
replay_judge = await judge.judge_conversation(full_transcript=replay_transcript)
|
||
yield {
|
||
"event": "replay_judge",
|
||
"ok": replay_judge is not None,
|
||
"judge": replay_judge.model_dump() if replay_judge else None,
|
||
}
|
||
if not replay_judge:
|
||
yield {
|
||
"event": "error",
|
||
"phase": "replay_glm",
|
||
"message": "回放对话整体 GLM 打分失败（空密钥、限流或 JSON 解析失败，见服务端日志）",
|
||
}
|
||
yield {"event": "done"}
|
||
return
|
||
|
||
async for piece in judge.stream_conversation_compare(
|
||
baseline_transcript=baseline_transcript,
|
||
replay_transcript=replay_transcript,
|
||
baseline_judge=baseline_judge,
|
||
replay_judge=replay_judge,
|
||
):
|
||
if piece:
|
||
yield {"event": "compare_delta", "text": piece}
|
||
|
||
yield {"event": "done"}
|
||
|
||
async def judge_memoir_for_user(
|
||
self,
|
||
user_id: str,
|
||
baseline_sections: list[MemoirSectionBaselineOut] | None,
|
||
) -> dict[str, Any]:
|
||
uid = (user_id or "").strip()
|
||
if not uid:
|
||
raise EvaluationBadRequestError("user_id is required")
|
||
|
||
judge_llm = get_eval_judge_langchain_llm()
|
||
judge = EvalJudgeService(judge_llm)
|
||
baselines = list(baseline_sections or [])
|
||
|
||
chapter_results: list[dict[str, Any]] = []
|
||
try:
|
||
chapters = await get_chapters_for_memoir_list(
|
||
uid, self._db, active_only=True, is_new_only=None
|
||
)
|
||
for i, ch in enumerate(chapters[:_MAX_EVAL_CHAPTERS]):
|
||
body = (ch.canonical_markdown or "").strip()
|
||
if not body:
|
||
continue
|
||
bl = _baseline_for_chapter_title(baselines, ch.title or "", i)
|
||
baseline_excerpt = ""
|
||
if bl and (bl.body or "").strip():
|
||
baseline_excerpt = _clip_md_for_judge(bl.body, max_chars=6000)
|
||
md = f"# 章节：{ch.title}\n\n"
|
||
if baseline_excerpt:
|
||
md += f"## 导出基线（节选）\n\n{baseline_excerpt}\n\n"
|
||
md += f"## 当前成稿\n\n{_clip_md_for_judge(body)}"
|
||
cj = await judge.judge_memoir(memoir_markdown=md)
|
||
chapter_results.append(
|
||
{
|
||
"id": ch.id,
|
||
"title": ch.title,
|
||
"order_index": ch.order_index,
|
||
"baseline_title": bl.title if bl else None,
|
||
"judge": cj.model_dump() if cj else None,
|
||
}
|
||
)
|
||
except Exception as e:
|
||
logger.warning("manual memoir chapter judges failed: {}", e)
|
||
|
||
story_results: list[dict[str, Any]] = []
|
||
try:
|
||
stories = await get_stories_for_user(self._db, uid, status="active")
|
||
for st in stories[:_MAX_EVAL_STORIES]:
|
||
body = (st.canonical_markdown or "").strip()
|
||
if not body:
|
||
continue
|
||
md = f"# 故事：{st.title}\n\n{_clip_md_for_judge(body)}"
|
||
sj = await judge.judge_memoir(memoir_markdown=md)
|
||
story_results.append(
|
||
{
|
||
"id": st.id,
|
||
"title": st.title,
|
||
"stage": st.stage,
|
||
"judge": sj.model_dump() if sj else None,
|
||
}
|
||
)
|
||
except Exception as e:
|
||
logger.warning("manual memoir story judges failed: {}", e)
|
||
|
||
return {
|
||
"user_id": uid,
|
||
"chapter_results": chapter_results,
|
||
"story_results": story_results,
|
||
}
|
||
|
||
async def memoir_snapshot(self, user_id: str) -> dict[str, Any]:
|
||
uid = (user_id or "").strip()
|
||
if not uid:
|
||
raise EvaluationBadRequestError("user_id is required")
|
||
|
||
chapters_out: list[dict[str, Any]] = []
|
||
stories_out: list[dict[str, Any]] = []
|
||
try:
|
||
chapters = await get_chapters_for_memoir_list(
|
||
uid, self._db, active_only=True, is_new_only=None
|
||
)
|
||
for ch in chapters[:_MAX_EVAL_CHAPTERS]:
|
||
chapters_out.append(
|
||
{
|
||
"id": ch.id,
|
||
"title": ch.title,
|
||
"category": ch.category,
|
||
"order_index": ch.order_index,
|
||
"canonical_markdown": ch.canonical_markdown,
|
||
}
|
||
)
|
||
except Exception as e:
|
||
logger.warning("memoir snapshot chapters failed: {}", e)
|
||
try:
|
||
stories = await get_stories_for_user(self._db, uid, status="active")
|
||
for st in stories[:_MAX_EVAL_STORIES]:
|
||
stories_out.append(
|
||
{
|
||
"id": st.id,
|
||
"title": st.title,
|
||
"stage": st.stage,
|
||
"canonical_markdown": st.canonical_markdown,
|
||
}
|
||
)
|
||
except Exception as e:
|
||
logger.warning("memoir snapshot stories failed: {}", e)
|
||
|
||
return {
|
||
"user_id": uid,
|
||
"chapters": chapters_out,
|
||
"stories": stories_out,
|
||
}
|