life-echo/api/app/features/evaluation/execution_service.py

"""执行单次评测 run 与整实验（供 Celery / 内联调试）。"""

from __future__ import annotations

from datetime import datetime, timezone
from typing import Any

from sqlalchemy.ext.asyncio import AsyncSession

from app.core.config import settings
from app.core.db import AsyncSessionLocal
from app.core.dependencies import get_eval_judge_langchain_llm, get_llm_provider
from app.core.logging import get_logger
from app.features.evaluation import repo as eval_repo
from app.features.evaluation.candidate_runner import (
    EvalCandidateRunner,
    simple_memoir_from_transcript,
)
from app.features.evaluation.eval_trace_service import EvalTraceService
from app.features.evaluation.gate_report_service import gate_result_to_details
from app.features.evaluation.gating_service import compute_gate
from app.features.evaluation.judge_service import EvalJudgeService
from app.features.evaluation.models import EvalCase, EvalRun, EvalVersion
from app.features.evaluation.transcript_for_judge import (
    assistant_text_for_eval_display,
    format_eval_turn_block,
)

logger = get_logger(__name__)

_MAX_JUDGE_MARKDOWN_CHARS = 20_000
_MAX_EVAL_CHAPTERS = 30
_MAX_EVAL_STORIES = 40
_MAX_EVIDENCE_TRANSCRIPT_CHARS = 16_000


def _clip_md_for_judge(text: str, max_chars: int = _MAX_JUDGE_MARKDOWN_CHARS) -> str:
    s = (text or "").strip()
    if len(s) <= max_chars:
        return s
    return f"{s[:max_chars]}\n\n…（已截断供评审）"


def _composite(
    conv: float | None, mem: float | None, weights: dict[str, Any] | None
) -> float | None:
    """合成总分；缺失的一侧不计为 0，避免把评审失败误标为极差。

    仅一侧有分：返回该侧原始分（不乘权重），表示当前 run 仅完成了部分评审维度。
    """
    w = weights or {}
    wc = float(w.get("conversation", 0.5))
    wm = float(w.get("memoir", 0.5))
    has_c = conv is not None
    has_m = mem is not None
    if not has_c and not has_m:
        return None
    if has_c and has_m:
        return float(wc) * float(conv) + float(wm) * float(mem)
    if has_c:
        return float(conv)
    return float(mem)


def _utterances_for_case(case: EvalCase) -> list[str]:
    raw = case.user_utterances or []
    return [str(u).strip() for u in raw if str(u).strip()]


def _trim_evidence_text(text: str, max_chars: int = _MAX_EVIDENCE_TRANSCRIPT_CHARS) -> str:
    s = (text or "").strip()
    if len(s) <= max_chars:
        return s
    return f"{s[:max_chars]}\n\n…（访谈证据已截断）"


async def execute_eval_run(
    db: AsyncSession,
    *,
    run: EvalRun,
    case: EvalCase,
    version: EvalVersion,
) -> None:
    fresh_run = await db.get(EvalRun, run.id)
    if not fresh_run:
        return
    if (fresh_run.status or "").lower() == "completed":
        logger.info("eval run skip already completed run_id={}", fresh_run.id)
        return
    run = fresh_run

    if not settings.eval_execution_enabled:
        await eval_repo.update_run(
            db,
            run,
            status="failed",
            error_message="EVAL_EXECUTION_ENABLED=false",
            completed_at=datetime.now(timezone.utc),
        )
        return

    utterances = _utterances_for_case(case)
    if not utterances:
        await eval_repo.update_run(
            db,
            run,
            status="failed",
            error_message="empty user_utterances",
            completed_at=datetime.now(timezone.utc),
        )
        return

    await eval_repo.update_run(
        db,
        run,
        status="running",
        started_at=datetime.now(timezone.utc),
        error_message=None,
    )
    await db.commit()

    provider_llm = getattr(get_llm_provider(), "langchain_llm", None)
    if provider_llm is None:
        await eval_repo.update_run(
            db,
            run,
            status="failed",
            error_message="生产 LLM 未配置",
            completed_at=datetime.now(timezone.utc),
        )
        await db.commit()
        return

    judge_llm = get_eval_judge_langchain_llm()
    judge = EvalJudgeService(judge_llm)
    runner = EvalCandidateRunner(provider_llm)
    cfg = version.config_json if isinstance(version.config_json, dict) else None

    try:
        replies, latencies = await runner.replay_utterances(
            utterances,
            version_config=cfg,
            temperature=settings.eval_candidate_temperature,
        )
    except Exception as e:
        logger.exception("eval replay failed: {}", e)
        await eval_repo.update_run(
            db,
            run,
            status="failed",
            error_message=str(e)[:2000],
            completed_at=datetime.now(timezone.utc),
        )
        await db.commit()
        return

    transcript_parts: list[str] = []
    for i, u in enumerate(utterances):
        if i >= len(replies):
            break
        transcript_parts.append(
            format_eval_turn_block(i, u, assistant_text_for_eval_display(replies[i]))
        )
    prior_blocks: list[str] = []
    for idx, u in enumerate(utterances):
        if idx >= len(replies):
            break
        reply = assistant_text_for_eval_display(replies[idx])
        lat = latencies[idx] if idx < len(latencies) else None
        prior = "\n\n".join(prior_blocks)
        if len(prior) > 8000:
            prior = prior[-8000:]
        tj = await judge.judge_turn(
            prior_transcript=prior,
            user_utterance=u,
            assistant_reply=reply,
            turn_index_0=idx,
        )
        scores = tj.model_dump() if tj else None
        rationale = tj.rationale if tj else None
        await eval_repo.add_turn(
            db,
            run_id=str(run.id),
            turn_index=idx,
            user_utterance=u,
            assistant_reply=replies[idx],
            duration_ms=lat,
            judge_scores_json=scores,
            judge_rationale=rationale,
        )
        await db.commit()
        prior_blocks.append(format_eval_turn_block(idx, u, reply))

    full_transcript = "\n\n".join(transcript_parts)
    conv_out = await judge.judge_conversation(full_transcript=full_transcript)
    conv_total = conv_out.total_score if conv_out else None

    memoir_md = simple_memoir_from_transcript(utterances, replies)
    source_transcript = _trim_evidence_text(full_transcript)
    reference_memoir = (case.reference_memoir_markdown or "").strip()
    synthetic_notes = (
        "本项为 replay 合成的短 memoir：证据闭包仅为重放对话 transcript（无 library artifact lineage）。"
        f" turns={len(utterances)}"
    )
    mem_out = await judge.judge_memoir(
        memoir_markdown=memoir_md,
        source_transcript=source_transcript,
        structured_evidence=(
            "（结构化记忆证据：自动化 replay 路径未绑定用户 memory chunk/fact/timeline/summary。）"
        ),
        reference_memoir_markdown=reference_memoir,
        evidence_notes=synthetic_notes,
    )

    chapter_entries: list[dict[str, Any]] = []
    story_entries: list[dict[str, Any]] = []
    uid = (case.source_user_id or "").strip()
    trace_svc = EvalTraceService(db)

    def _library_evidence_notes(
        lineage_tier: str,
        evidence_summary: str,
        truncated: bool,
        dropped: list[str],
    ) -> str:
        drops = ",".join(dropped[:12]) if dropped else ""
        return (
            "library artifact 评审：以证据闭包为准；若 lineage 为 fallback 或不足须保守打分。"
            f" lineage_tier={lineage_tier}；summary={evidence_summary}；"
            f" prompt_truncated={truncated}；dropped_sections={drops or 'none'}。"
            " 单章节/单故事节选；跨篇上下文不足写入 insufficient_evidence。"
        )

    if uid:
        from app.features.memoir.repo import get_chapters_for_memoir_list
        from app.features.story.repo import get_stories_for_user

        try:
            chapters = await get_chapters_for_memoir_list(
                uid, db, active_only=True, is_new_only=None
            )
            for ch in chapters[:_MAX_EVAL_CHAPTERS]:
                body = (ch.canonical_markdown or "").strip()
                if not body:
                    continue
                md = f"# 章节：{ch.title}\n\n{_clip_md_for_judge(body)}"
                cb = await trace_svc.build_chapter_bundle(uid, ch)
                formatted, cb2 = await trace_svc.format_chapter_bundle(cb)
                fm = formatted.format_meta
                cj = await judge.judge_memoir(
                    memoir_markdown=md,
                    source_transcript=formatted.source_transcript,
                    structured_evidence=formatted.structured_evidence,
                    reference_memoir_markdown=reference_memoir,
                    evidence_notes=_library_evidence_notes(
                        cb2.lineage_tier,
                        formatted.evidence_summary,
                        fm.truncated,
                        fm.dropped_sections,
                    ),
                )
                chapter_entries.append(
                    {
                        "id": ch.id,
                        "title": ch.title,
                        "order_index": ch.order_index,
                        "lineage_tier": cb2.lineage_tier,
                        "evidence_summary": formatted.evidence_summary,
                        "evidence_trace": cb2.model_dump(),
                        "format_meta": fm.model_dump(),
                        "judge": cj.model_dump() if cj else None,
                    }
                )
        except Exception as e:
            logger.warning("eval chapter judges skipped: {}", e)

        try:
            stories = await get_stories_for_user(db, uid, status="active")
            for st in stories[:_MAX_EVAL_STORIES]:
                body = (st.canonical_markdown or "").strip()
                if not body:
                    continue
                md = f"# 故事：{st.title}\n\n{_clip_md_for_judge(body)}"
                sb = await trace_svc.build_story_bundle(uid, str(st.id))
                formatted, sb2 = await trace_svc.format_story_bundle(sb)
                fm = formatted.format_meta
                sj = await judge.judge_memoir(
                    memoir_markdown=md,
                    source_transcript=formatted.source_transcript,
                    structured_evidence=formatted.structured_evidence,
                    reference_memoir_markdown=reference_memoir,
                    evidence_notes=_library_evidence_notes(
                        sb2.lineage_tier,
                        formatted.evidence_summary,
                        fm.truncated,
                        fm.dropped_sections,
                    ),
                )
                story_entries.append(
                    {
                        "id": st.id,
                        "title": st.title,
                        "stage": st.stage,
                        "lineage_tier": sb2.lineage_tier,
                        "evidence_summary": formatted.evidence_summary,
                        "evidence_trace": sb2.model_dump(),
                        "format_meta": fm.model_dump(),
                        "judge": sj.model_dump() if sj else None,
                    }
                )
        except Exception as e:
            logger.warning("eval story judges skipped: {}", e)

    synth_scores: list[float] = []
    if mem_out is not None:
        synth_scores.append(float(mem_out.total_score))

    library_scores: list[float] = []
    for row in chapter_entries:
        j = row.get("judge")
        if isinstance(j, dict) and j.get("total_score") is not None:
            library_scores.append(float(j["total_score"]))
    for row in story_entries:
        j = row.get("judge")
        if isinstance(j, dict) and j.get("total_score") is not None:
            library_scores.append(float(j["total_score"]))

    def _mean(xs: list[float]) -> float:
        return sum(xs) / len(xs) if xs else 0.0

    if synth_scores and library_scores:
        mem_total = 0.5 * _mean(synth_scores) + 0.5 * _mean(library_scores)
    elif synth_scores:
        mem_total = _mean(synth_scores)
    elif library_scores:
        mem_total = _mean(library_scores)
    else:
        mem_total = None

    exp = await eval_repo.get_experiment(db, str(run.experiment_id))
    weights = (
        exp.composite_weights_json
        if exp and isinstance(exp.composite_weights_json, dict)
        else None
    )
    comp = _composite(conv_total, mem_total, weights)

    bundle: dict[str, Any] = {
        "conversation_judge": conv_out.model_dump() if conv_out else None,
        "memoir_judge": mem_out.model_dump() if mem_out else None,
        "synthetic_memoir_judge": mem_out.model_dump() if mem_out else None,
        "chapters": chapter_entries,
        "stories": story_entries,
        "judge_meta": {
            "conversation_judge_ok": conv_out is not None,
            "memoir_synthetic_ok": mem_out is not None,
            "memoir_synth_scores_n": len(synth_scores),
            "memoir_library_scores_n": len(library_scores),
            "synthetic_memoir_lineage_tier": "replay_transcript_only",
            "synthetic_memoir_evidence_summary": (
                f"replay_turns={len(utterances)};structured_memory=unbound"
            ),
            "memoir_aggregate_rule": (
                "synthetic_memoir_judge_plus_library_memoir_judge_weighted_mean"
                if synth_scores and library_scores
                else ("synthetic_memoir_only" if synth_scores else "library_memoir_only")
            ),
        },
    }
    await eval_repo.update_run(
        db,
        run,
        status="completed",
        memoir_markdown=memoir_md,
        conversation_score_total=conv_total,
        memoir_score_total=mem_total,
        composite_score=comp,
        judge_bundle_json=bundle,
        completed_at=datetime.now(timezone.utc),
    )
    await db.commit()


async def _finalize_experiment_gate(db: AsyncSession, experiment_id: str) -> None:
    runs = await eval_repo.list_runs_for_experiment(db, experiment_id)
    exp = await eval_repo.get_experiment(db, experiment_id)
    if not exp:
        return
    cases = await eval_repo.list_cases(db, str(exp.regression_set_id))

    incomplete = [r for r in runs if str(r.status) not in ("completed", "failed")]
    if incomplete:
        return

    failed = [r for r in runs if str(r.status) == "failed"]
    if failed:
        await eval_repo.update_experiment(
            db,
            exp,
            status="failed",
            error_message="部分 run 失败",
            completed_at=datetime.now(timezone.utc),
        )
        await db.commit()
        return

    gr = compute_gate(cases=cases, runs=runs)
    await eval_repo.upsert_gate_verdict(
        db,
        experiment_id=experiment_id,
        passed=gr.passed,
        mean_composite_delta=gr.mean_delta,
        protected_regressions_json=gr.protected_regressions,
        details_json=gate_result_to_details(gr),
    )
    await eval_repo.update_experiment(
        db,
        exp,
        status="completed",
        completed_at=datetime.now(timezone.utc),
    )
    await db.commit()


async def execute_experiment_full(experiment_id: str) -> None:
    from app.core.redis_lock import acquire_redis_lock, release_redis_lock

    lock_key = f"lock:eval_experiment:{experiment_id}"
    lock_handle = acquire_redis_lock(lock_key, ttl_seconds=7200)
    if lock_handle is None:
        logger.warning(
            "eval experiment already running or lock busy experiment_id={}",
            experiment_id,
        )
        return

    try:
        async with AsyncSessionLocal() as db:
            exp = await eval_repo.get_experiment(db, experiment_id)
            if not exp:
                return
            await eval_repo.update_experiment(db, exp, status="running")
            await db.commit()

            cases = await eval_repo.list_cases(db, str(exp.regression_set_id))
            base_v = await eval_repo.get_version(db, str(exp.baseline_version_id))
            cand_v = await eval_repo.get_version(db, str(exp.candidate_version_id))
            if base_v is None or cand_v is None:
                await eval_repo.update_experiment(
                    db,
                    exp,
                    status="failed",
                    error_message="version 不存在",
                    completed_at=datetime.now(timezone.utc),
                )
                await db.commit()
                return

            for case in cases:
                for side, ver in ("baseline", base_v), ("candidate", cand_v):
                    run = await eval_repo.get_run(db, experiment_id, str(case.id), side)
                    if not run:
                        run = await eval_repo.create_run(
                            db,
                            experiment_id=experiment_id,
                            case_id=str(case.id),
                            side=side,
                        )
                    await db.commit()
                    await execute_eval_run(db, run=run, case=case, version=ver)

            await _finalize_experiment_gate(db, experiment_id)
    finally:
        release_redis_lock(lock_handle)