kevin/life-echo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(eval+memoir)：精简内部评测路由与服务，composite/对话摘要与 judge 能力补强

Browse Source 
- 访谈：新增 interview_state_hints，联动 orchestrator 与提示词
- 回忆录：story_pipeline_sync/state/memory/post_commit 与 Celery 任务调整
- 基建：开发用 celery broker、compose/development 脚本、依赖注入
- eval-web：移除数据集/实验/版本等页面与流式轮询，突出 Playground
- 文档与单测同步
This commit is contained in:
Kevin
2026-04-08 21:36:12 +08:00
parent 2a0c80987d
commit 064ad2161d
64 changed files with 3412 additions and 3068 deletions
Download Patch File Download Diff File Expand all files Collapse all files

152
api/app/tasks/memoir_tasks.py
View File

@@ -3,6 +3,7 @@
"""
import json
import time
import uuid
from datetime import datetime, timezone
from typing import Dict, List, Set
@@ -67,6 +68,56 @@ logger = get_logger(__name__)
_REDIS_CLIENTS: dict[bool, redis.Redis] = {}
def _run_post_pipeline_commit(
*,
user_id: str,
story_dispatch_ids: set[str],
recompose_chapter_ids: set[str],
cover_chapter_ids: set[str],
trigger_source: str,
need_compaction: bool,
need_quality_pass: bool = False,
memoir_correlation_id: str | None = None,
compaction_extra: dict | None = None,
) -> None:
"""Shared post-commit dispatch: images, recompose, compaction, quality pass, covers."""
from app.features.story.post_commit import enqueue_story_post_commit_effects
pc = enqueue_story_post_commit_effects(
user_id=user_id,
story_ids=set(story_dispatch_ids),
chapter_ids=recompose_chapter_ids,
trigger_source=trigger_source,
need_compaction=need_compaction,
need_quality_pass=need_quality_pass,
memoir_correlation_id=memoir_correlation_id,
compaction_extra=compaction_extra,
)
logger.info(
"event=story_post_commit user_id={} trigger={} "
"enqueued_story_image_count={} enqueued_chapter_recompose_count={} "
"compaction_scheduled={} quality_pass_scheduled={} errors={}",
user_id,
trigger_source,
pc.enqueued_story_image_count,
pc.enqueued_chapter_recompose_count,
pc.compaction_scheduled,
pc.quality_pass_scheduled,
pc.errors,
)
if cover_chapter_ids:
image_settings = MemoirImageSettings.from_env()
if image_settings.enabled:
from app.tasks.chapter_cover_enqueue import (
try_enqueue_generate_chapter_cover,
)
for ch_id in sorted(cover_chapter_ids):
if try_enqueue_generate_chapter_cover(ch_id, source=trigger_source):
logger.info("派发章节封面任务: chapter={}", ch_id)
def _get_llm():
"""Celery 任务内获取 LangChain LLM（通过 port）"""
try:
@@ -352,6 +403,7 @@ def process_memoir_phase2(
cid = effective_correlation_id(
explicit=memoir_correlation_id, celery_task_id=str(task_id)
)
phase2_t0 = time.perf_counter()
logger.info(
"event=memoir_phase2_start user_id={} task_id={} chapter_category={} "
"memoir_correlation_id={}",
@@ -408,9 +460,11 @@ def process_memoir_phase2(
chapters_to_enqueue: Set[str] = set()
affected_chapter_ids: Set[str] = set()
lock_t0 = time.perf_counter()
lock_handle = _acquire_chapter_lock(
user_id, chapter_category, ttl_seconds=_chapter_lock_ttl()
)
lock_elapsed = time.perf_counter() - lock_t0
if lock_handle is None:
logger.warning(
"event=memoir_phase2_lock_busy user_id={} chapter_category={}",
@@ -426,6 +480,7 @@ def process_memoir_phase2(
return {"status": "noop"}
state = get_or_create_state_sync(user_id, db)
pipeline_t0 = time.perf_counter()
chapter, needs_cover, disp = run_story_pipeline_for_category_batch(
db,
user_id=user_id,
@@ -439,6 +494,7 @@ def process_memoir_phase2(
occupation=user_occupation,
memoir_correlation_id=cid,
)
pipeline_elapsed = time.perf_counter() - pipeline_t0
story_dispatch_ids |= disp
db.flush()
if chapter is None:
@@ -489,16 +545,15 @@ def process_memoir_phase2(
db.commit()
from app.features.story.post_commit import (
enqueue_story_post_commit_effects,
)
pc = enqueue_story_post_commit_effects(
_run_post_pipeline_commit(
user_id=user_id,
story_ids=set(story_dispatch_ids),
chapter_ids=affected_chapter_ids,
story_dispatch_ids=story_dispatch_ids,
recompose_chapter_ids=affected_chapter_ids,
cover_chapter_ids=chapters_to_enqueue,
trigger_source="pipeline_phase2",
need_compaction=True,
need_quality_pass=True,
memoir_correlation_id=cid,
compaction_extra={
"pipeline_run_id": str(task_id),
"memoir_correlation_id": cid,
@@ -507,35 +562,21 @@ def process_memoir_phase2(
"chapter_category": chapter_category,
},
)
logger.info(
"event=story_post_commit user_id={} trigger=pipeline_phase2 "
"enqueued_story_image_count={} enqueued_chapter_recompose_count={} "
"compaction_scheduled={} errors={}",
user_id,
pc.enqueued_story_image_count,
pc.enqueued_chapter_recompose_count,
pc.compaction_scheduled,
pc.errors,
)
from app.tasks.chapter_cover_enqueue import (
try_enqueue_generate_chapter_cover,
)
for chapter_id in sorted(chapters_to_enqueue):
if try_enqueue_generate_chapter_cover(
chapter_id, source="pipeline_phase2"
):
logger.info(f"派发章节封面任务: chapter={chapter_id}")
phase2_elapsed = time.perf_counter() - phase2_t0
logger.info(
"event=memoir_phase2_done user_id={} task_id={} chapter_category={} "
"segment_count={} memoir_correlation_id={}",
"segment_count={} memoir_correlation_id={} "
"lock_seconds={:.3f} pipeline_seconds={:.3f} "
"phase2_total_seconds={:.3f}",
user_id,
task_id,
chapter_category,
len(category_segments),
cid,
lock_elapsed,
pipeline_elapsed,
phase2_elapsed,
)
return {
"status": "success",
@@ -574,6 +615,7 @@ def process_memoir_phase1(self, user_id: str, segment_ids: List[str]):
memoir_correlation_id,
)
_update_task_status_sync(user_id, task_id, "running")
phase1_t0 = time.perf_counter()
try:
with get_sync_db() as db:
@@ -595,6 +637,7 @@ def process_memoir_phase1(self, user_id: str, segment_ids: List[str]):
)
return {"status": "no_segments"}
ingest_t0 = time.perf_counter()
for seg in segments:
conv_id = getattr(seg, "conversation_id", None) or ""
text = (seg.user_input_text or "").strip()
@@ -629,15 +672,17 @@ def process_memoir_phase1(self, user_id: str, segment_ids: List[str]):
e,
type(e).__name__,
)
ingest_elapsed = time.perf_counter() - ingest_t0
llm = _get_llm()
llm_fast = _get_llm_fast()
llm_fast = _get_llm_fast() or llm
if (settings.llm_fast_model or "").strip():
logger.info(
"event=llm_fast_tier_used pipeline=memoir_prepare_batches model={}",
settings.llm_fast_model,
)
prep_t0 = time.perf_counter()
memoir_orchestrator = MemoirOrchestrator()
prepared = memoir_orchestrator.prepare_batches(
segments=list(segments),
@@ -654,6 +699,7 @@ def process_memoir_phase1(self, user_id: str, segment_ids: List[str]):
memoir_batch=True,
),
)
prep_elapsed = time.perf_counter() - prep_t0
skip_ids = prepared.segment_skip_story_ids
missing_cat = [
@@ -709,6 +755,7 @@ def process_memoir_phase1(self, user_id: str, segment_ids: List[str]):
_schedule_phase2_timeout(user_id, cc, memoir_correlation_id)
categories_processed = sorted(prepared.category_to_segments.keys())
phase1_elapsed = time.perf_counter() - phase1_t0
_update_task_status_sync(
user_id,
task_id,
@@ -721,12 +768,17 @@ def process_memoir_phase1(self, user_id: str, segment_ids: List[str]):
)
logger.info(
"event=memoir_phase1_done user_id={} task_id={} segment_count={} "
"categories={} memoir_correlation_id={}",
"categories={} memoir_correlation_id={} "
"memory_ingest_seconds={:.3f} prepare_batches_seconds={:.3f} "
"phase1_total_seconds={:.3f}",
user_id,
task_id,
len(segments),
categories_processed,
memoir_correlation_id,
ingest_elapsed,
prep_elapsed,
phase1_elapsed,
)
return {
"status": "success",
@@ -818,38 +870,22 @@ def generate_chapter_content(self, user_id: str, stage: str, new_content: str):
db.commit()
db.refresh(chapter)
from app.features.story.post_commit import enqueue_story_post_commit_effects
ch_ids: set[str] = {str(chapter.id)}
pc = enqueue_story_post_commit_effects(
cover_ids = (
ch_ids
if chapter_needs_cover_enqueue(chapter)
else set()
)
_run_post_pipeline_commit(
user_id=user_id,
story_ids=set(dispatch_ids),
chapter_ids=ch_ids,
trigger_source="pipeline",
story_dispatch_ids=set(dispatch_ids),
recompose_chapter_ids=ch_ids,
cover_chapter_ids=cover_ids,
trigger_source="pipeline_generate_chapter",
need_compaction=False,
need_quality_pass=True,
memoir_correlation_id=cid,
)
logger.info(
"event=story_post_commit user_id={} trigger=pipeline_generate_chapter "
"enqueued_story_image_count={} enqueued_chapter_recompose_count={} "
"compaction_scheduled={} errors={}",
user_id,
pc.enqueued_story_image_count,
pc.enqueued_chapter_recompose_count,
pc.compaction_scheduled,
pc.errors,
)
image_settings = MemoirImageSettings.from_env()
if (
image_settings.enabled
and chapter
and chapter_needs_cover_enqueue(chapter)
):
from app.tasks.chapter_cover_enqueue import (
try_enqueue_generate_chapter_cover,
)
try_enqueue_generate_chapter_cover(chapter.id, source="pipeline")
return {"status": "success"}
except Retry: