refactor(api,expo): 多智能体与会话收敛、回忆录兼容层移除、后端测试集大幅删减
- 对齐「多智能体收敛」与「回忆录 stories-first / markdown-first」方向：收紧运行时契约、 删除过渡兼容路径与双轨逻辑，并同步更新客户端与文档。 - Chat：以 ChatOrchestrator 为实时编排入口；删除独立 conversation_agent，精简 prompts。 - Memoir：删除 memory_agent；MemoirOrchestrator、classification / story_route 与 prompts 收敛到 prepare_batches + run_story_pipeline_for_category_batch 主链路。 - 将 agents 侧 processor 迁入 feature 层为 background_runner，并移除 features 下重复/过时 processor 封装。 - 新增 history_store，强化「conversation_messages 为 DB 真源、Redis 为缓存」模型。 - 调整 models、repo、service、session_history；精简 WS message_types，重构 pipeline 与 router。 - 移除章节占位、整章再生等旧路径；章节列表与封面逻辑要求 story 关联；收紧 cover 资格与 enqueue。 - helpers、repo、service、router、reading_segment_materialize、story_pipeline_sync、pdf_service 等按 canonical markdown / cover_asset_id 收缩；删除 memoir_images/provider 等冗余。 - tasks：memoir_tasks、chapter_cover_tasks 等大幅瘦身；story_image_tasks 等与当前图片任务对齐。 - core：config、logging、redis、task_tracker 小幅调整。 - auth / user / payment / quota：路由或服务侧删减过时接口或逻辑（如 payment router 行数减少）。 - pyproject.toml、development.sh、.env.example / .env.production、README 等同步说明或变量。 - Alembic 0001_initial_schema 微调（与当前 schema 叙事一致的小改动）。 - 回忆录：types / mappers / api、章节页与 memoir 页与后端契约对齐；markdown-renderer 调整。 - 语音：删除 voice/player，voice-segment-store 相应精简。 - api/tests：删除 conftest 及绝大部分既有测试文件（websocket_baseline、conversation、memoir 图片、PDF、SMS 等），属有意收缩/待按 backend-test-system 重建的信号。 - docs：新增多智能体收敛与移除兼容层计划摘要；更新 story-first 设计、backend-test-system、 multi-agent-refactor-plan、实施总结等。 BREAKING CHANGE: 后端对外契约、回忆录章节字段与若干路由/任务行为已变更；大量 API 测试被移除， CI 若依赖这些用例需按新策略补测或调整流水线。
This commit is contained in:
Kevin
@@ -12,7 +12,11 @@ from sqlalchemy.orm import Session, joinedload
|
||||
|
||||
from app.agents.memoir.narrative_agent import NarrativeAgent
|
||||
from app.agents.memoir.prompts import STAGE_TO_ORDER, format_evidence_chunks_for_prompt
|
||||
from app.agents.memoir.story_route_agent import StoryRouteAgent
|
||||
from app.agents.memoir.story_route_agent import (
|
||||
PLAN_BATCH_MAX_SEGMENTS,
|
||||
StoryBatchPlan,
|
||||
StoryRouteAgent,
|
||||
)
|
||||
from app.agents.state_schema import MemoirStateSchema
|
||||
from app.core.logging import get_logger
|
||||
from app.features.memoir.cover_eligibility import chapter_needs_cover_enqueue
|
||||
@@ -21,6 +25,7 @@ from app.features.memoir.memoir_images.settings import MemoirImageSettings
|
||||
from app.features.memoir.models import Chapter
|
||||
from app.features.memoir.narrative_to_markdown import narrative_to_markdown
|
||||
from app.features.memoir.repo import compose_chapter_from_story_links_sync
|
||||
from app.features.memory.repo import retrieve_evidence_sync
|
||||
from app.features.story.models import Story
|
||||
from app.features.story.sync_write import (
|
||||
append_story_version_sync,
|
||||
@@ -28,7 +33,6 @@ from app.features.story.sync_write import (
|
||||
ensure_chapter_story_link_sync,
|
||||
list_active_stories_for_user_sync,
|
||||
)
|
||||
from app.features.memory.repo import retrieve_evidence_sync
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
@@ -40,6 +44,172 @@ def _is_json_narrative(text: str) -> bool:
|
||||
return s.startswith("{") and "paragraphs" in s
|
||||
|
||||
|
||||
def _ordered_text_for_segment_ids(
|
||||
category_segments: list, segment_ids: list[str]
|
||||
) -> str:
|
||||
id_to_text = {seg.id: (seg.transcript_text or "") for seg in category_segments}
|
||||
return "\n\n".join(id_to_text.get(sid, "") for sid in segment_ids)
|
||||
|
||||
|
||||
def _apply_narrative_fallbacks(
|
||||
narrative_raw: str,
|
||||
combined_unit_text: str,
|
||||
existing_for_narrative: str,
|
||||
existing_chapter_md: str,
|
||||
*,
|
||||
chapter_category: str,
|
||||
) -> str:
|
||||
if (
|
||||
existing_for_narrative
|
||||
and not _is_json_narrative(narrative_raw)
|
||||
and len(narrative_raw) < len(existing_for_narrative) * 0.8
|
||||
):
|
||||
logger.warning("叙事长度异常: 回退为原文追加")
|
||||
return f"{existing_for_narrative}\n\n{combined_unit_text}"
|
||||
|
||||
if (
|
||||
not existing_for_narrative
|
||||
and existing_chapter_md
|
||||
and not _is_json_narrative(narrative_raw)
|
||||
and len(narrative_raw) < len(existing_chapter_md) * 0.8
|
||||
):
|
||||
logger.warning(
|
||||
"章节级长度异常: 回退为 transcript 追加, category=%s",
|
||||
chapter_category,
|
||||
)
|
||||
return f"{existing_chapter_md}\n\n{combined_unit_text}"
|
||||
return narrative_raw
|
||||
|
||||
|
||||
def _ensure_chapter_record(
|
||||
session: Session,
|
||||
*,
|
||||
user_id: str,
|
||||
chapter_category: str,
|
||||
title: str,
|
||||
source_ids: list[str],
|
||||
calculated_order_index: int,
|
||||
) -> Chapter:
|
||||
stmt_chapter = (
|
||||
select(Chapter)
|
||||
.where(
|
||||
Chapter.user_id == user_id,
|
||||
Chapter.category == chapter_category,
|
||||
Chapter.is_active == True, # noqa: E712
|
||||
)
|
||||
.options(
|
||||
joinedload(Chapter.images),
|
||||
joinedload(Chapter.story_links),
|
||||
)
|
||||
)
|
||||
chapter = session.execute(stmt_chapter).unique().scalar_one_or_none()
|
||||
if not chapter:
|
||||
chapter = Chapter(
|
||||
id=str(uuid.uuid4()),
|
||||
user_id=user_id,
|
||||
title=title,
|
||||
order_index=calculated_order_index,
|
||||
status="completed",
|
||||
category=chapter_category,
|
||||
is_new=True,
|
||||
source_segments=source_ids,
|
||||
)
|
||||
session.add(chapter)
|
||||
session.flush()
|
||||
else:
|
||||
chapter.source_segments = list(
|
||||
set((chapter.source_segments or []) + source_ids)
|
||||
)
|
||||
chapter.is_new = True
|
||||
session.flush()
|
||||
return chapter
|
||||
|
||||
|
||||
def _run_batch_plan_writes(
|
||||
session: Session,
|
||||
*,
|
||||
plan: StoryBatchPlan,
|
||||
category_segments: list,
|
||||
chapter: Chapter,
|
||||
chapter_category: str,
|
||||
evidence_text: str,
|
||||
existing_chapter_md: str,
|
||||
slot_snippets: dict[str, str],
|
||||
user_id: str,
|
||||
user_profile: str,
|
||||
user_birth_year: int | None,
|
||||
llm: Any,
|
||||
narrative_agent: NarrativeAgent,
|
||||
) -> set[str]:
|
||||
dispatch_ids: set[str] = set()
|
||||
for unit in plan.units:
|
||||
unit_text = _ordered_text_for_segment_ids(category_segments, unit.segment_ids)
|
||||
new_content_input = (
|
||||
f"{unit_text}\n\n【相关记忆摘录】\n{evidence_text}"
|
||||
if evidence_text.strip()
|
||||
else unit_text
|
||||
)
|
||||
|
||||
target_story_id: str | None = None
|
||||
existing_for_narrative = ""
|
||||
if unit.decision == "append_story" and unit.target_story_id:
|
||||
st = session.get(Story, unit.target_story_id)
|
||||
if st and st.user_id == user_id:
|
||||
target_story_id = st.id
|
||||
existing_for_narrative = (st.canonical_markdown or "").strip()
|
||||
|
||||
narrative_raw = narrative_agent.generate_narrative(
|
||||
stage=chapter_category,
|
||||
slots=slot_snippets,
|
||||
new_content=new_content_input,
|
||||
existing_content=existing_for_narrative,
|
||||
user_profile=user_profile,
|
||||
birth_year=user_birth_year,
|
||||
llm=llm,
|
||||
)
|
||||
narrative_raw = _apply_narrative_fallbacks(
|
||||
narrative_raw,
|
||||
unit_text,
|
||||
existing_for_narrative,
|
||||
existing_chapter_md,
|
||||
chapter_category=chapter_category,
|
||||
)
|
||||
|
||||
md = narrative_to_markdown(narrative_raw)
|
||||
if not md.strip():
|
||||
md = unit_text.strip()
|
||||
|
||||
if target_story_id:
|
||||
append_story_version_sync(session, target_story_id, md)
|
||||
dispatch_ids.add(target_story_id)
|
||||
ensure_chapter_story_link_sync(
|
||||
session, chapter_id=chapter.id, story_id=target_story_id
|
||||
)
|
||||
else:
|
||||
story_title = (unit.new_story_title or "").strip()
|
||||
if not story_title:
|
||||
story_title = narrative_agent.generate_title(
|
||||
stage=chapter_category,
|
||||
emotion="neutral",
|
||||
slots=slot_snippets,
|
||||
user_profile=user_profile,
|
||||
birth_year=user_birth_year,
|
||||
llm=llm,
|
||||
)
|
||||
st = create_story_with_version_sync(
|
||||
session,
|
||||
user_id=user_id,
|
||||
title=story_title,
|
||||
canonical_markdown=md,
|
||||
stage=chapter_category,
|
||||
)
|
||||
dispatch_ids.add(st.id)
|
||||
ensure_chapter_story_link_sync(
|
||||
session, chapter_id=chapter.id, story_id=st.id
|
||||
)
|
||||
return dispatch_ids
|
||||
|
||||
|
||||
def run_story_pipeline_for_category_batch(
|
||||
session: Session,
|
||||
*,
|
||||
@@ -125,107 +295,121 @@ def run_story_pipeline_for_category_batch(
|
||||
if evidence_text.strip()
|
||||
else combined_text
|
||||
)
|
||||
route = route_agent.decide(
|
||||
chapter_category=chapter_category,
|
||||
chapter_title=title,
|
||||
batch_transcript=batch_for_route,
|
||||
candidate_stories=candidates,
|
||||
llm=llm,
|
||||
valid_story_ids=valid_ids,
|
||||
)
|
||||
|
||||
target_story_id: str | None = None
|
||||
existing_for_narrative = ""
|
||||
if route.decision == "append_story" and route.target_story_id:
|
||||
st = session.get(Story, route.target_story_id)
|
||||
if st and st.user_id == user_id:
|
||||
target_story_id = st.id
|
||||
existing_for_narrative = (st.canonical_markdown or "").strip()
|
||||
|
||||
narrative_raw = narrative_agent.generate_narrative(
|
||||
stage=chapter_category,
|
||||
slots=slot_snippets,
|
||||
new_content=new_content_input,
|
||||
existing_content=existing_for_narrative,
|
||||
user_profile=user_profile,
|
||||
birth_year=user_birth_year,
|
||||
llm=llm,
|
||||
)
|
||||
|
||||
if (
|
||||
existing_for_narrative
|
||||
and not _is_json_narrative(narrative_raw)
|
||||
and len(narrative_raw) < len(existing_for_narrative) * 0.8
|
||||
):
|
||||
logger.warning("叙事长度异常: 回退为原文追加")
|
||||
narrative_raw = f"{existing_for_narrative}\n\n{combined_text}"
|
||||
|
||||
if (
|
||||
not existing_for_narrative
|
||||
and existing_chapter_md
|
||||
and not _is_json_narrative(narrative_raw)
|
||||
and len(narrative_raw) < len(existing_chapter_md) * 0.8
|
||||
):
|
||||
logger.warning(
|
||||
"章节级长度异常: 回退为 transcript 追加, category=%s",
|
||||
chapter_category,
|
||||
)
|
||||
narrative_raw = f"{existing_chapter_md}\n\n{combined_text}"
|
||||
|
||||
md = narrative_to_markdown(narrative_raw)
|
||||
if not md.strip():
|
||||
md = combined_text.strip()
|
||||
|
||||
calculated_order_index = STAGE_TO_ORDER.get(chapter_category, 999)
|
||||
|
||||
if not chapter:
|
||||
chapter = Chapter(
|
||||
id=str(uuid.uuid4()),
|
||||
user_id=user_id,
|
||||
title=title,
|
||||
order_index=calculated_order_index,
|
||||
status="completed",
|
||||
category=chapter_category,
|
||||
cover_image=None,
|
||||
is_new=True,
|
||||
source_segments=source_ids,
|
||||
use_batch_plan = (
|
||||
llm
|
||||
and len(category_segments) >= 2
|
||||
and len(category_segments) <= PLAN_BATCH_MAX_SEGMENTS
|
||||
)
|
||||
plan: StoryBatchPlan | None = None
|
||||
if use_batch_plan:
|
||||
segs = [(seg.id, seg.transcript_text or "") for seg in category_segments]
|
||||
plan = route_agent.plan_batch(
|
||||
chapter_category=chapter_category,
|
||||
chapter_title=title,
|
||||
segments=segs,
|
||||
candidate_stories=candidates,
|
||||
llm=llm,
|
||||
valid_story_ids=valid_ids,
|
||||
)
|
||||
session.add(chapter)
|
||||
session.flush()
|
||||
else:
|
||||
chapter.source_segments = list(
|
||||
set((chapter.source_segments or []) + source_ids)
|
||||
)
|
||||
chapter.is_new = True
|
||||
|
||||
do_append = target_story_id is not None
|
||||
chapter = _ensure_chapter_record(
|
||||
session,
|
||||
user_id=user_id,
|
||||
chapter_category=chapter_category,
|
||||
title=title,
|
||||
source_ids=source_ids,
|
||||
calculated_order_index=calculated_order_index,
|
||||
)
|
||||
|
||||
if do_append:
|
||||
append_story_version_sync(session, target_story_id, md)
|
||||
dispatch_ids.add(target_story_id)
|
||||
ensure_chapter_story_link_sync(
|
||||
session, chapter_id=chapter.id, story_id=target_story_id
|
||||
)
|
||||
else:
|
||||
story_title = (route.new_story_title or "").strip()
|
||||
if not story_title:
|
||||
story_title = narrative_agent.generate_title(
|
||||
stage=chapter_category,
|
||||
emotion="neutral",
|
||||
slots=slot_snippets,
|
||||
user_profile=user_profile,
|
||||
birth_year=user_birth_year,
|
||||
llm=llm,
|
||||
)
|
||||
st = create_story_with_version_sync(
|
||||
if plan is not None:
|
||||
dispatch_ids = _run_batch_plan_writes(
|
||||
session,
|
||||
plan=plan,
|
||||
category_segments=category_segments,
|
||||
chapter=chapter,
|
||||
chapter_category=chapter_category,
|
||||
evidence_text=evidence_text,
|
||||
existing_chapter_md=existing_chapter_md,
|
||||
slot_snippets=slot_snippets,
|
||||
user_id=user_id,
|
||||
title=story_title,
|
||||
canonical_markdown=md,
|
||||
stage=chapter_category,
|
||||
user_profile=user_profile,
|
||||
user_birth_year=user_birth_year,
|
||||
llm=llm,
|
||||
narrative_agent=narrative_agent,
|
||||
)
|
||||
dispatch_ids.add(st.id)
|
||||
ensure_chapter_story_link_sync(session, chapter_id=chapter.id, story_id=st.id)
|
||||
else:
|
||||
route = route_agent.decide(
|
||||
chapter_category=chapter_category,
|
||||
chapter_title=title,
|
||||
batch_transcript=batch_for_route,
|
||||
candidate_stories=candidates,
|
||||
llm=llm,
|
||||
valid_story_ids=valid_ids,
|
||||
)
|
||||
|
||||
target_story_id: str | None = None
|
||||
existing_for_narrative = ""
|
||||
if route.decision == "append_story" and route.target_story_id:
|
||||
st = session.get(Story, route.target_story_id)
|
||||
if st and st.user_id == user_id:
|
||||
target_story_id = st.id
|
||||
existing_for_narrative = (st.canonical_markdown or "").strip()
|
||||
|
||||
narrative_raw = narrative_agent.generate_narrative(
|
||||
stage=chapter_category,
|
||||
slots=slot_snippets,
|
||||
new_content=new_content_input,
|
||||
existing_content=existing_for_narrative,
|
||||
user_profile=user_profile,
|
||||
birth_year=user_birth_year,
|
||||
llm=llm,
|
||||
)
|
||||
|
||||
narrative_raw = _apply_narrative_fallbacks(
|
||||
narrative_raw,
|
||||
combined_text,
|
||||
existing_for_narrative,
|
||||
existing_chapter_md,
|
||||
chapter_category=chapter_category,
|
||||
)
|
||||
|
||||
md = narrative_to_markdown(narrative_raw)
|
||||
if not md.strip():
|
||||
md = combined_text.strip()
|
||||
|
||||
do_append = target_story_id is not None
|
||||
|
||||
if do_append:
|
||||
append_story_version_sync(session, target_story_id, md)
|
||||
dispatch_ids.add(target_story_id)
|
||||
ensure_chapter_story_link_sync(
|
||||
session, chapter_id=chapter.id, story_id=target_story_id
|
||||
)
|
||||
else:
|
||||
story_title = (route.new_story_title or "").strip()
|
||||
if not story_title:
|
||||
story_title = narrative_agent.generate_title(
|
||||
stage=chapter_category,
|
||||
emotion="neutral",
|
||||
slots=slot_snippets,
|
||||
user_profile=user_profile,
|
||||
birth_year=user_birth_year,
|
||||
llm=llm,
|
||||
)
|
||||
st = create_story_with_version_sync(
|
||||
session,
|
||||
user_id=user_id,
|
||||
title=story_title,
|
||||
canonical_markdown=md,
|
||||
stage=chapter_category,
|
||||
)
|
||||
dispatch_ids.add(st.id)
|
||||
ensure_chapter_story_link_sync(
|
||||
session, chapter_id=chapter.id, story_id=st.id
|
||||
)
|
||||
|
||||
compose_chapter_from_story_links_sync(session, chapter.id)
|
||||
session.flush()
|
||||
|
||||
Reference in New Issue
Block a user