feat(api): 拆分章节物化与 Story 后处理,并加固 Redis 锁与腾讯 ASR
回忆录 Story 流水线(同步) - 同步路径仅写入 Story 与章节关联,改为 mark_chapter_dirty_sync,不再内联 compose - 物化由 Celery recompose_chapter 异步完成;compose 不变量与异常时保留 dirty 的语义在 repo 中补充说明 - Evidence:大批次时降低 top_k;路由候选 story 携带 char_count/version_count;append 超长/版本过多时强制新开 story - 叙事 prompt:relevant_chunks 去重,减少重复证据噪声 - 叙事回退与忠实度 gate:返回 fallback 类型并记录结构化日志(含耗时、JSON 有效性等) Post-commit 与任务编排 - 新增 post_commit.enqueue_story_post_commit_effects:统一派发 generate_story_image(Redis 去重)、延迟 recompose_chapter、可选 memory compaction - memoir_tasks / story_service / story_image_tasks 改为调用 post-commit 入口;主图回填后按关联章节重算并调度物化与 compacs(锁委托、Redis 单例、ASR to_thread) - 更新 test_narrative_pipeline 以适配 _apply_narrative_fallbacks 返回值
This commit is contained in:
@@ -673,18 +673,58 @@ def format_narrative_user_content(oral_text: str, evidence_text: str = "") -> st
|
||||
)
|
||||
|
||||
|
||||
def _normalize_evidence_line(s: str) -> str:
|
||||
return re.sub(r"\s+", " ", (s or "").strip().lower())
|
||||
|
||||
|
||||
def dedupe_evidence_chunk_rows(chunks: list) -> list:
|
||||
"""
|
||||
对 relevant_chunks 做稳定去重:按归一化后长度降序 + 原下标,单遍包含判定;
|
||||
复杂度 O(n log n);输出按原顺序中保留条目的相对顺序稳定。
|
||||
"""
|
||||
extracted: list[tuple[int, str, object]] = []
|
||||
for i, c in enumerate(chunks):
|
||||
content = (
|
||||
c.get("content", "") if isinstance(c, dict) else getattr(c, "content", "")
|
||||
)
|
||||
t = (content or "").strip()
|
||||
if not t:
|
||||
continue
|
||||
extracted.append((i, t, c))
|
||||
if len(extracted) <= 1:
|
||||
return [x[2] for x in extracted]
|
||||
extracted.sort(
|
||||
key=lambda x: (-len(_normalize_evidence_line(x[1])), x[0]),
|
||||
)
|
||||
kept_norms: list[str] = []
|
||||
kept: list[tuple[int, object]] = []
|
||||
for orig_idx, text, c in extracted:
|
||||
n = _normalize_evidence_line(text)
|
||||
dup = False
|
||||
for kn in kept_norms:
|
||||
if len(n) <= len(kn) and n in kn:
|
||||
dup = True
|
||||
break
|
||||
if not dup:
|
||||
kept_norms.append(n)
|
||||
kept.append((orig_idx, c))
|
||||
kept.sort(key=lambda x: x[0])
|
||||
return [x[1] for x in kept]
|
||||
|
||||
|
||||
def format_evidence_chunks_for_prompt(evidence: dict) -> str:
|
||||
"""将 retrieve_evidence / retrieve_evidence_sync 结果格式化为简短文本,供叙事 prompt 使用。
|
||||
|
||||
包含 chunks、摘要(若有)、confirmed facts、timeline、故事摘要(若有)。
|
||||
"""
|
||||
chunks = evidence.get("relevant_chunks") or []
|
||||
chunks = dedupe_evidence_chunk_rows(chunks[:10])
|
||||
summaries = evidence.get("relevant_summaries") or []
|
||||
facts = evidence.get("relevant_facts") or []
|
||||
timeline = evidence.get("timeline_hints") or []
|
||||
stories = evidence.get("relevant_stories") or []
|
||||
parts: list[str] = []
|
||||
for c in chunks[:10]:
|
||||
for c in chunks:
|
||||
content = (
|
||||
c.get("content", "") if isinstance(c, dict) else getattr(c, "content", "")
|
||||
)
|
||||
|
||||
@@ -63,8 +63,15 @@ class StoryRouteDecision(BaseModel):
|
||||
return str(v)
|
||||
|
||||
|
||||
def _build_candidate_json(stories: list[Story], *, preview_chars: int = 220) -> str:
|
||||
def _build_candidate_json(
|
||||
stories: list[Story],
|
||||
*,
|
||||
preview_chars: int = 220,
|
||||
story_meta: dict[str, dict[str, int]] | None = None,
|
||||
) -> str:
|
||||
"""story_meta: story_id -> { char_count, version_count },供路由感知篇幅与版本数。"""
|
||||
rows: list[dict[str, Any]] = []
|
||||
meta = story_meta or {}
|
||||
for s in stories:
|
||||
md = (s.canonical_markdown or "").strip().replace("\n", " ")
|
||||
preview = md[:preview_chars] + ("…" if len(md) > preview_chars else "")
|
||||
@@ -76,14 +83,17 @@ def _build_candidate_json(stories: list[Story], *, preview_chars: int = 220) ->
|
||||
cat = getattr(ch, "category", None) or ""
|
||||
tit = getattr(ch, "title", None) or ""
|
||||
links.append(f"{tit}({cat})")
|
||||
rows.append(
|
||||
{
|
||||
"id": s.id,
|
||||
"title": s.title,
|
||||
"preview": preview,
|
||||
"linked_chapters": links,
|
||||
}
|
||||
)
|
||||
row: dict[str, Any] = {
|
||||
"id": s.id,
|
||||
"title": s.title,
|
||||
"preview": preview,
|
||||
"linked_chapters": links,
|
||||
}
|
||||
m = meta.get(str(s.id))
|
||||
if m:
|
||||
row["char_count"] = int(m.get("char_count", 0))
|
||||
row["version_count"] = int(m.get("version_count", 0))
|
||||
rows.append(row)
|
||||
return json.dumps(rows, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
@@ -142,6 +152,7 @@ class StoryRouteAgent:
|
||||
candidate_stories: list[Story],
|
||||
llm: Any,
|
||||
valid_story_ids: set[str],
|
||||
story_meta: dict[str, dict[str, int]] | None = None,
|
||||
) -> StoryRouteDecision:
|
||||
if not llm:
|
||||
return StoryRouteDecision(
|
||||
@@ -149,7 +160,7 @@ class StoryRouteAgent:
|
||||
new_story_title=None,
|
||||
reason="no_llm",
|
||||
)
|
||||
payload = _build_candidate_json(candidate_stories)
|
||||
payload = _build_candidate_json(candidate_stories, story_meta=story_meta)
|
||||
prompt = get_story_route_prompt(
|
||||
chapter_category=chapter_category,
|
||||
chapter_title=chapter_title,
|
||||
@@ -200,13 +211,14 @@ class StoryRouteAgent:
|
||||
candidate_stories: list[Story],
|
||||
llm: Any,
|
||||
valid_story_ids: set[str],
|
||||
story_meta: dict[str, dict[str, int]] | None = None,
|
||||
) -> StoryBatchPlan | None:
|
||||
"""
|
||||
将本批 segment 划分为多个写入单元。解析失败返回 None,由调用方回退 decide()。
|
||||
"""
|
||||
if not llm or len(segments) < 2:
|
||||
return None
|
||||
payload = _build_candidate_json(candidate_stories)
|
||||
payload = _build_candidate_json(candidate_stories, story_meta=story_meta)
|
||||
segments_json = _build_segments_json_for_plan(segments)
|
||||
prompt = get_story_batch_plan_prompt(
|
||||
chapter_category=chapter_category,
|
||||
|
||||
Reference in New Issue
Block a user