7f57f96c25
本次 squash merge 将 codex-story-first-image-intent 的整体改动合入 development，核心内容包括： 1. 后端数据与迁移：新增 stories、story_versions、story_image_intents、chapter_cover_intents、assets 等模型与 Alembic 迁移，建立 story-first、markdown-first、asset-first 的主数据链路。 2. 生成与任务链：引入 StoryBuilderOrchestrator、ChapterComposerOrchestrator、story_image_tasks、chapter_cover_tasks，图片生成从正文占位符改为结构化 intent -> asset -> markdown 回填。 3. 并发与一致性：为 story/chapter intent 增加 claim_token、claimed_at、attempt_count，采用数据库原子 claim 为主、Redis 锁为辅，避免重复生成、锁误删和 processing 卡死。 4. Memoir 读写路径：章节 canonical_markdown 成为正文真源，列表/详情接口补齐 markdown、cover_asset、word_count 等字段，PDF 与 asset 解析链路同步升级。 5. Memory / Retrieval：扩展 transcript ingest、chunking、evidence 检索与 story 聚合基础设施，为后续 story-first RAG 与多 agent 编排提供底座。 6. App 端体验：章节页继续走 MarkdownRenderer 阅读链，同时吸收 fix3-19 的跨平台 UI glitch 修复；更新对话页、首页、文案资源与章节列表映射逻辑。 7. 测试与文档：补充 asset resolver、story image task、章节封面派发、markdown 映射等回归测试，并加入图片占位符退役设计文档。
103 lines
3.2 KiB
Python
103 lines
3.2 KiB
Python
"""Hybrid retriever — metadata filter + FTS + vector retrieval + score fusion."""
|
|
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.features.memory.repo import (
|
|
get_facts_for_user,
|
|
get_timeline_events_for_user,
|
|
search_chunks_fts,
|
|
search_chunks_vector,
|
|
)
|
|
from app.ports.embedding import EmbeddingProvider
|
|
|
|
|
|
def _rrf_merge(
|
|
fts_items: list[dict], vector_items: list[dict], k: int = 60
|
|
) -> list[dict]:
|
|
"""Reciprocal Rank Fusion. Merge FTS and vector results by id."""
|
|
scores: dict[str, float] = {}
|
|
for rank, item in enumerate(fts_items):
|
|
cid = item["id"]
|
|
scores[cid] = scores.get(cid, 0) + 1 / (k + rank + 1)
|
|
for rank, item in enumerate(vector_items):
|
|
cid = item["id"]
|
|
scores[cid] = scores.get(cid, 0) + 1 / (k + rank + 1)
|
|
|
|
all_items = {x["id"]: x for x in fts_items + vector_items}
|
|
sorted_ids = sorted(scores.keys(), key=lambda i: scores[i], reverse=True)
|
|
return [all_items[i] for i in sorted_ids]
|
|
|
|
|
|
class HybridRetriever:
|
|
"""Combine FTS, vector, and metadata filter into evidence bundle."""
|
|
|
|
def __init__(
|
|
self,
|
|
db: AsyncSession,
|
|
*,
|
|
embedding_provider: EmbeddingProvider | None = None,
|
|
):
|
|
self._db = db
|
|
self._embedding = embedding_provider
|
|
|
|
async def retrieve(self, user_id: str, query: str, *, top_k: int = 10) -> dict:
|
|
"""
|
|
Return evidence bundle:
|
|
{relevant_chunks, relevant_summaries, relevant_facts, timeline_hints, relevant_stories}
|
|
"""
|
|
fts_chunks = await search_chunks_fts(
|
|
self._db, user_id=user_id, query=query, limit=top_k * 2
|
|
)
|
|
|
|
vector_chunks: list[dict] = []
|
|
if self._embedding and query.strip():
|
|
q_emb = await self._embedding.embed_text(query.strip())
|
|
if q_emb:
|
|
vector_chunks = await search_chunks_vector(
|
|
self._db, user_id=user_id, query_embedding=q_emb, limit=top_k * 2
|
|
)
|
|
|
|
merged = _rrf_merge(fts_chunks, vector_chunks)[:top_k]
|
|
relevant_chunks = [
|
|
{
|
|
"id": c["id"],
|
|
"content": c["content"],
|
|
"chunk_index": c.get("chunk_index", 0),
|
|
}
|
|
for c in merged
|
|
]
|
|
|
|
facts = await get_facts_for_user(self._db, user_id=user_id, limit=top_k)
|
|
relevant_facts = [
|
|
{
|
|
"id": f.id,
|
|
"fact_type": f.fact_type,
|
|
"subject": f.subject,
|
|
"predicate": f.predicate,
|
|
"object_json": f.object_json,
|
|
}
|
|
for f in facts
|
|
]
|
|
|
|
events = await get_timeline_events_for_user(
|
|
self._db, user_id=user_id, limit=top_k
|
|
)
|
|
timeline_hints = [
|
|
{
|
|
"id": e.id,
|
|
"event_year": e.event_year,
|
|
"event_date": e.event_date,
|
|
"title": e.title,
|
|
"description": e.description,
|
|
}
|
|
for e in events
|
|
]
|
|
|
|
return {
|
|
"relevant_chunks": relevant_chunks,
|
|
"relevant_summaries": [],
|
|
"relevant_facts": relevant_facts,
|
|
"timeline_hints": timeline_hints,
|
|
"relevant_stories": [],
|
|
}
|