e4bf0710c7
数据库 - 新增迁移 0003：timeline_events.memory_source_id 外键 → memory_sources，便于按 ingest 源做时间线幂等 后端 - 记忆 - 新增 ingest 后 LLM 富化（摘要/事实/时间线），可配置开关与最大字符数 - 新增证据包组装：合并 chunk、摘要、事实、时间线、故事等检索结果；支持空 query 时是否仍带 rolling 等开关 - repo/retriever/service/router/schemas/summarizer/timeline/extractor 等扩展；文档 memory-retrieval.md 更新 后端 - 对话 WS - 增加 PING/PONG；分段 ASR 日志与空音频处理；转写失败与「无助手回复」错误提示更明确 - 助手多段回复持久化使用统一分隔符，与分段逻辑一致 后端 - Agent - reply_limits：按 [SPLIT] 与段落拆段，并保证非空 fallback，供 WS 与 TTS 多段下发 后端 - 回忆录任务 - transcript ingest 记录 source_id；任务成功结?
83 lines
2.7 KiB
Python
83 lines
2.7 KiB
Python
"""Hybrid retriever — metadata filter + FTS + vector retrieval + score fusion."""
|
|
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.features.memory.evidence import retrieve_evidence_bundle_async
|
|
from app.features.memory.repo import search_chunks_fts, search_chunks_vector
|
|
from app.ports.embedding import EmbeddingProvider
|
|
|
|
|
|
def _rrf_merge(
|
|
fts_items: list[dict], vector_items: list[dict], k: int = 60
|
|
) -> list[dict]:
|
|
"""Reciprocal Rank Fusion. Merge FTS and vector results by id."""
|
|
scores: dict[str, float] = {}
|
|
for rank, item in enumerate(fts_items):
|
|
cid = item["id"]
|
|
scores[cid] = scores.get(cid, 0) + 1 / (k + rank + 1)
|
|
for rank, item in enumerate(vector_items):
|
|
cid = item["id"]
|
|
scores[cid] = scores.get(cid, 0) + 1 / (k + rank + 1)
|
|
|
|
all_items = {x["id"]: x for x in fts_items + vector_items}
|
|
sorted_ids = sorted(scores.keys(), key=lambda i: scores[i], reverse=True)
|
|
return [all_items[i] for i in sorted_ids]
|
|
|
|
|
|
class HybridRetriever:
|
|
"""Combine FTS, vector, and metadata filter into evidence bundle."""
|
|
|
|
def __init__(
|
|
self,
|
|
db: AsyncSession,
|
|
*,
|
|
embedding_provider: EmbeddingProvider | None = None,
|
|
):
|
|
self._db = db
|
|
self._embedding = embedding_provider
|
|
|
|
async def retrieve(self, user_id: str, query: str, *, top_k: int = 10) -> dict:
|
|
"""
|
|
Return evidence bundle:
|
|
{relevant_chunks, relevant_summaries, relevant_facts, timeline_hints, relevant_stories}
|
|
"""
|
|
if not query.strip():
|
|
return await retrieve_evidence_bundle_async(
|
|
self._db,
|
|
user_id,
|
|
query,
|
|
top_k=top_k,
|
|
merged_chunk_dicts=[],
|
|
)
|
|
|
|
q = query.strip()
|
|
fts_chunks = await search_chunks_fts(
|
|
self._db, user_id=user_id, query=query, limit=top_k * 2
|
|
)
|
|
|
|
vector_chunks: list[dict] = []
|
|
if self._embedding and q:
|
|
q_emb = await self._embedding.embed_text(q)
|
|
if q_emb:
|
|
vector_chunks = await search_chunks_vector(
|
|
self._db, user_id=user_id, query_embedding=q_emb, limit=top_k * 2
|
|
)
|
|
|
|
merged = _rrf_merge(fts_chunks, vector_chunks)[:top_k]
|
|
merged_chunk_dicts = [
|
|
{
|
|
"id": c["id"],
|
|
"content": c["content"],
|
|
"chunk_index": c.get("chunk_index", 0),
|
|
}
|
|
for c in merged
|
|
]
|
|
|
|
return await retrieve_evidence_bundle_async(
|
|
self._db,
|
|
user_id,
|
|
query,
|
|
top_k=top_k,
|
|
merged_chunk_dicts=merged_chunk_dicts,
|
|
)
|