聊天和回忆录证据检索都走 pgvector,去掉 Postgres FTS/content_tsv,新迁移删掉 content_tsv 列(部署要先 alembic upgrade)。
Embedding 端口增加 is_available(),聊天和回忆录日志用统一方式表示向量是否真能调用。 记忆整理(compaction)支持 Beat 定期扫用户; 事实抽取提示与 subject 归一化,减少同一人多种称呼;
This commit is contained in:
@@ -1,31 +1,17 @@
|
||||
"""Hybrid retriever — metadata filter + FTS + vector retrieval + score fusion."""
|
||||
"""Hybrid retriever — 向量检索 + 元数据证据包。"""
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.features.memory.evidence import retrieve_evidence_bundle_async
|
||||
from app.features.memory.repo import search_chunks_fts, search_chunks_vector
|
||||
from app.features.memory.repo import search_chunks_vector
|
||||
from app.ports.embedding import EmbeddingProvider
|
||||
|
||||
|
||||
def _rrf_merge(
|
||||
fts_items: list[dict], vector_items: list[dict], k: int = 60
|
||||
) -> list[dict]:
|
||||
"""Reciprocal Rank Fusion. Merge FTS and vector results by id."""
|
||||
scores: dict[str, float] = {}
|
||||
for rank, item in enumerate(fts_items):
|
||||
cid = item["id"]
|
||||
scores[cid] = scores.get(cid, 0) + 1 / (k + rank + 1)
|
||||
for rank, item in enumerate(vector_items):
|
||||
cid = item["id"]
|
||||
scores[cid] = scores.get(cid, 0) + 1 / (k + rank + 1)
|
||||
|
||||
all_items = {x["id"]: x for x in fts_items + vector_items}
|
||||
sorted_ids = sorted(scores.keys(), key=lambda i: scores[i], reverse=True)
|
||||
return [all_items[i] for i in sorted_ids]
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class HybridRetriever:
|
||||
"""Combine FTS, vector, and metadata filter into evidence bundle."""
|
||||
"""向量 chunk 检索 + facts/timeline/summaries/stories。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -51,27 +37,27 @@ class HybridRetriever:
|
||||
)
|
||||
|
||||
q = query.strip()
|
||||
fts_chunks = await search_chunks_fts(
|
||||
self._db, user_id=user_id, query=query, limit=top_k * 2
|
||||
)
|
||||
|
||||
vector_chunks: list[dict] = []
|
||||
if self._embedding and q:
|
||||
merged_chunk_dicts: list[dict] = []
|
||||
if self._embedding:
|
||||
q_emb = await self._embedding.embed_text(q)
|
||||
if q_emb:
|
||||
vector_chunks = await search_chunks_vector(
|
||||
self._db, user_id=user_id, query_embedding=q_emb, limit=top_k * 2
|
||||
vector_rows = await search_chunks_vector(
|
||||
self._db, user_id, q_emb, limit=top_k
|
||||
)
|
||||
|
||||
merged = _rrf_merge(fts_chunks, vector_chunks)[:top_k]
|
||||
merged_chunk_dicts = [
|
||||
{
|
||||
"id": c["id"],
|
||||
"content": c["content"],
|
||||
"chunk_index": c.get("chunk_index", 0),
|
||||
}
|
||||
for c in merged
|
||||
]
|
||||
merged_chunk_dicts = [
|
||||
{
|
||||
"id": c["id"],
|
||||
"content": c["content"],
|
||||
"chunk_index": c.get("chunk_index", 0),
|
||||
}
|
||||
for c in vector_rows
|
||||
]
|
||||
else:
|
||||
logger.warning(
|
||||
"HybridRetriever empty_query_embedding user_id={}", user_id
|
||||
)
|
||||
else:
|
||||
logger.warning("HybridRetriever no_embedding_provider user_id={}", user_id)
|
||||
|
||||
return await retrieve_evidence_bundle_async(
|
||||
self._db,
|
||||
|
||||
Reference in New Issue
Block a user