feat(api): 收敛对话与记忆流程边界,引入 LLM 网关与专用服务
- MemoryService 异步路径委托 MemoryIngestService / MemoryRetrievalService;富化派发经 MemoryEnrichmentScheduler - WebSocket pipeline 经 ChatTurnService 与显式 DTO 编排单轮对话;回忆录片段入队由 MemoirIngestScheduler 封装 - 新增 LlmGateway(LlmUseCase),各 agent、任务与适配器对齐 ports - 补充 memory 提示适配、runtime 类型、memory-retrieval 文档、ai-touchpoints 说明与扫描脚本及配套测试 Made-with: Cursor
This commit is contained in:
@@ -15,18 +15,14 @@ from app.core.logging import get_logger
|
||||
from app.features.conversation.lineage_schemas import (
|
||||
primary_user_message_id_from_lineage,
|
||||
)
|
||||
from app.features.memory.chunker import chunk_transcript
|
||||
from app.features.memory.enrichment_scheduler import MemoryEnrichmentScheduler
|
||||
from app.features.memory.ingest_service import MemoryIngestService
|
||||
from app.features.memory.repo import (
|
||||
create_chunk,
|
||||
create_curation_action,
|
||||
create_source,
|
||||
set_chunk_excluded,
|
||||
set_memory_fact_status,
|
||||
update_chunk_embedding,
|
||||
)
|
||||
from app.features.conversation.lineage_schemas import (
|
||||
primary_user_message_id_from_lineage,
|
||||
)
|
||||
from app.features.memory.retrieval_service import MemoryRetrievalService
|
||||
from app.features.memory.schemas import EvidenceBundle
|
||||
from app.ports.embedding import EmbeddingProvider
|
||||
|
||||
@@ -56,101 +52,20 @@ class MemoryService:
|
||||
Creates MemorySource, chunks, populates embedding.
|
||||
Returns source_id.
|
||||
"""
|
||||
if not transcript or not transcript.strip():
|
||||
raise ValueError("transcript cannot be empty")
|
||||
|
||||
primary_mid = (
|
||||
primary_user_message_id_from_lineage(lineage_json) if lineage_json else None
|
||||
)
|
||||
source = await create_source(
|
||||
self._db,
|
||||
user_id=user_id,
|
||||
source_type="transcript",
|
||||
raw_text=transcript.strip(),
|
||||
conversation_id=conversation_id,
|
||||
lineage_json=lineage_json,
|
||||
primary_user_message_id=primary_mid,
|
||||
)
|
||||
|
||||
chunks_text = chunk_transcript(transcript.strip())
|
||||
chunk_records = []
|
||||
for i, content in enumerate(chunks_text):
|
||||
chunk = await create_chunk(
|
||||
self._db,
|
||||
source_id=source.id,
|
||||
user_id=user_id,
|
||||
content=content,
|
||||
chunk_index=i,
|
||||
)
|
||||
chunk_records.append((chunk.id, content))
|
||||
|
||||
await self._db.flush()
|
||||
|
||||
from app.core.config import settings
|
||||
|
||||
vectors_written = 0
|
||||
# Embedding: 若有 provider 则写入
|
||||
if self._embedding and chunk_records:
|
||||
texts = [c for _, c in chunk_records]
|
||||
embeddings = await self._embedding.embed_texts(texts)
|
||||
for (chunk_id, _), emb in zip(chunk_records, embeddings):
|
||||
if emb:
|
||||
vectors_written += 1
|
||||
await update_chunk_embedding(self._db, chunk_id, emb)
|
||||
|
||||
await self._db.commit()
|
||||
emb_ok = self._embedding.is_available() if self._embedding else False
|
||||
enrichment_task_id: str | None = None
|
||||
try:
|
||||
from app.tasks.memory_enrichment_tasks import schedule_memory_enrichment
|
||||
|
||||
enrichment_task_id = schedule_memory_enrichment(
|
||||
user_id, source.id, memoir_correlation_id=None
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"memory enrichment 派发跳过: {} exc_type={}", e, type(e).__name__
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"event=memory_ingest_done user_id={} conversation_id={} source_id={} "
|
||||
"chunks={} vectors_written={} embedding_available={} enrichment_enabled={} enrichment_task_id={}",
|
||||
service = MemoryIngestService(self._db, embedding_provider=self._embedding)
|
||||
return await service.ingest_transcript(
|
||||
user_id,
|
||||
conversation_id,
|
||||
source.id,
|
||||
len(chunk_records),
|
||||
vectors_written,
|
||||
emb_ok,
|
||||
settings.memory_enrichment_enabled,
|
||||
enrichment_task_id,
|
||||
transcript,
|
||||
lineage_json=lineage_json,
|
||||
)
|
||||
return source.id
|
||||
|
||||
async def retrieve(
|
||||
self, user_id: str, query: str, *, top_k: int = 10
|
||||
) -> EvidenceBundle:
|
||||
"""Retrieve relevant evidence. 委托 HybridRetriever。"""
|
||||
from app.features.memory.retriever import HybridRetriever
|
||||
|
||||
retriever = HybridRetriever(self._db, embedding_provider=self._embedding)
|
||||
raw = await retriever.retrieve(user_id=user_id, query=query, top_k=top_k)
|
||||
bundle = EvidenceBundle.model_validate(raw)
|
||||
bd = bundle.model_dump()
|
||||
vec_ok = self._embedding.is_available() if self._embedding else False
|
||||
logger.info(
|
||||
"event=memory_retrieve_done user_id={} query_len={} top_k={} "
|
||||
"chunks={} facts={} summaries={} timeline={} stories={} vector_ok={}",
|
||||
user_id,
|
||||
len((query or "").strip()),
|
||||
top_k,
|
||||
len(bd.get("relevant_chunks") or []),
|
||||
len(bd.get("relevant_facts") or []),
|
||||
len(bd.get("relevant_summaries") or []),
|
||||
len(bd.get("timeline_hints") or []),
|
||||
len(bd.get("relevant_stories") or []),
|
||||
vec_ok,
|
||||
)
|
||||
return bundle
|
||||
service = MemoryRetrievalService(self._db, embedding_provider=self._embedding)
|
||||
return await service.retrieve(user_id, query, top_k=top_k)
|
||||
|
||||
async def exclude_chunk(
|
||||
self, user_id: str, chunk_id: str, *, reason: str = ""
|
||||
@@ -292,7 +207,9 @@ def ingest_transcript_sync(
|
||||
if chunk_records and embedding_provider is not None:
|
||||
texts = [content for _, content in chunk_records]
|
||||
embeddings = embedding_provider.embed_texts_sync(texts)
|
||||
for (chunk_id, _), emb in zip(chunk_records, embeddings):
|
||||
for (chunk_id, _), emb in zip(
|
||||
chunk_records, embeddings, strict=False
|
||||
):
|
||||
if emb:
|
||||
vectors_written += 1
|
||||
update_chunk_embedding_sync(session, chunk_id, emb)
|
||||
@@ -405,7 +322,9 @@ def ingest_transcripts_batch_sync(
|
||||
if all_chunk_records and embedding_provider is not None:
|
||||
texts = [content for _, content in all_chunk_records]
|
||||
embeddings = embedding_provider.embed_texts_sync(texts)
|
||||
for (chunk_id, _), emb in zip(all_chunk_records, embeddings):
|
||||
for (chunk_id, _), emb in zip(
|
||||
all_chunk_records, embeddings, strict=False
|
||||
):
|
||||
if emb:
|
||||
vectors_written += 1
|
||||
update_chunk_embedding_sync(session, chunk_id, emb)
|
||||
@@ -438,10 +357,8 @@ def schedule_enrichment_for_sources(
|
||||
memoir_correlation_id: str | None = None,
|
||||
) -> None:
|
||||
"""After successful ingest commit, enqueue LLM enrichment for each source (memory_idle queue)."""
|
||||
from app.tasks.memory_enrichment_tasks import schedule_memory_enrichment
|
||||
|
||||
for sid in source_ids:
|
||||
if sid:
|
||||
schedule_memory_enrichment(
|
||||
user_id, sid, memoir_correlation_id=memoir_correlation_id
|
||||
)
|
||||
MemoryEnrichmentScheduler().schedule_many(
|
||||
user_id,
|
||||
source_ids,
|
||||
memoir_correlation_id=memoir_correlation_id,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user