ac49bc7f23
- Judge baseline excerpt and library chapter separately; build_memoir_compare_summary for gate, nine-dim and leaf deltas. - Memoir SSE chapter payload: baseline_judge, compare_summary, baseline_judge_error. - MemoirJudgeOutput: loose score coercion and post-validate clamp; memoir judge prompt caps from settings. - app-eval-web: two-column MemoirScoreCard layout, MemoirCompareSummary, chapter blocks and CSS. - Add memoir_compare_summary, log_events, celery_log_context, memoir_pipeline_progress; tests and migration 0014. - Misc: memory/evidence and enrichment paths, task/orchestrator updates, internal-eval docs, env examples.
451 lines
14 KiB
Python
451 lines
14 KiB
Python
"""
|
||
MemoryService — conversation / memoir 的统一门面。
|
||
|
||
- ingest_transcript: transcript -> memory_sources, chunks, embedding
|
||
- ingest 成功后：向 ``memory_idle`` 队列派发 LLM 富化（见 ``schedule_memory_enrichment``），不阻塞请求
|
||
- retrieve: 委托 HybridRetriever 返回 evidence bundle（向量 chunks）
|
||
|
||
Celery 侧使用 `ingest_transcript_sync` + `retrieve_evidence_sync`，与异步路径对齐见
|
||
`api/docs/memory-retrieval.md`。
|
||
"""
|
||
|
||
from sqlalchemy.ext.asyncio import AsyncSession
|
||
|
||
from app.core.logging import get_logger
|
||
from app.features.conversation.lineage_schemas import (
|
||
primary_user_message_id_from_lineage,
|
||
)
|
||
from app.features.memory.chunker import chunk_transcript
|
||
from app.features.memory.repo import (
|
||
create_chunk,
|
||
create_curation_action,
|
||
create_source,
|
||
set_chunk_excluded,
|
||
set_memory_fact_status,
|
||
update_chunk_embedding,
|
||
)
|
||
from app.features.memory.schemas import EvidenceBundle
|
||
from app.ports.embedding import EmbeddingProvider
|
||
|
||
logger = get_logger(__name__)
|
||
|
||
|
||
class MemoryService:
|
||
def __init__(
|
||
self,
|
||
db: AsyncSession,
|
||
*,
|
||
embedding_provider: EmbeddingProvider | None = None,
|
||
):
|
||
self._db = db
|
||
self._embedding = embedding_provider
|
||
|
||
async def ingest_transcript(
|
||
self,
|
||
user_id: str,
|
||
conversation_id: str,
|
||
transcript: str,
|
||
*,
|
||
lineage_json: dict | None = None,
|
||
) -> str:
|
||
"""
|
||
Ingest conversation transcript into memory.
|
||
Creates MemorySource, chunks, populates embedding.
|
||
Returns source_id.
|
||
"""
|
||
if not transcript or not transcript.strip():
|
||
raise ValueError("transcript cannot be empty")
|
||
|
||
primary_mid = (
|
||
primary_user_message_id_from_lineage(lineage_json)
|
||
if lineage_json
|
||
else None
|
||
)
|
||
source = await create_source(
|
||
self._db,
|
||
user_id=user_id,
|
||
source_type="transcript",
|
||
raw_text=transcript.strip(),
|
||
conversation_id=conversation_id,
|
||
lineage_json=lineage_json,
|
||
primary_user_message_id=primary_mid,
|
||
)
|
||
|
||
chunks_text = chunk_transcript(transcript.strip())
|
||
chunk_records = []
|
||
for i, content in enumerate(chunks_text):
|
||
chunk = await create_chunk(
|
||
self._db,
|
||
source_id=source.id,
|
||
user_id=user_id,
|
||
content=content,
|
||
chunk_index=i,
|
||
)
|
||
chunk_records.append((chunk.id, content))
|
||
|
||
await self._db.flush()
|
||
|
||
from app.core.config import settings
|
||
|
||
vectors_written = 0
|
||
# Embedding: 若有 provider 则写入
|
||
if self._embedding and chunk_records:
|
||
texts = [c for _, c in chunk_records]
|
||
embeddings = await self._embedding.embed_texts(texts)
|
||
for (chunk_id, _), emb in zip(chunk_records, embeddings):
|
||
if emb:
|
||
vectors_written += 1
|
||
await update_chunk_embedding(self._db, chunk_id, emb)
|
||
|
||
await self._db.commit()
|
||
emb_ok = self._embedding.is_available() if self._embedding else False
|
||
enrichment_task_id: str | None = None
|
||
try:
|
||
from app.tasks.memory_enrichment_tasks import schedule_memory_enrichment
|
||
|
||
enrichment_task_id = schedule_memory_enrichment(
|
||
user_id, source.id, memoir_correlation_id=None
|
||
)
|
||
except Exception as e:
|
||
logger.warning(
|
||
"memory enrichment 派发跳过: {} exc_type={}", e, type(e).__name__
|
||
)
|
||
|
||
logger.info(
|
||
"event=memory_ingest_done user_id={} conversation_id={} source_id={} "
|
||
"chunks={} vectors_written={} embedding_available={} enrichment_enabled={} enrichment_task_id={}",
|
||
user_id,
|
||
conversation_id,
|
||
source.id,
|
||
len(chunk_records),
|
||
vectors_written,
|
||
emb_ok,
|
||
settings.memory_enrichment_enabled,
|
||
enrichment_task_id,
|
||
)
|
||
return source.id
|
||
|
||
async def retrieve(
|
||
self, user_id: str, query: str, *, top_k: int = 10
|
||
) -> EvidenceBundle:
|
||
"""Retrieve relevant evidence. 委托 HybridRetriever。"""
|
||
from app.features.memory.retriever import HybridRetriever
|
||
|
||
retriever = HybridRetriever(self._db, embedding_provider=self._embedding)
|
||
raw = await retriever.retrieve(user_id=user_id, query=query, top_k=top_k)
|
||
bundle = EvidenceBundle.model_validate(raw)
|
||
bd = bundle.model_dump()
|
||
vec_ok = self._embedding.is_available() if self._embedding else False
|
||
logger.info(
|
||
"event=memory_retrieve_done user_id={} query_len={} top_k={} "
|
||
"chunks={} facts={} summaries={} timeline={} stories={} vector_ok={}",
|
||
user_id,
|
||
len((query or "").strip()),
|
||
top_k,
|
||
len(bd.get("relevant_chunks") or []),
|
||
len(bd.get("relevant_facts") or []),
|
||
len(bd.get("relevant_summaries") or []),
|
||
len(bd.get("timeline_hints") or []),
|
||
len(bd.get("relevant_stories") or []),
|
||
vec_ok,
|
||
)
|
||
return bundle
|
||
|
||
async def exclude_chunk(
|
||
self, user_id: str, chunk_id: str, *, reason: str = ""
|
||
) -> bool:
|
||
ok = await set_chunk_excluded(self._db, chunk_id, user_id, True)
|
||
if not ok:
|
||
return False
|
||
await create_curation_action(
|
||
self._db,
|
||
user_id=user_id,
|
||
action_type="exclude",
|
||
target_type="chunk",
|
||
target_id=chunk_id,
|
||
details={"reason": reason} if reason else None,
|
||
)
|
||
await self._db.commit()
|
||
return True
|
||
|
||
async def restore_chunk(self, user_id: str, chunk_id: str) -> bool:
|
||
ok = await set_chunk_excluded(self._db, chunk_id, user_id, False)
|
||
if not ok:
|
||
return False
|
||
await create_curation_action(
|
||
self._db,
|
||
user_id=user_id,
|
||
action_type="restore",
|
||
target_type="chunk",
|
||
target_id=chunk_id,
|
||
details=None,
|
||
)
|
||
await self._db.commit()
|
||
return True
|
||
|
||
async def confirm_fact(self, user_id: str, fact_id: str) -> bool:
|
||
ok = await set_memory_fact_status(self._db, fact_id, user_id, "confirmed")
|
||
if not ok:
|
||
return False
|
||
await create_curation_action(
|
||
self._db,
|
||
user_id=user_id,
|
||
action_type="confirm",
|
||
target_type="fact",
|
||
target_id=fact_id,
|
||
details=None,
|
||
)
|
||
await self._db.commit()
|
||
return True
|
||
|
||
async def reject_fact(
|
||
self, user_id: str, fact_id: str, *, reason: str = ""
|
||
) -> bool:
|
||
ok = await set_memory_fact_status(self._db, fact_id, user_id, "rejected")
|
||
if not ok:
|
||
return False
|
||
await create_curation_action(
|
||
self._db,
|
||
user_id=user_id,
|
||
action_type="reject",
|
||
target_type="fact",
|
||
target_id=fact_id,
|
||
details={"reason": reason} if reason else None,
|
||
)
|
||
await self._db.commit()
|
||
return ok
|
||
|
||
|
||
def ingest_transcript_sync(
|
||
session,
|
||
user_id: str,
|
||
conversation_id: str,
|
||
transcript: str,
|
||
*,
|
||
lineage_json: dict | None = None,
|
||
) -> str:
|
||
"""
|
||
Sync transcript ingest for Celery tasks.
|
||
Creates source + chunks, and best-effort populates embeddings.
|
||
Returns source_id.
|
||
"""
|
||
from app.core.dependencies import get_embedding_provider
|
||
from app.features.memory.chunker import chunk_transcript
|
||
from app.features.memory.repo import (
|
||
create_chunk_sync,
|
||
create_source_sync,
|
||
update_chunk_embedding_sync,
|
||
)
|
||
|
||
if not transcript or not transcript.strip():
|
||
raise ValueError("transcript cannot be empty")
|
||
|
||
primary_mid = (
|
||
primary_user_message_id_from_lineage(lineage_json) if lineage_json else None
|
||
)
|
||
source = create_source_sync(
|
||
session,
|
||
user_id=user_id,
|
||
source_type="transcript",
|
||
raw_text=transcript.strip(),
|
||
conversation_id=conversation_id,
|
||
lineage_json=lineage_json,
|
||
primary_user_message_id=primary_mid,
|
||
)
|
||
session.flush()
|
||
|
||
chunks_text = chunk_transcript(transcript.strip())
|
||
chunk_records: list[tuple[str, str]] = []
|
||
for i, content in enumerate(chunks_text):
|
||
chunk = create_chunk_sync(
|
||
session,
|
||
source_id=source.id,
|
||
user_id=user_id,
|
||
content=content,
|
||
chunk_index=i,
|
||
)
|
||
session.flush()
|
||
chunk_records.append((chunk.id, content))
|
||
|
||
from app.core.config import settings
|
||
|
||
vectors_written = 0
|
||
embedding_available = False
|
||
|
||
try:
|
||
embedding_provider = get_embedding_provider()
|
||
if embedding_provider is not None:
|
||
embedding_available = embedding_provider.is_available()
|
||
except Exception as e:
|
||
logger.warning(
|
||
"memory embedding provider 不可用(sync): {} exc_type={}",
|
||
e,
|
||
type(e).__name__,
|
||
)
|
||
embedding_provider = None
|
||
|
||
# 向量写入在 SAVEPOINT 内；失败仅回滚本段，source/chunks 主体仍可由外层提交。
|
||
# LLM enrichment 在 commit 后由 schedule_memory_enrichment 入 memory_idle 队列。
|
||
try:
|
||
with session.begin_nested():
|
||
if chunk_records and embedding_provider is not None:
|
||
texts = [content for _, content in chunk_records]
|
||
embeddings = embedding_provider.embed_texts_sync(texts)
|
||
for (chunk_id, _), emb in zip(chunk_records, embeddings):
|
||
if emb:
|
||
vectors_written += 1
|
||
update_chunk_embedding_sync(session, chunk_id, emb)
|
||
except Exception as e:
|
||
logger.warning(
|
||
"memory embedding 跳过(sync): {} exc_type={}",
|
||
e,
|
||
type(e).__name__,
|
||
)
|
||
|
||
session.commit()
|
||
|
||
enrichment_task_id: str | None = None
|
||
if settings.memory_enrichment_enabled:
|
||
try:
|
||
from app.tasks.memory_enrichment_tasks import schedule_memory_enrichment
|
||
|
||
enrichment_task_id = schedule_memory_enrichment(
|
||
user_id, source.id, memoir_correlation_id=None
|
||
)
|
||
except Exception as e:
|
||
logger.warning(
|
||
"memory enrichment 任务派发失败: {} exc_type={}",
|
||
e,
|
||
type(e).__name__,
|
||
)
|
||
|
||
logger.info(
|
||
"event=memory_ingest_done user_id={} conversation_id={} source_id={} "
|
||
"chunks={} vectors_written={} embedding_available={} enrichment_enabled={} enrichment_task_id={} sync=1",
|
||
user_id,
|
||
conversation_id,
|
||
source.id,
|
||
len(chunk_records),
|
||
vectors_written,
|
||
embedding_available,
|
||
settings.memory_enrichment_enabled,
|
||
enrichment_task_id,
|
||
)
|
||
return source.id
|
||
|
||
|
||
def ingest_transcripts_batch_sync(
|
||
session,
|
||
user_id: str,
|
||
items: list[tuple[str, str, dict | None]],
|
||
) -> list[str]:
|
||
"""
|
||
Phase1 批量：多段 transcript 在同一会话内建 source/chunks，并单次 embed_texts_sync（在适配器 batch 限制内）。
|
||
|
||
不 commit；不派发 enrichment（由调用方 commit 后 ``schedule_enrichment_for_sources``）。
|
||
items: (conversation_id, transcript, lineage_json)
|
||
返回与有效 items 顺序一致的 source_id 列表。
|
||
"""
|
||
from app.core.dependencies import get_embedding_provider
|
||
from app.features.memory.chunker import chunk_transcript
|
||
from app.features.memory.repo import (
|
||
create_chunk_sync,
|
||
create_source_sync,
|
||
update_chunk_embedding_sync,
|
||
)
|
||
|
||
source_ids: list[str] = []
|
||
all_chunk_records: list[tuple[str, str]] = []
|
||
|
||
for conversation_id, transcript, lineage_json in items:
|
||
text = (transcript or "").strip()
|
||
if not text:
|
||
continue
|
||
primary_mid = (
|
||
primary_user_message_id_from_lineage(lineage_json)
|
||
if lineage_json
|
||
else None
|
||
)
|
||
source = create_source_sync(
|
||
session,
|
||
user_id=user_id,
|
||
source_type="transcript",
|
||
raw_text=text,
|
||
conversation_id=conversation_id or None,
|
||
lineage_json=lineage_json,
|
||
primary_user_message_id=primary_mid,
|
||
)
|
||
session.flush()
|
||
|
||
chunks_text = chunk_transcript(text)
|
||
for i, content in enumerate(chunks_text):
|
||
chunk = create_chunk_sync(
|
||
session,
|
||
source_id=source.id,
|
||
user_id=user_id,
|
||
content=content,
|
||
chunk_index=i,
|
||
)
|
||
session.flush()
|
||
all_chunk_records.append((chunk.id, content))
|
||
source_ids.append(source.id)
|
||
|
||
embedding_provider = None
|
||
try:
|
||
embedding_provider = get_embedding_provider()
|
||
except Exception as e:
|
||
logger.warning(
|
||
"memory embedding provider 不可用(batch sync): {} exc_type={}",
|
||
e,
|
||
type(e).__name__,
|
||
)
|
||
|
||
vectors_written = 0
|
||
try:
|
||
with session.begin_nested():
|
||
if all_chunk_records and embedding_provider is not None:
|
||
texts = [content for _, content in all_chunk_records]
|
||
embeddings = embedding_provider.embed_texts_sync(texts)
|
||
for (chunk_id, _), emb in zip(all_chunk_records, embeddings):
|
||
if emb:
|
||
vectors_written += 1
|
||
update_chunk_embedding_sync(session, chunk_id, emb)
|
||
except Exception as e:
|
||
logger.warning(
|
||
"memory embedding 跳过(batch sync): {} exc_type={}",
|
||
e,
|
||
type(e).__name__,
|
||
)
|
||
|
||
emb_ok = (
|
||
embedding_provider.is_available()
|
||
if embedding_provider is not None
|
||
else False
|
||
)
|
||
logger.info(
|
||
"event=memory_ingest_batch_done user_id={} sources={} chunks={} "
|
||
"vectors_written={} embedding_available={}",
|
||
user_id,
|
||
len(source_ids),
|
||
len(all_chunk_records),
|
||
vectors_written,
|
||
emb_ok,
|
||
)
|
||
return source_ids
|
||
|
||
|
||
def schedule_enrichment_for_sources(
|
||
user_id: str,
|
||
source_ids: list[str],
|
||
*,
|
||
memoir_correlation_id: str | None = None,
|
||
) -> None:
|
||
"""After successful ingest commit, enqueue LLM enrichment for each source (memory_idle queue)."""
|
||
from app.tasks.memory_enrichment_tasks import schedule_memory_enrichment
|
||
|
||
for sid in source_ids:
|
||
if sid:
|
||
schedule_memory_enrichment(
|
||
user_id, sid, memoir_correlation_id=memoir_correlation_id
|
||
)
|