kevin/life-echo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'eval/elapsed-time-memoir-batch-chunk' into development

Browse Source
This commit is contained in:
Kevin
2026-04-10 10:27:41 +08:00
parent 31fc1c24cf ac49bc7f23
commit 204ae24697
66 changed files with 5246 additions and 705 deletions
Download Patch File Download Diff File Expand all files Collapse all files

171
api/app/features/memory/service.py
View File

@@ -2,7 +2,7 @@
MemoryService — conversation / memoir 的统一门面。
- ingest_transcript: transcript -> memory_sources, chunks, embedding
- ingest 后可选：LLM 富化（session/rolling 摘要、事实、时间线）
- ingest 成功后：向 ``memory_idle`` 队列派发 LLM 富化（见 ``schedule_memory_enrichment``），不阻塞请求
- retrieve: 委托 HybridRetriever 返回 evidence bundle（向量 chunks）
Celery 侧使用 `ingest_transcript_sync` + `retrieve_evidence_sync`，与异步路径对齐见
@@ -12,6 +12,9 @@ Celery 侧使用 `ingest_transcript_sync` + `retrieve_evidence_sync`，与异步
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.logging import get_logger
from app.features.conversation.lineage_schemas import (
primary_user_message_id_from_lineage,
)
from app.features.memory.chunker import chunk_transcript
from app.features.memory.repo import (
create_chunk,
@@ -95,29 +98,23 @@ class MemoryService:
vectors_written += 1
await update_chunk_embedding(self._db, chunk_id, emb)
enrichment_ok: bool | None = None
try:
from app.core.dependencies import get_llm_provider_fast
from app.features.memory.enrichment import enrich_memory_after_ingest_async
if settings.memory_enrichment_enabled:
llm = get_llm_provider_fast().langchain_llm
await enrich_memory_after_ingest_async(
self._db, user_id, source.id, llm
)
enrichment_ok = True
except Exception as e:
if settings.memory_enrichment_enabled:
enrichment_ok = False
logger.warning(
"memory enrichment 跳过: {} exc_type={}", e, type(e).__name__
)
await self._db.commit()
emb_ok = self._embedding.is_available() if self._embedding else False
enrichment_task_id: str | None = None
try:
from app.tasks.memory_enrichment_tasks import schedule_memory_enrichment
enrichment_task_id = schedule_memory_enrichment(
user_id, source.id, memoir_correlation_id=None
)
except Exception as e:
logger.warning(
"memory enrichment 派发跳过: {} exc_type={}", e, type(e).__name__
)
logger.info(
"event=memory_ingest_done user_id={} conversation_id={} source_id={} "
"chunks={} vectors_written={} embedding_available={} enrichment_enabled={} enrichment_ok={}",
"chunks={} vectors_written={} embedding_available={} enrichment_enabled={} enrichment_task_id={}",
user_id,
conversation_id,
source.id,
@@ -125,7 +122,7 @@ class MemoryService:
vectors_written,
emb_ok,
settings.memory_enrichment_enabled,
enrichment_ok,
enrichment_task_id,
)
return source.id
@@ -275,7 +272,6 @@ def ingest_transcript_sync(
vectors_written = 0
embedding_available = False
enrichment_ok: bool | None = None
try:
embedding_provider = get_embedding_provider()
@@ -290,7 +286,7 @@ def ingest_transcript_sync(
embedding_provider = None
# 向量写入在 SAVEPOINT 内；失败仅回滚本段，source/chunks 主体仍可由外层提交。
# enrichment 已迁移到独立异步任务 (memory_enrichment_tasks.enrich_memory_source)。
# LLM enrichment 在 commit 后由 schedule_memory_enrichment 入 memory_idle 队列。
try:
with session.begin_nested():
if chunk_records and embedding_provider is not None:
@@ -309,14 +305,15 @@ def ingest_transcript_sync(
session.commit()
enrichment_task_id: str | None = None
if settings.memory_enrichment_enabled:
try:
from app.tasks.memory_enrichment_tasks import enrich_memory_source
from app.tasks.memory_enrichment_tasks import schedule_memory_enrichment
enrich_memory_source.delay(user_id, source.id)
enrichment_ok = True
enrichment_task_id = schedule_memory_enrichment(
user_id, source.id, memoir_correlation_id=None
)
except Exception as e:
enrichment_ok = False
logger.warning(
"memory enrichment 任务派发失败: {} exc_type={}",
e,
@@ -325,7 +322,7 @@ def ingest_transcript_sync(
logger.info(
"event=memory_ingest_done user_id={} conversation_id={} source_id={} "
"chunks={} vectors_written={} embedding_available={} enrichment_enabled={} enrichment_ok={} sync=1",
"chunks={} vectors_written={} embedding_available={} enrichment_enabled={} enrichment_task_id={} sync=1",
user_id,
conversation_id,
source.id,
@@ -333,6 +330,122 @@ def ingest_transcript_sync(
vectors_written,
embedding_available,
settings.memory_enrichment_enabled,
enrichment_ok,
enrichment_task_id,
)
return source.id
def ingest_transcripts_batch_sync(
session,
user_id: str,
items: list[tuple[str, str, dict | None]],
) -> list[str]:
"""
Phase1 批量：多段 transcript 在同一会话内建 source/chunks，并单次 embed_texts_sync（在适配器 batch 限制内）。
不 commit；不派发 enrichment（由调用方 commit 后 ``schedule_enrichment_for_sources``）。
items: (conversation_id, transcript, lineage_json)
返回与有效 items 顺序一致的 source_id 列表。
"""
from app.core.dependencies import get_embedding_provider
from app.features.memory.chunker import chunk_transcript
from app.features.memory.repo import (
create_chunk_sync,
create_source_sync,
update_chunk_embedding_sync,
)
source_ids: list[str] = []
all_chunk_records: list[tuple[str, str]] = []
for conversation_id, transcript, lineage_json in items:
text = (transcript or "").strip()
if not text:
continue
primary_mid = (
primary_user_message_id_from_lineage(lineage_json)
if lineage_json
else None
)
source = create_source_sync(
session,
user_id=user_id,
source_type="transcript",
raw_text=text,
conversation_id=conversation_id or None,
lineage_json=lineage_json,
primary_user_message_id=primary_mid,
)
session.flush()
chunks_text = chunk_transcript(text)
for i, content in enumerate(chunks_text):
chunk = create_chunk_sync(
session,
source_id=source.id,
user_id=user_id,
content=content,
chunk_index=i,
)
session.flush()
all_chunk_records.append((chunk.id, content))
source_ids.append(source.id)
embedding_provider = None
try:
embedding_provider = get_embedding_provider()
except Exception as e:
logger.warning(
"memory embedding provider 不可用(batch sync): {} exc_type={}",
e,
type(e).__name__,
)
vectors_written = 0
try:
with session.begin_nested():
if all_chunk_records and embedding_provider is not None:
texts = [content for _, content in all_chunk_records]
embeddings = embedding_provider.embed_texts_sync(texts)
for (chunk_id, _), emb in zip(all_chunk_records, embeddings):
if emb:
vectors_written += 1
update_chunk_embedding_sync(session, chunk_id, emb)
except Exception as e:
logger.warning(
"memory embedding 跳过(batch sync): {} exc_type={}",
e,
type(e).__name__,
)
emb_ok = (
embedding_provider.is_available()
if embedding_provider is not None
else False
)
logger.info(
"event=memory_ingest_batch_done user_id={} sources={} chunks={} "
"vectors_written={} embedding_available={}",
user_id,
len(source_ids),
len(all_chunk_records),
vectors_written,
emb_ok,
)
return source_ids
def schedule_enrichment_for_sources(
user_id: str,
source_ids: list[str],
*,
memoir_correlation_id: str | None = None,
) -> None:
"""After successful ingest commit, enqueue LLM enrichment for each source (memory_idle queue)."""
from app.tasks.memory_enrichment_tasks import schedule_memory_enrichment
for sid in source_ids:
if sid:
schedule_memory_enrichment(
user_id, sid, memoir_correlation_id=memoir_correlation_id
)