feat(api): Memory compaction 管线与调度修复,同步环境变量示例
Memory compaction(近重复 chunk 软排除) - 新增 compaction 调度:Redis debounce、scheduler gate、增量游标;任务结束时 finalize,避免 gate 长期占用并处理运行期新 trigger。 - Celery memory_compaction_run:debounce 未到点则 retry;用户级 Redis 锁;成功路径更新游标并 finalize;异常时释放 scheduler gate 并 self.retry,避免静默卡死调度与瞬时失败不重试。 - compaction_service:多层判定 + canonical 打分;无 embedding 时停止前移游标(awaiting_embeddings);curation details 补全 trigger 等上下文。 - ingest_transcript_sync:同步路径尽力写入 embedding,与异步 ingest 行为对齐,避免 compaction 永远扫不到无向量 chunk。 - repo:新增 update_chunk_embedding_sync。 测试 - 扩展 test_memory_compaction:调度合并、finalize、ingest embedding、无向量游标、异常路径 gate+retry 等回归用
This commit is contained in:
@@ -9,6 +9,8 @@ Celery 侧使用 `ingest_transcript_sync` + `retrieve_evidence_sync`,与异步
|
||||
`api/docs/memory-retrieval.md`。
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.logging import get_logger
|
||||
@@ -184,13 +186,15 @@ def ingest_transcript_sync(
|
||||
) -> str:
|
||||
"""
|
||||
Sync transcript ingest for Celery tasks.
|
||||
Creates source + chunks + FTS. Skips embedding (async).
|
||||
Creates source + chunks + FTS, and best-effort populates embeddings.
|
||||
Returns source_id.
|
||||
"""
|
||||
from app.core.dependencies import get_embedding_provider
|
||||
from app.features.memory.chunker import chunk_transcript
|
||||
from app.features.memory.repo import (
|
||||
create_chunk_sync,
|
||||
create_source_sync,
|
||||
update_chunk_embedding_sync,
|
||||
update_chunk_fts_sync,
|
||||
)
|
||||
|
||||
@@ -207,6 +211,7 @@ def ingest_transcript_sync(
|
||||
session.flush()
|
||||
|
||||
chunks_text = chunk_transcript(transcript.strip())
|
||||
chunk_records: list[tuple[str, str]] = []
|
||||
for i, content in enumerate(chunks_text):
|
||||
chunk = create_chunk_sync(
|
||||
session,
|
||||
@@ -216,8 +221,22 @@ def ingest_transcript_sync(
|
||||
chunk_index=i,
|
||||
)
|
||||
session.flush()
|
||||
chunk_records.append((chunk.id, content))
|
||||
update_chunk_fts_sync(session, chunk.id)
|
||||
|
||||
try:
|
||||
embedding_provider = get_embedding_provider()
|
||||
if chunk_records and embedding_provider is not None:
|
||||
texts = [content for _, content in chunk_records]
|
||||
embeddings = asyncio.run(embedding_provider.embed_texts(texts))
|
||||
for (chunk_id, _), emb in zip(chunk_records, embeddings):
|
||||
if emb:
|
||||
update_chunk_embedding_sync(session, chunk_id, emb)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"memory embedding 跳过(sync): {} exc_type={}", e, type(e).__name__
|
||||
)
|
||||
|
||||
try:
|
||||
from app.core.config import settings
|
||||
from app.features.memory.enrichment import enrich_memory_after_ingest_sync
|
||||
|
||||
Reference in New Issue
Block a user