feat(api)!: memory single chain — async MemoryService, strict eval closure

Route all memory ingest/retrieve/enrichment/compaction through async MemoryService.
Remove legacy sync memory implementations (ingest/retrieve/compaction); Celery and
memoir Phase2 call asyncio.run into MemoryService-backed helpers.

Memoir Phase1 batch ingest uses MemoryService.ingest_transcripts_batch; drop chapters.
evidence_bundle_json mirror (Alembic 0015). Evaluation uses snapshot/link-only bundles;
raise EvidenceClosureMissing instead of partial/fallback lineage tiers.

Split memoir state into NarrativeCoverageState and InterviewControlState; delete the
_interview_meta_store adapter layer. Remove rolling-query and recent-fact fallback
settings from config and evidence assembly.

Update judges, docs, tests, and PlaygroundPage alignment.

Made-with: Cursor
This commit is contained in:
Kevin
2026-04-30 14:11:46 +08:00
parent ac436b87a2
commit 71fbd39e32
53 changed files with 953 additions and 2448 deletions

View File

@@ -2,6 +2,7 @@
回忆录处理 Celery 任务
"""
import asyncio
import json
import time
import uuid
@@ -26,7 +27,8 @@ from app.core.chapter_pipeline_lock import (
release_chapter_pipeline_lock as _release_chapter_lock,
)
from app.core.config import settings
from app.core.db import get_sync_db
from app.core.db import AsyncSessionLocal, get_sync_db
from app.core.dependencies import get_embedding_provider
from app.core.llm_gateway import LlmGateway, LlmUseCase
from app.core.logging import get_logger
from app.core.memoir_pipeline_progress import (
@@ -65,10 +67,7 @@ from app.features.memoir.state_service import (
from app.features.memoir.story_pipeline_sync import (
run_story_pipeline_for_category_batch,
)
from app.features.memory.service import (
ingest_transcripts_batch_sync,
schedule_enrichment_for_sources,
)
from app.features.memory.service import MemoryService
from app.features.user.models import User
from app.tasks.celery_app import celery_app
@@ -144,6 +143,33 @@ def _get_llm_fast():
return None
async def _memory_ingest_transcripts_batch(
user_id: str,
items: list[tuple[str, str, dict | None]],
*,
memoir_correlation_id: str,
) -> list[str]:
async with AsyncSessionLocal() as db:
service = MemoryService(db, embedding_provider=get_embedding_provider())
return await service.ingest_transcripts_batch(
user_id,
items,
memoir_correlation_id=memoir_correlation_id,
)
async def _memory_retrieve_evidence(
user_id: str,
query: str,
*,
top_k: int,
) -> dict:
async with AsyncSessionLocal() as db:
service = MemoryService(db, embedding_provider=get_embedding_provider())
bundle = await service.retrieve(user_id, query, top_k=top_k)
return bundle.model_dump()
def _get_redis_client(*, decode_responses: bool = False) -> redis.Redis:
from app.core.config import settings
@@ -557,6 +583,29 @@ def process_memoir_phase2(
return {"status": "noop"}
state = get_or_create_state_sync(user_id, db)
segment_texts = [seg.user_input_text or "" for seg in category_segments]
combined_text = "\n\n".join(segment_texts)
n_units = len(category_segments)
evidence_top_k = int(settings.evidence_top_k_default)
if n_units > int(settings.evidence_large_batch_threshold):
evidence_top_k = int(settings.evidence_top_k_large_batch)
try:
memory_evidence = asyncio.run(
_memory_retrieve_evidence(
user_id,
combined_text,
top_k=evidence_top_k,
)
)
except Exception as e:
logger.warning("Evidence 检索跳过: {}", e)
memory_evidence = {
"relevant_chunks": [],
"relevant_summaries": [],
"relevant_facts": [],
"timeline_hints": [],
"relevant_stories": [],
}
pipeline_t0 = time.perf_counter()
chapter, needs_cover, disp = run_story_pipeline_for_category_batch(
db,
@@ -571,6 +620,7 @@ def process_memoir_phase2(
occupation=user_occupation,
memoir_correlation_id=cid,
llm_fast=llm_fast,
memory_evidence=memory_evidence,
)
pipeline_elapsed = time.perf_counter() - pipeline_t0
story_dispatch_ids |= disp
@@ -786,8 +836,12 @@ def process_memoir_phase1(self, user_id: str, segment_ids: List[str]):
ingested_source_ids: list[str] = []
if ingest_items:
try:
ingested_source_ids = ingest_transcripts_batch_sync(
db, user_id, ingest_items
ingested_source_ids = asyncio.run(
_memory_ingest_transcripts_batch(
user_id,
ingest_items,
memoir_correlation_id=memoir_correlation_id,
)
)
for seg, sid in zip(
non_empty_segments, ingested_source_ids, strict=True
@@ -927,13 +981,6 @@ def process_memoir_phase1(self, user_id: str, segment_ids: List[str]):
},
)
if ingested_source_ids:
schedule_enrichment_for_sources(
user_id,
ingested_source_ids,
memoir_correlation_id=memoir_correlation_id,
)
for cc in phase2_immediate:
p2tid = _dispatch_phase2_immediate(user_id, cc, memoir_correlation_id)
if p2tid: