feat(api)!: memory single chain — async MemoryService, strict eval closure
Route all memory ingest/retrieve/enrichment/compaction through async MemoryService. Remove legacy sync memory implementations (ingest/retrieve/compaction); Celery and memoir Phase2 call asyncio.run into MemoryService-backed helpers. Memoir Phase1 batch ingest uses MemoryService.ingest_transcripts_batch; drop chapters. evidence_bundle_json mirror (Alembic 0015). Evaluation uses snapshot/link-only bundles; raise EvidenceClosureMissing instead of partial/fallback lineage tiers. Split memoir state into NarrativeCoverageState and InterviewControlState; delete the _interview_meta_store adapter layer. Remove rolling-query and recent-fact fallback settings from config and evidence assembly. Update judges, docs, tests, and PlaygroundPage alignment. Made-with: Cursor
This commit is contained in:
@@ -2,6 +2,7 @@
|
||||
回忆录处理 Celery 任务
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
@@ -26,7 +27,8 @@ from app.core.chapter_pipeline_lock import (
|
||||
release_chapter_pipeline_lock as _release_chapter_lock,
|
||||
)
|
||||
from app.core.config import settings
|
||||
from app.core.db import get_sync_db
|
||||
from app.core.db import AsyncSessionLocal, get_sync_db
|
||||
from app.core.dependencies import get_embedding_provider
|
||||
from app.core.llm_gateway import LlmGateway, LlmUseCase
|
||||
from app.core.logging import get_logger
|
||||
from app.core.memoir_pipeline_progress import (
|
||||
@@ -65,10 +67,7 @@ from app.features.memoir.state_service import (
|
||||
from app.features.memoir.story_pipeline_sync import (
|
||||
run_story_pipeline_for_category_batch,
|
||||
)
|
||||
from app.features.memory.service import (
|
||||
ingest_transcripts_batch_sync,
|
||||
schedule_enrichment_for_sources,
|
||||
)
|
||||
from app.features.memory.service import MemoryService
|
||||
from app.features.user.models import User
|
||||
from app.tasks.celery_app import celery_app
|
||||
|
||||
@@ -144,6 +143,33 @@ def _get_llm_fast():
|
||||
return None
|
||||
|
||||
|
||||
async def _memory_ingest_transcripts_batch(
|
||||
user_id: str,
|
||||
items: list[tuple[str, str, dict | None]],
|
||||
*,
|
||||
memoir_correlation_id: str,
|
||||
) -> list[str]:
|
||||
async with AsyncSessionLocal() as db:
|
||||
service = MemoryService(db, embedding_provider=get_embedding_provider())
|
||||
return await service.ingest_transcripts_batch(
|
||||
user_id,
|
||||
items,
|
||||
memoir_correlation_id=memoir_correlation_id,
|
||||
)
|
||||
|
||||
|
||||
async def _memory_retrieve_evidence(
|
||||
user_id: str,
|
||||
query: str,
|
||||
*,
|
||||
top_k: int,
|
||||
) -> dict:
|
||||
async with AsyncSessionLocal() as db:
|
||||
service = MemoryService(db, embedding_provider=get_embedding_provider())
|
||||
bundle = await service.retrieve(user_id, query, top_k=top_k)
|
||||
return bundle.model_dump()
|
||||
|
||||
|
||||
def _get_redis_client(*, decode_responses: bool = False) -> redis.Redis:
|
||||
from app.core.config import settings
|
||||
|
||||
@@ -557,6 +583,29 @@ def process_memoir_phase2(
|
||||
return {"status": "noop"}
|
||||
|
||||
state = get_or_create_state_sync(user_id, db)
|
||||
segment_texts = [seg.user_input_text or "" for seg in category_segments]
|
||||
combined_text = "\n\n".join(segment_texts)
|
||||
n_units = len(category_segments)
|
||||
evidence_top_k = int(settings.evidence_top_k_default)
|
||||
if n_units > int(settings.evidence_large_batch_threshold):
|
||||
evidence_top_k = int(settings.evidence_top_k_large_batch)
|
||||
try:
|
||||
memory_evidence = asyncio.run(
|
||||
_memory_retrieve_evidence(
|
||||
user_id,
|
||||
combined_text,
|
||||
top_k=evidence_top_k,
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("Evidence 检索跳过: {}", e)
|
||||
memory_evidence = {
|
||||
"relevant_chunks": [],
|
||||
"relevant_summaries": [],
|
||||
"relevant_facts": [],
|
||||
"timeline_hints": [],
|
||||
"relevant_stories": [],
|
||||
}
|
||||
pipeline_t0 = time.perf_counter()
|
||||
chapter, needs_cover, disp = run_story_pipeline_for_category_batch(
|
||||
db,
|
||||
@@ -571,6 +620,7 @@ def process_memoir_phase2(
|
||||
occupation=user_occupation,
|
||||
memoir_correlation_id=cid,
|
||||
llm_fast=llm_fast,
|
||||
memory_evidence=memory_evidence,
|
||||
)
|
||||
pipeline_elapsed = time.perf_counter() - pipeline_t0
|
||||
story_dispatch_ids |= disp
|
||||
@@ -786,8 +836,12 @@ def process_memoir_phase1(self, user_id: str, segment_ids: List[str]):
|
||||
ingested_source_ids: list[str] = []
|
||||
if ingest_items:
|
||||
try:
|
||||
ingested_source_ids = ingest_transcripts_batch_sync(
|
||||
db, user_id, ingest_items
|
||||
ingested_source_ids = asyncio.run(
|
||||
_memory_ingest_transcripts_batch(
|
||||
user_id,
|
||||
ingest_items,
|
||||
memoir_correlation_id=memoir_correlation_id,
|
||||
)
|
||||
)
|
||||
for seg, sid in zip(
|
||||
non_empty_segments, ingested_source_ids, strict=True
|
||||
@@ -927,13 +981,6 @@ def process_memoir_phase1(self, user_id: str, segment_ids: List[str]):
|
||||
},
|
||||
)
|
||||
|
||||
if ingested_source_ids:
|
||||
schedule_enrichment_for_sources(
|
||||
user_id,
|
||||
ingested_source_ids,
|
||||
memoir_correlation_id=memoir_correlation_id,
|
||||
)
|
||||
|
||||
for cc in phase2_immediate:
|
||||
p2tid = _dispatch_phase2_immediate(user_id, cc, memoir_correlation_id)
|
||||
if p2tid:
|
||||
|
||||
Reference in New Issue
Block a user