ac49bc7f23
- Judge baseline excerpt and library chapter separately; build_memoir_compare_summary for gate, nine-dim and leaf deltas. - Memoir SSE chapter payload: baseline_judge, compare_summary, baseline_judge_error. - MemoirJudgeOutput: loose score coercion and post-validate clamp; memoir judge prompt caps from settings. - app-eval-web: two-column MemoirScoreCard layout, MemoirCompareSummary, chapter blocks and CSS. - Add memoir_compare_summary, log_events, celery_log_context, memoir_pipeline_progress; tests and migration 0014. - Misc: memory/evidence and enrichment paths, task/orchestrator updates, internal-eval docs, env examples.
122 lines
4.1 KiB
Python
122 lines
4.1 KiB
Python
"""Celery：memory compaction（近重复 chunk 软排除）。"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import time
|
||
from datetime import datetime
|
||
from typing import Any
|
||
|
||
from celery import shared_task
|
||
|
||
from app.core.config import settings
|
||
from app.core.db import get_sync_db
|
||
from app.core.logging import get_logger
|
||
from app.core.memory_compaction_schedule import (
|
||
finalize_memory_compaction_run,
|
||
read_debounce_deadline_ts,
|
||
release_scheduler_gate,
|
||
schedule_memory_compaction_run,
|
||
set_incremental_cursor_pair,
|
||
)
|
||
from app.core.redis_lock import acquire_redis_lock, release_redis_lock
|
||
from app.features.memory.compaction_service import run_memory_compaction_sync
|
||
from app.features.memory.repo import list_users_with_recent_chunks_sync
|
||
|
||
logger = get_logger(__name__)
|
||
|
||
|
||
@shared_task
|
||
def memory_compaction_sweep() -> dict[str, Any]:
|
||
"""Beat：为近期有记忆写入的用户调度 compaction（debounce 仍由 schedule 合并）。"""
|
||
t0 = time.perf_counter()
|
||
if not settings.memory_compaction_enabled:
|
||
return {"skipped": True, "reason": "disabled"}
|
||
hours = int(settings.memory_compaction_sweep_recent_hours)
|
||
with get_sync_db() as session:
|
||
user_ids = list_users_with_recent_chunks_sync(session, hours=hours)
|
||
ctx_base: dict[str, Any] = {"trigger_source": "beat", "sweep_hours": hours}
|
||
for uid in user_ids:
|
||
schedule_memory_compaction_run(uid, dict(ctx_base))
|
||
ms = (time.perf_counter() - t0) * 1000
|
||
logger.info(
|
||
"event=memory_compaction_sweep_done hours={} scheduled_users={} duration_ms={:.1f} "
|
||
"msg=记忆压缩定时扫描已调度",
|
||
hours,
|
||
len(user_ids),
|
||
ms,
|
||
)
|
||
return {"scheduled": len(user_ids), "user_ids": user_ids}
|
||
|
||
|
||
@shared_task(bind=True, max_retries=12, default_retry_delay=20)
|
||
def memory_compaction_run(
|
||
self, user_id: str, context: dict[str, Any] | None = None
|
||
) -> dict[str, Any]:
|
||
run_t0 = time.perf_counter()
|
||
if not settings.memory_compaction_enabled:
|
||
return {"skipped": True, "reason": "disabled"}
|
||
|
||
ctx = dict(context or {})
|
||
deadline = read_debounce_deadline_ts(user_id)
|
||
now = time.time()
|
||
if deadline is not None and now < deadline:
|
||
delay = max(1.0, deadline - now)
|
||
raise self.retry(countdown=int(delay))
|
||
|
||
lock = acquire_redis_lock(
|
||
f"lock:memory_compaction:{user_id}",
|
||
ttl_seconds=settings.memory_compaction_lock_ttl_seconds,
|
||
)
|
||
if lock is None:
|
||
ms = (time.perf_counter() - run_t0) * 1000
|
||
logger.info(
|
||
"event=memory_compaction_skipped user_id={} reason=lock_not_acquired "
|
||
"duration_ms={:.1f} msg=记忆压缩跳过（未拿到锁）",
|
||
user_id,
|
||
ms,
|
||
)
|
||
out = {"skipped": True, "reason": "lock_not_acquired"}
|
||
finalize_memory_compaction_run(
|
||
user_id,
|
||
observed_deadline_ts=deadline,
|
||
context=ctx,
|
||
)
|
||
return out
|
||
|
||
try:
|
||
with get_sync_db() as session:
|
||
out = run_memory_compaction_sync(session, user_id, ctx)
|
||
session.commit()
|
||
|
||
if out.get("new_cursor_ts") and out.get("new_cursor_id") is not None:
|
||
set_incremental_cursor_pair(
|
||
user_id,
|
||
datetime.fromisoformat(out["new_cursor_ts"]),
|
||
str(out["new_cursor_id"]),
|
||
)
|
||
finalize_memory_compaction_run(
|
||
user_id,
|
||
observed_deadline_ts=deadline,
|
||
context=ctx,
|
||
)
|
||
ms = (time.perf_counter() - run_t0) * 1000
|
||
logger.info(
|
||
"event=memory_compaction_done user_id={} duration_ms={:.1f} msg=记忆压缩运行完成",
|
||
user_id,
|
||
ms,
|
||
)
|
||
return out
|
||
except Exception as exc:
|
||
ms = (time.perf_counter() - run_t0) * 1000
|
||
logger.warning(
|
||
"event=memory_compaction_failed user_id={} duration_ms={:.1f} err={} "
|
||
"msg=记忆压缩运行失败",
|
||
user_id,
|
||
ms,
|
||
exc,
|
||
)
|
||
release_scheduler_gate(user_id)
|
||
raise self.retry(exc=exc) from exc
|
||
finally:
|
||
release_redis_lock(lock)
|