e884409410
Memory compaction（近重复 chunk 软排除） - 新增 compaction 调度：Redis debounce、scheduler gate、增量游标；任务结束时 finalize，避免 gate 长期占用并处理运行期新 trigger。 - Celery memory_compaction_run：debounce 未到点则 retry；用户级 Redis 锁；成功路径更新游标并 finalize；异常时释放 scheduler gate 并 self.retry，避免静默卡死调度与瞬时失败不重试。 - compaction_service：多层判定 + canonical 打分；无 embedding 时停止前移游标（awaiting_embeddings）；curation details 补全 trigger 等上下文。 - ingest_transcript_sync：同步路径尽力写入 embedding，与异步 ingest 行为对齐，避免 compaction 永远扫不到无向量 chunk。 - repo：新增 update_chunk_embedding_sync。 测试 - 扩展 test_memory_compaction：调度合并、finalize、ingest embedding、无向量游标、异常路径 gate+retry 等回归用
74 lines
2.5 KiB
Python
74 lines
2.5 KiB
Python
"""
|
||
Celery 应用配置
|
||
配置从 app.core.config.settings 读取。
|
||
Worker 启动时需聚合注册所有 feature 的 model，否则 User 等 relationship("Order", ...) 解析时会报找不到 Order。
|
||
与 main.py / Alembic 一致：下方 import 仅用于注册 ORM model。
|
||
"""
|
||
|
||
from app.core.logging import setup_logging
|
||
|
||
# 与 app.main 一致：先配置 loguru + InterceptHandler，再加载会打日志的依赖
|
||
setup_logging()
|
||
|
||
from celery import Celery
|
||
|
||
from app.core.config import settings
|
||
from app.features.asset import models as _asset_models # noqa: F401 - register Asset
|
||
from app.features.auth import models as _auth_models # noqa: F401
|
||
from app.features.conversation import models as _conv_models # noqa: F401
|
||
from app.features.memoir import models as _memoir_models # noqa: F401
|
||
from app.features.memory import models as _memory_models # noqa: F401
|
||
from app.features.payment import models as _payment_models # noqa: F401
|
||
from app.features.story import models as _story_models # noqa: F401
|
||
from app.features.user import models as _user_models # noqa: F401
|
||
|
||
REDIS_URL = settings.redis_url
|
||
|
||
# 创建 Celery 应用
|
||
celery_app = Celery(
|
||
"life_echo",
|
||
broker=REDIS_URL,
|
||
backend=REDIS_URL,
|
||
include=[
|
||
"app.tasks.memoir_tasks",
|
||
"app.tasks.story_image_tasks",
|
||
"app.tasks.chapter_cover_tasks",
|
||
"app.tasks.chapter_compose_tasks",
|
||
"app.tasks.memory_compaction_tasks",
|
||
],
|
||
)
|
||
|
||
# Celery 配置
|
||
celery_app.conf.update(
|
||
# 不劫持根 logger，便于与 loguru + InterceptHandler 统一格式与等级
|
||
worker_hijack_root_logger=False,
|
||
# 任务序列化
|
||
task_serializer="json",
|
||
accept_content=["json"],
|
||
result_serializer="json",
|
||
# 时区
|
||
timezone="UTC",
|
||
enable_utc=True,
|
||
# 任务结果过期时间（1小时）
|
||
result_expires=3600,
|
||
# 任务执行设置
|
||
task_soft_time_limit=300, # 5分钟软超时
|
||
task_time_limit=600, # 10分钟硬超时
|
||
# 并发设置
|
||
worker_prefetch_multiplier=1, # 每次只预取一个任务
|
||
worker_concurrency=4, # 并发 worker 数量
|
||
# 任务重试设置
|
||
task_acks_late=True, # 任务完成后再确认
|
||
task_reject_on_worker_lost=True, # worker 丢失时拒绝任务
|
||
# 不设置自定义队列路由，使用 Celery 默认队列
|
||
)
|
||
|
||
# 定时任务配置（如果需要）
|
||
celery_app.conf.beat_schedule = {
|
||
# 示例：每小时清理过期会话
|
||
# "cleanup-expired-sessions": {
|
||
# "task": "app.tasks.cleanup.cleanup_sessions",
|
||
# "schedule": 3600.0,
|
||
# },
|
||
}
|