kevin/life-echo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore/ 删除无用文件

Browse Source
This commit is contained in:
Kevin
2026-03-19 14:36:14 +08:00
parent 2f60858c9c
commit c6e07ce5ca
135 changed files with 2111 additions and 4510 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
api/app/tasks/__init__.py
View File

@@ -1,6 +1,7 @@
"""
Celery 任务模块
"""
from .celery_app import celery_app
from .memoir_tasks import process_memoir_segments, generate_chapter_images

1
api/app/tasks/celery_app.py
View File

@@ -3,6 +3,7 @@ Celery 应用配置
配置从 app.core.config.settings 读取。
Worker 启动时需聚合注册所有 feature 的 model，否则 User 等 relationship("Order", ...) 解析时会报找不到 Order。
"""
from celery import Celery
from app.core.config import settings

292
api/app/tasks/memoir_tasks.py
View File

@@ -1,6 +1,7 @@
"""
回忆录处理 Celery 任务
"""
import json
from app.core.logging import get_logger
import uuid
@@ -125,28 +126,30 @@ def _release_chapter_image_lock(chapter_id: str):
r.delete(lock_key)
def _update_task_status_sync(user_id: str, task_id: str, status: str, result: Dict = None):
def _update_task_status_sync(
user_id: str, task_id: str, status: str, result: Dict = None
):
"""同步更新任务状态（在 Celery 任务中使用）"""
try:
r = _get_redis_client(decode_responses=True)
key = f"task:user:{user_id}:tasks"
# 获取现有任务信息
data = r.hget(key, task_id)
if data:
task_info = json.loads(data)
else:
task_info = {"task_id": task_id}
task_info["status"] = status
task_info["updated_at"] = datetime.now(timezone.utc).isoformat()
if result is not None:
task_info["result"] = result
r.hset(key, task_id, json.dumps(task_info))
r.expire(key, 3600) # 1小时过期
logger.info(f"任务状态已更新: task_id={task_id}, status={status}")
except Exception as e:
logger.error(f"更新任务状态失败: {e}")
@@ -284,7 +287,16 @@ def _select_placeholders_for_effective_max(
return [{**item, "index": index} for index, item in enumerate(selected)]
def _save_narrative_to_sections(db: Session, chapter, narrative: str, title: str, category: str, order_index: int, source_segments: list, user_id: str):
def _save_narrative_to_sections(
db: Session,
chapter,
narrative: str,
title: str,
category: str,
order_index: int,
source_segments: list,
user_id: str,
):
"""
将带占位符的 narrative 拆成 chapter_sections 并写入；为每段占位符创建 pending 配图。
已有 section 与图片不删除，仅追加新内容。若无封面 MemoirImage 则创建 pending 封面（section_id=None）。
@@ -313,20 +325,25 @@ def _save_narrative_to_sections(db: Session, chapter, narrative: str, title: str
.where(ChapterSection.chapter_id == chapter.id)
.order_by(ChapterSection.order_index)
)
.scalars().all()
.scalars()
.all()
)
if existing_sections:
existing_content = "\n\n".join(
(s.content or "").strip() for s in existing_sections if (s.content or "").strip()
(s.content or "").strip()
for s in existing_sections
if (s.content or "").strip()
)
if existing_content and narrative.startswith(existing_content):
new_part = narrative[len(existing_content):].lstrip()
new_part = narrative[len(existing_content) :].lstrip()
else:
new_part = (narrative or "").strip()
if not new_part:
chapter.title = title
chapter.is_new = True
chapter.source_segments = list(set((chapter.source_segments or []) + (source_segments or [])))
chapter.source_segments = list(
set((chapter.source_segments or []) + (source_segments or []))
)
return chapter
narrative_to_parse = new_part
order_base = max(s.order_index for s in existing_sections) + 1
@@ -335,7 +352,11 @@ def _save_narrative_to_sections(db: Session, chapter, narrative: str, title: str
order_base = 0
img_settings = MemoirImageSettings.from_env()
prompt_service = MemoirImagePromptService(llm=None, settings=img_settings) if img_settings.enabled else None
prompt_service = (
MemoirImagePromptService(llm=None, settings=img_settings)
if img_settings.enabled
else None
)
segments = parse_narrative_to_sections(narrative_to_parse)
if not segments:
@@ -349,12 +370,9 @@ def _save_narrative_to_sections(db: Session, chapter, narrative: str, title: str
db.add(sec)
db.flush()
if img_settings.enabled:
stmt_cover = (
select(MemoirImage)
.where(
MemoirImage.chapter_id == chapter.id,
MemoirImage.section_id.is_(None),
)
stmt_cover = select(MemoirImage).where(
MemoirImage.chapter_id == chapter.id,
MemoirImage.section_id.is_(None),
)
if not db.execute(stmt_cover).scalar_one_or_none():
cover_ph = {
@@ -365,7 +383,11 @@ def _save_narrative_to_sections(db: Session, chapter, narrative: str, title: str
cover_asset = build_initial_image_assets(
[cover_ph],
img_settings.provider,
prompt_service.CATEGORY_STYLE_MAP.get(category, img_settings.default_style) if prompt_service else img_settings.default_style,
prompt_service.CATEGORY_STYLE_MAP.get(
category, img_settings.default_style
)
if prompt_service
else img_settings.default_style,
img_settings.default_size,
now_iso,
)[0]
@@ -374,7 +396,9 @@ def _save_narrative_to_sections(db: Session, chapter, narrative: str, title: str
db.flush()
chapter.title = title
chapter.is_new = True
chapter.source_segments = list(set((chapter.source_segments or []) + (source_segments or [])))
chapter.source_segments = list(
set((chapter.source_segments or []) + (source_segments or []))
)
return chapter
def _should_have_image(seg: dict, order_idx: int) -> bool:
@@ -393,7 +417,11 @@ def _save_narrative_to_sections(db: Session, chapter, narrative: str, title: str
return ph
content = (seg.get("content") or "").strip()
desc = (content[:50] + "…") if len(content) > 50 else (content or "章节配图")
return {"placeholder": f"{{{{{{{{IMAGE:{desc}}}}}}}}}", "description": desc, "index": order_idx}
return {
"placeholder": f"{{{{{{{{IMAGE:{desc}}}}}}}}}",
"description": desc,
"index": order_idx,
}
# 按顺序创建 section，每 3 个 section 对应 1 张配图
for i, seg in enumerate(segments):
@@ -402,7 +430,13 @@ def _save_narrative_to_sections(db: Session, chapter, narrative: str, title: str
image_asset = None
if img_settings.enabled and _should_have_image(seg, order_idx):
ph = _placeholder_for_segment(seg, order_idx)
style = prompt_service.CATEGORY_STYLE_MAP.get(category, img_settings.default_style) if prompt_service else img_settings.default_style
style = (
prompt_service.CATEGORY_STYLE_MAP.get(
category, img_settings.default_style
)
if prompt_service
else img_settings.default_style
)
image_asset = build_initial_image_assets(
[ph],
img_settings.provider,
@@ -422,7 +456,9 @@ def _save_narrative_to_sections(db: Session, chapter, narrative: str, title: str
db.flush()
if image_asset:
# 本段配图与当前 section 绑定，memoir_images.order_index = section.order_index + 1（封面 0 预留）
mi = _memoir_image_from_asset(chapter.id, sec.id, order_idx + 1, image_asset)
mi = _memoir_image_from_asset(
chapter.id, sec.id, order_idx + 1, image_asset
)
db.add(mi)
db.flush()
sec.image_id = mi.id
@@ -430,12 +466,9 @@ def _save_narrative_to_sections(db: Session, chapter, narrative: str, title: str
# 封面图：若无则创建 pending MemoirImage（section_id=None, order_index=0）
if img_settings.enabled:
stmt_cover = (
select(MemoirImage)
.where(
MemoirImage.chapter_id == chapter.id,
MemoirImage.section_id.is_(None),
)
stmt_cover = select(MemoirImage).where(
MemoirImage.chapter_id == chapter.id,
MemoirImage.section_id.is_(None),
)
existing_cover = db.execute(stmt_cover).scalar_one_or_none()
if not existing_cover:
@@ -447,7 +480,11 @@ def _save_narrative_to_sections(db: Session, chapter, narrative: str, title: str
cover_asset = build_initial_image_assets(
[cover_ph],
img_settings.provider,
prompt_service.CATEGORY_STYLE_MAP.get(category, img_settings.default_style) if prompt_service else img_settings.default_style,
prompt_service.CATEGORY_STYLE_MAP.get(
category, img_settings.default_style
)
if prompt_service
else img_settings.default_style,
img_settings.default_size,
now_iso,
)[0]
@@ -457,7 +494,9 @@ def _save_narrative_to_sections(db: Session, chapter, narrative: str, title: str
chapter.title = title
chapter.is_new = True
chapter.source_segments = list(set((chapter.source_segments or []) + (source_segments or [])))
chapter.source_segments = list(
set((chapter.source_segments or []) + (source_segments or []))
)
return chapter
@@ -465,7 +504,9 @@ def initialize_chapter_images(_chapter):
"""
兼容旧调用：若章节已改为 sections 存储，则图片初始化已在 _save_narrative_to_sections 中完成，直接返回。
"""
logger.info("initialize_chapter_images: 已由 _save_narrative_to_sections 处理 section 配图，跳过")
logger.info(
"initialize_chapter_images: 已由 _save_narrative_to_sections 处理 section 配图，跳过"
)
return []
@@ -489,7 +530,9 @@ def _coerce_state(model: MemoirState) -> MemoirStateSchema:
"stage_order": model.stage_order or default_state().stage_order,
"current_stage": model.current_stage,
"covered_stages": model.covered_stages or [],
"slots": model.slots if isinstance(model.slots, dict) else default_state().slots,
"slots": model.slots
if isinstance(model.slots, dict)
else default_state().slots,
}
)
@@ -509,7 +552,10 @@ def _get_or_create_state_sync(user_id: str, db: Session) -> MemoirStateSchema:
stage_order=default.stage_order,
current_stage=default.current_stage,
covered_stages=default.covered_stages,
slots={k: {sk: sv.model_dump() for sk, sv in v.items()} for k, v in default.slots.items()},
slots={
k: {sk: sv.model_dump() for sk, sv in v.items()}
for k, v in default.slots.items()
},
)
db.add(state)
db.commit()
@@ -539,7 +585,9 @@ def _update_slot_sync(
existing = stage_slots.get(slot_name, {})
merged_segment_ids = list({*(existing.get("segment_ids") or []), *segment_ids})
stage_slots[slot_name] = SlotData(snippet=snippet, segment_ids=merged_segment_ids).model_dump()
stage_slots[slot_name] = SlotData(
snippet=snippet, segment_ids=merged_segment_ids
).model_dump()
slots[stage] = stage_slots
state.slots = slots
state.current_stage = state.current_stage or stage
@@ -552,17 +600,19 @@ def _update_slot_sync(
def process_memoir_segments(self, user_id: str, segment_ids: List[str]):
"""
处理回忆录段落的 Celery 任务
Args:
user_id: 用户 ID
segment_ids: 段落 ID 列表
"""
task_id = self.request.id
logger.info(f"开始处理回忆录段落: user_id={user_id}, task_id={task_id}, segments={len(segment_ids)}")
logger.info(
f"开始处理回忆录段落: user_id={user_id}, task_id={task_id}, segments={len(segment_ids)}"
)
# 更新任务状态为 running
_update_task_status_sync(user_id, task_id, "running")
try:
with get_sync_db() as db:
chapters_to_enqueue: set[str] = set()
@@ -570,11 +620,11 @@ def process_memoir_segments(self, user_id: str, segment_ids: List[str]):
stmt = select(Segment).where(Segment.id.in_(segment_ids))
result = db.execute(stmt)
segments = result.scalars().all()
if not segments:
logger.warning(f"未找到段落: {segment_ids}")
return {"status": "no_segments"}
# 获取用户状态和资料
state = _get_or_create_state_sync(user_id, db)
llm = _get_llm()
@@ -615,7 +665,9 @@ def process_memoir_segments(self, user_id: str, segment_ids: List[str]):
Chapter.is_active == True,
)
.options(
joinedload(Chapter.sections).joinedload(ChapterSection.image_record),
joinedload(Chapter.sections).joinedload(
ChapterSection.image_record
),
joinedload(Chapter.images),
)
)
@@ -625,7 +677,9 @@ def process_memoir_segments(self, user_id: str, segment_ids: List[str]):
slot_snippets = {}
stage_slots = state.slots.get(chapter_category, {}) or {}
for key, value in stage_slots.items():
snip = getattr(value, "snippet", None) or (value.get("snippet") if isinstance(value, dict) else None)
snip = getattr(value, "snippet", None) or (
value.get("snippet") if isinstance(value, dict) else None
)
if snip:
slot_snippets[key] = snip
@@ -633,7 +687,9 @@ def process_memoir_segments(self, user_id: str, segment_ids: List[str]):
existing_content = ""
if chapter and getattr(chapter, "sections", None):
existing_content = "\n\n".join(
s.content for s in sorted(chapter.sections, key=lambda x: x.order_index) if (s.content or "").strip()
s.content
for s in sorted(chapter.sections, key=lambda x: x.order_index)
if (s.content or "").strip()
)
narrative = combined_text
@@ -662,7 +718,11 @@ def process_memoir_segments(self, user_id: str, segment_ids: List[str]):
else:
narrative = new_narrative
if existing_content and not _is_json_narrative(narrative) and len(narrative) < len(existing_content) * 0.8:
if (
existing_content
and not _is_json_narrative(narrative)
and len(narrative) < len(existing_content) * 0.8
):
logger.warning(
"内容长度异常: existing=%d, new=%d, category=%s. 回退为追加模式",
len(existing_content),
@@ -693,7 +753,11 @@ def process_memoir_segments(self, user_id: str, segment_ids: List[str]):
or _chapter_has_cover_to_generate(chapter)
)
stmt_book = select(Book).where(Book.user_id == user_id).order_by(Book.updated_at.desc())
stmt_book = (
select(Book)
.where(Book.user_id == user_id)
.order_by(Book.updated_at.desc())
)
result_book = db.execute(stmt_book)
book = result_book.scalar_one_or_none()
if not book:
@@ -721,19 +785,19 @@ def process_memoir_segments(self, user_id: str, segment_ids: List[str]):
user_profile=user_profile,
user_birth_year=user_birth_year,
get_or_create_state=lambda: _get_or_create_state_sync(user_id, db),
update_slot=lambda stage, slot_name, snippet, seg_ids: _update_slot_sync(
user_id, stage, slot_name, snippet, seg_ids, db
update_slot=lambda stage, slot_name, snippet, seg_ids: (
_update_slot_sync(user_id, stage, slot_name, snippet, seg_ids, db)
),
acquire_lock=lambda stage: _acquire_chapter_lock(user_id, stage),
release_lock=lambda stage: _release_chapter_lock(user_id, stage),
process_category=_process_category,
raise_retry=_raise_retry,
)
# 标记段落为已处理
for seg in segments:
seg.processed = True
db.commit()
for chapter_id in sorted(chapters_to_enqueue):
@@ -741,21 +805,25 @@ def process_memoir_segments(self, user_id: str, segment_ids: List[str]):
logger.info(f"派发章节补图任务: chapter={chapter_id}")
generate_chapter_images.delay(chapter_id)
except Exception as exc:
logger.warning(f"补图任务派发失败: chapter={chapter_id}, error={exc}")
logger.warning(
f"补图任务派发失败: chapter={chapter_id}, error={exc}"
)
logger.info(f"回忆录处理完成: user_id={user_id}, task_id={task_id}")
# 更新任务状态为成功
_update_task_status_sync(user_id, task_id, "success", {"processed": len(segments)})
_update_task_status_sync(
user_id, task_id, "success", {"processed": len(segments)}
)
return {"status": "success", "processed": len(segments)}
except Exception as e:
logger.error(f"回忆录处理失败: {e}")
# 更新任务状态为失败
_update_task_status_sync(user_id, task_id, "failure", {"error": str(e)})
# 重试
raise self.retry(exc=e)
@@ -764,18 +832,18 @@ def process_memoir_segments(self, user_id: str, segment_ids: List[str]):
def generate_chapter_content(self, user_id: str, stage: str, new_content: str):
"""
单独生成章节内容的任务（用于实时更新）
Args:
user_id: 用户 ID
stage: 阶段
new_content: 新内容
"""
logger.info(f"生成章节内容: user_id={user_id}, stage={stage}")
try:
with get_sync_db() as db:
llm = _get_llm()
# 查找 active 章节并预加载 sections
stmt = (
select(Chapter)
@@ -791,7 +859,9 @@ def generate_chapter_content(self, user_id: str, stage: str, new_content: str):
existing_content = ""
if chapter and getattr(chapter, "sections", None):
existing_content = "\n\n".join(
s.content for s in sorted(chapter.sections, key=lambda x: x.order_index) if (s.content or "").strip()
s.content
for s in sorted(chapter.sections, key=lambda x: x.order_index)
if (s.content or "").strip()
)
if llm:
@@ -814,10 +884,18 @@ def generate_chapter_content(self, user_id: str, stage: str, new_content: str):
else:
narrative = new_narrative
else:
narrative = f"{existing_content}\n\n{new_content}" if existing_content else new_content
narrative = (
f"{existing_content}\n\n{new_content}"
if existing_content
else new_content
)
# 安全检查：新内容不应比旧内容短（仅非 JSON 格式）
if existing_content and not _is_json_narrative(narrative) and len(narrative) < len(existing_content) * 0.8:
if (
existing_content
and not _is_json_narrative(narrative)
and len(narrative) < len(existing_content) * 0.8
):
logger.warning(
f"内容长度异常: existing={len(existing_content)}, "
f"new={len(narrative)}, stage={stage}. 回退为追加模式"
@@ -841,14 +919,20 @@ def generate_chapter_content(self, user_id: str, stage: str, new_content: str):
db.commit()
db.refresh(chapter)
image_settings = MemoirImageSettings.from_env()
if image_settings.enabled and chapter and (
_chapter_has_any_section_images_to_generate(chapter)
or _chapter_has_cover_to_generate(chapter)
if (
image_settings.enabled
and chapter
and (
_chapter_has_any_section_images_to_generate(chapter)
or _chapter_has_cover_to_generate(chapter)
)
):
try:
generate_chapter_images.delay(chapter.id)
except Exception as exc:
logger.warning("补图任务派发失败: chapter=%s, error=%s", chapter.id, exc)
logger.warning(
"补图任务派发失败: chapter=%s, error=%s", chapter.id, exc
)
return {"status": "success"}
except Exception as e:
@@ -873,7 +957,9 @@ def generate_chapter_images(self, chapter_id: str):
select(Chapter)
.where(Chapter.id == chapter_id)
.options(
joinedload(Chapter.sections).joinedload(ChapterSection.image_record),
joinedload(Chapter.sections).joinedload(
ChapterSection.image_record
),
joinedload(Chapter.images),
)
)
@@ -883,7 +969,9 @@ def generate_chapter_images(self, chapter_id: str):
return {"status": "no_chapter"}
sections = getattr(chapter, "sections", None) or []
sections_with_pending = [
(idx, s) for idx, s in enumerate(sections) if _section_has_image_to_generate(s)
(idx, s)
for idx, s in enumerate(sections)
if _section_has_image_to_generate(s)
]
cover_rec = _get_cover_memoir_image(chapter)
cover_to_generate = (
@@ -894,7 +982,9 @@ def generate_chapter_images(self, chapter_id: str):
else None
)
if not sections_with_pending and not cover_to_generate:
logger.info("章节补图跳过: chapter=%s, reason=no_pending_images", chapter_id)
logger.info(
"章节补图跳过: chapter=%s, reason=no_pending_images", chapter_id
)
return {"status": "no_images"}
settings = MemoirImageSettings.from_env()
@@ -943,8 +1033,14 @@ def generate_chapter_images(self, chapter_id: str):
_apply_item_to_memoir_image(cover_to_generate, current_item)
db.commit()
try:
sections_ordered = sorted(sections, key=lambda s: getattr(s, "order_index", 0))
first_content = (sections_ordered[0].content or "").strip() if sections_ordered else ""
sections_ordered = sorted(
sections, key=lambda s: getattr(s, "order_index", 0)
)
first_content = (
(sections_ordered[0].content or "").strip()
if sections_ordered
else ""
)
context_excerpt = " ".join(first_content.split("\n")[:5])[:200]
prompt_data = prompt_orchestrator.build_cover_prompt(
chapter_title=chapter.title,
@@ -961,9 +1057,13 @@ def generate_chapter_images(self, chapter_id: str):
image_bytes = _normalize_image_bytes_for_storage(
image_generator.download_image(result.image_url)
)
key = build_cos_key(chapter.user_id, chapter.id, "cover", prompt_data["prompt"])
key = build_cos_key(
chapter.user_id, chapter.id, "cover", prompt_data["prompt"]
)
current_item["storage_key"] = key
current_item["url"] = storage.upload_bytes(image_bytes, key, "image/png")
current_item["url"] = storage.upload_bytes(
image_bytes, key, "image/png"
)
current_item["prompt"] = prompt_data["prompt"]
current_item["style"] = prompt_data["style"]
current_item["size"] = prompt_data["size"]
@@ -982,7 +1082,11 @@ def generate_chapter_images(self, chapter_id: str):
failure_msg = f"cover, error={exc}"
if isinstance(exc, CosUploadError) and not exc.retryable:
permanent_failures.append(failure_msg)
logger.error("封面图上传不可重试，清理: chapter=%s, %s", chapter_id, failure_msg)
logger.error(
"封面图上传不可重试，清理: chapter=%s, %s",
chapter_id,
failure_msg,
)
db.delete(cover_to_generate)
db.commit()
else:
@@ -990,14 +1094,24 @@ def generate_chapter_images(self, chapter_id: str):
current_item["status"] = IMAGE_STATUS_FAILED
current_item["error"] = str(exc)
current_item["retryable"] = True
current_item["updated_at"] = datetime.now(timezone.utc).isoformat()
current_item["updated_at"] = datetime.now(
timezone.utc
).isoformat()
retryable_failures.append(failure_msg)
logger.warning("封面图生成失败(可重试): chapter=%s, %s", chapter_id, failure_msg)
logger.warning(
"封面图生成失败(可重试): chapter=%s, %s",
chapter_id,
failure_msg,
)
_apply_item_to_memoir_image(cover_to_generate, current_item)
db.commit()
for sec_index, section in sections_with_pending:
item = memoir_image_to_dict(section.image_record) if section.image_record else {}
item = (
memoir_image_to_dict(section.image_record)
if section.image_record
else {}
)
current_item = dict(item) if item else {}
current_item.setdefault("placeholder", "")
current_item.setdefault("description", "")
@@ -1025,9 +1139,13 @@ def generate_chapter_images(self, chapter_id: str):
image_bytes = _normalize_image_bytes_for_storage(
image_generator.download_image(result.image_url)
)
key = build_cos_key(chapter.user_id, chapter.id, sec_index, prompt_data["prompt"])
key = build_cos_key(
chapter.user_id, chapter.id, sec_index, prompt_data["prompt"]
)
current_item["storage_key"] = key
current_item["url"] = storage.upload_bytes(image_bytes, key, "image/png")
current_item["url"] = storage.upload_bytes(
image_bytes, key, "image/png"
)
current_item["prompt"] = prompt_data["prompt"]
current_item["style"] = prompt_data["style"]
current_item["size"] = prompt_data["size"]
@@ -1047,7 +1165,11 @@ def generate_chapter_images(self, chapter_id: str):
failure_msg = f"section_index={sec_index}, error={exc}"
if isinstance(exc, CosUploadError) and not exc.retryable:
permanent_failures.append(failure_msg)
logger.error("图片上传不可重试，清理配图: chapter=%s, %s", chapter_id, failure_msg)
logger.error(
"图片上传不可重试，清理配图: chapter=%s, %s",
chapter_id,
failure_msg,
)
mi = section.image_record
section.image_id = None
if mi:
@@ -1058,8 +1180,14 @@ def generate_chapter_images(self, chapter_id: str):
current_item["error"] = str(exc)
current_item["retryable"] = True
retryable_failures.append(failure_msg)
logger.warning("图片生成失败(可重试): chapter=%s, %s", chapter_id, failure_msg)
current_item["updated_at"] = datetime.now(timezone.utc).isoformat()
logger.warning(
"图片生成失败(可重试): chapter=%s, %s",
chapter_id,
failure_msg,
)
current_item["updated_at"] = datetime.now(
timezone.utc
).isoformat()
_apply_item_to_memoir_image(section.image_record, current_item)
db.commit()