kevin/life-echo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: OpenTelemetry LGTM observability, dev tooling, and memoir UX fixes (#31) (#32)

Browse Source 
* add staging ios app build script

* feat(api): add OpenTelemetry LGTM stack for local observability

Wire OTel traces, metrics, and logs through a collector to Tempo,
Prometheus, and Loki, with custom LLM instrumentation, dev compose overlay,
Grafana provisioning, env templates, and development.sh auto-start.



* feat: expand observability, harden dev tooling, and fix expo staging UX

Add business and LLM Prometheus metrics with Grafana dashboards, alerting,
and a metrics verification script. Wire telemetry through adapters and core
LLM paths, and document the local LGTM workflow.

Fix development.sh for macOS bash 3.2, open Grafana and eval-web in Chrome,
and repair eval-web auto-open (unbound EVAL_WEB_BROWSER_SCHEDULED). Merge
internal-eval into the main dev script with improved compose handling.

Require EXPO_PUBLIC_* at build time, improve iOS HTTP ATS for staging IPs,
show memoir empty state instead of load errors when no chapters exist, and
add jest env setup plus chapter list response normalization.



* chore: enable Grafana Assistant Cursor plugin



* fix: memoir empty state and repair withdrawn 0020_chapters_book_id stamp

Show empty memoir UI when the chapter list succeeds with no items; treat auth/404 as non-fatal. Extend alembic revision repair so local dev DBs stamped with the removed 0020_chapters_book_id migration can roll back and upgrade to 0019.



---------

Co-authored-by: Kevin <kevin@brighteng.org>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Sully
2026-05-20 15:14:13 +08:00
committed by GitHub
parent 81458c7046
commit f09ae248f9
74 changed files with 3793 additions and 375 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
api/app/tasks/celery_app.py
View File

@@ -14,11 +14,17 @@ from app.core.logging import get_logger, setup_logging
# 与 app.main 一致：先配置 loguru + InterceptHandler，再加载会打日志的依赖
setup_logging()
from app.core.config import settings
from app.core.telemetry import instrument_celery, setup_telemetry
# Worker 与 API 共用 .env，固定 service.name，勿读 OTEL_SERVICE_NAME（留给主站 / internal）
setup_telemetry(service_name="life-echo-celery-worker")
instrument_celery()
from celery import Celery
from celery.signals import task_failure, task_postrun, task_prerun, task_success
from app.core.celery_log_context import clear_celery_log_extras, set_celery_log_extras
from app.core.config import settings
from app.core.log_events import celery_prerun_extras
from app.features.asset import models as _asset_models # noqa: F401 - register Asset
from app.features.auth import models as _auth_models # noqa: F401
@@ -123,9 +129,12 @@ def _log_task_prerun(
**_: object,
) -> None:
name = getattr(task, "name", None) or "?"
from app.core.telemetry import current_trace_context
extras = celery_prerun_extras(name, tuple(args or ()), dict(kwargs or {}))
if task_id:
extras["task_id"] = str(task_id).strip()
extras.update(current_trace_context())
set_celery_log_extras(extras if extras else None)
_celery_lifecycle_log.info(
"event=celery_task_start task={} task_id={} msg=Celery 任务已开始",

179
api/app/tasks/memoir_tasks.py
View File

@@ -26,6 +26,7 @@ from app.core.chapter_pipeline_lock import (
from app.core.chapter_pipeline_lock import (
release_chapter_pipeline_lock as _release_chapter_lock,
)
from app.core.business_telemetry import business_span
from app.core.config import settings
from app.core.db import AsyncSessionLocal, get_sync_db
from app.core.dependencies import get_embedding_provider
@@ -614,7 +615,10 @@ def process_memoir_phase2(
},
)
try:
with get_sync_db() as db:
with business_span(
"memoir.phase2",
chapter_category=chapter_category,
), get_sync_db() as db:
user_convs = select(Conversation.id).where(
Conversation.user_id == user_id,
Conversation.deleted_at.is_(None),
@@ -691,9 +695,13 @@ def process_memoir_phase2(
affected_chapter_ids: Set[str] = set()
lock_t0 = time.perf_counter()
lock_handle = _acquire_chapter_lock(
user_id, chapter_category, ttl_seconds=_chapter_lock_ttl()
)
with business_span(
"memoir.phase2.lock",
chapter_category=chapter_category,
):
lock_handle = _acquire_chapter_lock(
user_id, chapter_category, ttl_seconds=_chapter_lock_ttl()
)
lock_elapsed = time.perf_counter() - lock_t0
if lock_handle is None:
logger.warning(
@@ -746,22 +754,26 @@ def process_memoir_phase2(
"relevant_stories": [],
}
pipeline_t0 = time.perf_counter()
pipeline_result = run_story_pipeline_for_category_batch(
db,
user_id=user_id,
with business_span(
"memoir.phase2.story_pipeline",
chapter_category=chapter_category,
category_segments=category_segments,
state=state,
user_profile=user_profile,
user_birth_year=user_birth_year,
llm=llm,
background_voice=background_voice,
occupation=user_occupation,
memoir_correlation_id=cid,
llm_fast=llm_fast,
memory_evidence=memory_evidence,
language=user_language,
)
):
pipeline_result = run_story_pipeline_for_category_batch(
db,
user_id=user_id,
chapter_category=chapter_category,
category_segments=category_segments,
state=state,
user_profile=user_profile,
user_birth_year=user_birth_year,
llm=llm,
background_voice=background_voice,
occupation=user_occupation,
memoir_correlation_id=cid,
llm_fast=llm_fast,
memory_evidence=memory_evidence,
language=user_language,
)
pipeline_elapsed = time.perf_counter() - pipeline_t0
if pipeline_result.deferred:
@@ -939,7 +951,10 @@ def process_memoir_phase1(self, user_id: str, segment_ids: List[str]):
phase1_t0 = time.perf_counter()
try:
with get_sync_db() as db:
with business_span(
"memoir.phase1",
segment_count=len(segment_ids),
), get_sync_db() as db:
user_obj_for_lang = db.get(User, user_id)
user_language = (
"en"
@@ -986,47 +1001,48 @@ def process_memoir_phase1(self, user_id: str, segment_ids: List[str]):
},
)
ingest_t0 = time.perf_counter()
ingest_items: list[tuple[str, str, dict | None]] = []
non_empty_segments: list = []
for seg in segments:
text = (seg.user_input_text or "").strip()
if not text:
continue
conv_id = getattr(seg, "conversation_id", None) or ""
ln = getattr(seg, "lineage_json", None)
lineage_payload = ln if isinstance(ln, dict) else None
ingest_items.append((conv_id, text, lineage_payload))
non_empty_segments.append(seg)
with business_span("memoir.phase1.ingest"):
ingest_items: list[tuple[str, str, dict | None]] = []
non_empty_segments: list = []
for seg in segments:
text = (seg.user_input_text or "").strip()
if not text:
continue
conv_id = getattr(seg, "conversation_id", None) or ""
ln = getattr(seg, "lineage_json", None)
lineage_payload = ln if isinstance(ln, dict) else None
ingest_items.append((conv_id, text, lineage_payload))
non_empty_segments.append(seg)
ingested_source_ids: list[str] = []
if ingest_items:
try:
ingested_source_ids = asyncio.run(
_memory_ingest_transcripts_batch(
user_id,
ingest_items,
memoir_correlation_id=memoir_correlation_id,
ingested_source_ids: list[str] = []
if ingest_items:
try:
ingested_source_ids = asyncio.run(
_memory_ingest_transcripts_batch(
user_id,
ingest_items,
memoir_correlation_id=memoir_correlation_id,
)
)
)
for seg, sid in zip(
non_empty_segments, ingested_source_ids, strict=True
):
logger.info(
"event=memory_transcript_ingested user_id={} task_id={} "
"source_id={} conversation_id={} segment_id={} transcript_chars={}",
user_id,
task_id,
sid,
getattr(seg, "conversation_id", None) or "",
seg.id,
len((seg.user_input_text or "").strip()),
for seg, sid in zip(
non_empty_segments, ingested_source_ids, strict=True
):
logger.info(
"event=memory_transcript_ingested user_id={} task_id={} "
"source_id={} conversation_id={} segment_id={} transcript_chars={}",
user_id,
task_id,
sid,
getattr(seg, "conversation_id", None) or "",
seg.id,
len((seg.user_input_text or "").strip()),
)
except Exception as e:
logger.warning(
"Memory batch ingest 失败: {} exc_type={}",
e,
type(e).__name__,
)
except Exception as e:
logger.warning(
"Memory batch ingest 失败: {} exc_type={}",
e,
type(e).__name__,
)
ingest_elapsed = time.perf_counter() - ingest_t0
merge_pipeline_run(
memoir_correlation_id,
@@ -1050,31 +1066,32 @@ def process_memoir_phase1(self, user_id: str, segment_ids: List[str]):
)
prep_t0 = time.perf_counter()
memoir_orchestrator = MemoirOrchestrator()
with business_span("memoir.phase1.prepare_batches"):
memoir_orchestrator = MemoirOrchestrator()
def _phase1_chunk_cb(idx: int, total: int) -> None:
merge_pipeline_run(
memoir_correlation_id,
{"phase1": {"detail": {"prepare_batches_chunk": [idx, total]}}},
def _phase1_chunk_cb(idx: int, total: int) -> None:
merge_pipeline_run(
memoir_correlation_id,
{"phase1": {"detail": {"prepare_batches_chunk": [idx, total]}}},
)
prepared = memoir_orchestrator.prepare_batches(
segments=list(segments),
llm=llm,
llm_fast=llm_fast,
get_or_create_state=lambda: get_or_create_state_sync(user_id, db),
update_slot=lambda stage, slot_name, snippet, seg_ids: update_slot_sync(
user_id,
stage,
slot_name,
snippet,
seg_ids,
db,
memoir_batch=True,
),
on_phase1_chunk=_phase1_chunk_cb,
language=user_language,
)
prepared = memoir_orchestrator.prepare_batches(
segments=list(segments),
llm=llm,
llm_fast=llm_fast,
get_or_create_state=lambda: get_or_create_state_sync(user_id, db),
update_slot=lambda stage, slot_name, snippet, seg_ids: update_slot_sync(
user_id,
stage,
slot_name,
snippet,
seg_ids,
db,
memoir_batch=True,
),
on_phase1_chunk=_phase1_chunk_cb,
language=user_language,
)
prep_elapsed = time.perf_counter() - prep_t0
merge_pipeline_run(
memoir_correlation_id,

7
api/app/tasks/memory_compaction_tasks.py
View File

@@ -9,6 +9,7 @@ from typing import Any
from celery import shared_task
from app.core.business_telemetry import business_span
from app.core.config import settings
from app.core.db import AsyncSessionLocal
from app.core.logging import get_logger
@@ -49,7 +50,8 @@ def memory_compaction_sweep() -> dict[str, Any]:
if not settings.memory_compaction_enabled:
return {"skipped": True, "reason": "disabled"}
hours = int(settings.memory_compaction_sweep_recent_hours)
user_ids = asyncio.run(_list_users_with_recent_chunks_async(hours))
with business_span("memory.compaction.sweep", hours=hours):
user_ids = asyncio.run(_list_users_with_recent_chunks_async(hours))
ctx_base: dict[str, Any] = {"trigger_source": "beat", "sweep_hours": hours}
for uid in user_ids:
schedule_memory_compaction_run(uid, dict(ctx_base))
@@ -100,7 +102,8 @@ def memory_compaction_run(
return out
try:
out = asyncio.run(_run_memory_compaction_async(user_id, ctx))
with business_span("memory.compaction.run"):
out = asyncio.run(_run_memory_compaction_async(user_id, ctx))
if out.get("new_cursor_ts") and out.get("new_cursor_id") is not None:
set_incremental_cursor_pair(

7
api/app/tasks/memory_enrichment_tasks.py
View File

@@ -11,6 +11,7 @@ from typing import Any, cast
from celery import shared_task
from app.core.business_telemetry import business_span
from app.core.config import settings
from app.core.db import AsyncSessionLocal
from app.core.dependencies import get_embedding_provider
@@ -166,7 +167,8 @@ def embed_memory_source(
status="running",
)
try:
result = asyncio.run(_embed_memory_source_async(user_id, source_id))
with business_span("memory.embed_source"):
result = asyncio.run(_embed_memory_source_async(user_id, source_id))
ms = (time.perf_counter() - t0) * 1000
logger.info(
"event=memory_embedding_done user_id={} source_id={} duration_ms={:.1f} status={} vectors_written={} msg=记忆向量化完成",
@@ -241,7 +243,8 @@ def enrich_memory_source(
status="running",
)
try:
asyncio.run(_enrich_memory_source_async(user_id, source_id))
with business_span("memory.enrich_source"):
asyncio.run(_enrich_memory_source_async(user_id, source_id))
ms = (time.perf_counter() - t0) * 1000
logger.info(
"event=memory_enrichment_done user_id={} source_id={} duration_ms={:.1f} "