kevin/life-echo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: OpenTelemetry LGTM observability, dev tooling, and memoir UX fixes (#31) (#32)

Browse Source 
* add staging ios app build script

* feat(api): add OpenTelemetry LGTM stack for local observability

Wire OTel traces, metrics, and logs through a collector to Tempo,
Prometheus, and Loki, with custom LLM instrumentation, dev compose overlay,
Grafana provisioning, env templates, and development.sh auto-start.



* feat: expand observability, harden dev tooling, and fix expo staging UX

Add business and LLM Prometheus metrics with Grafana dashboards, alerting,
and a metrics verification script. Wire telemetry through adapters and core
LLM paths, and document the local LGTM workflow.

Fix development.sh for macOS bash 3.2, open Grafana and eval-web in Chrome,
and repair eval-web auto-open (unbound EVAL_WEB_BROWSER_SCHEDULED). Merge
internal-eval into the main dev script with improved compose handling.

Require EXPO_PUBLIC_* at build time, improve iOS HTTP ATS for staging IPs,
show memoir empty state instead of load errors when no chapters exist, and
add jest env setup plus chapter list response normalization.



* chore: enable Grafana Assistant Cursor plugin



* fix: memoir empty state and repair withdrawn 0020_chapters_book_id stamp

Show empty memoir UI when the chapter list succeeds with no items; treat auth/404 as non-fatal. Extend alembic revision repair so local dev DBs stamped with the removed 0020_chapters_book_id migration can roll back and upgrade to 0019.



---------

Co-authored-by: Kevin <kevin@brighteng.org>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Sully
2026-05-20 15:14:13 +08:00
committed by GitHub
parent 81458c7046
commit f09ae248f9
74 changed files with 3793 additions and 375 deletions
Download Patch File Download Diff File Expand all files Collapse all files

194
api/app/core/llm_call.py
View File

@@ -30,6 +30,12 @@ from app.core.langchain_llm import (
)
from app.core.llm_errors import LlmHttpErrorVendor, format_llm_http_error_message
from app.core.llm_http_openai_chat_errors import should_log_openai_error_as_warning
from app.core.llm_telemetry import (
extract_token_usage,
infer_provider_model,
llm_call_span,
record_llm_call,
)
from app.core.logging import get_logger
logger = get_logger(__name__)
@@ -138,14 +144,16 @@ def _invoke_raw_sync(
max_tokens: int,
agent: str,
retry_empty: bool,
) -> tuple[str, int]:
) -> tuple[str, int, int, int]:
prompt_for_api = ensure_json_object_prompt_has_json_keyword(prompt)
bound = bind_json_object_mode(llm, max_tokens=max_tokens)
tag = agent or "json_object"
sha = _prompt_sha12(prompt_for_api)
attempts = 2 if retry_empty else 1
last_in, last_out = 0, 0
for attempt in range(attempts):
response = bound.invoke(prompt_for_api)
last_in, last_out = extract_token_usage(response)
content = (getattr(response, "content", None) or "").strip()
if content:
if attempt > 0:
@@ -154,7 +162,7 @@ def _invoke_raw_sync(
tag,
sha,
)
return content, attempt + 1
return content, attempt + 1, last_in, last_out
if attempt == 0 and retry_empty:
logger.warning(
"json_object 返回空 content，将重试 agent={} attempt={} prompt_sha12={}",
@@ -163,7 +171,7 @@ def _invoke_raw_sync(
sha,
)
logger.warning("json_object 仍为空 agent={} prompt_sha12={}", tag, sha)
return "", attempts
return "", attempts, last_in, last_out
async def _invoke_raw_async(
@@ -173,14 +181,16 @@ async def _invoke_raw_async(
max_tokens: int,
agent: str,
retry_empty: bool,
) -> tuple[str, int]:
) -> tuple[str, int, int, int]:
prompt_for_api = ensure_json_object_prompt_has_json_keyword(prompt)
bound = bind_json_object_mode(llm, max_tokens=max_tokens)
tag = agent or "json_object"
sha = _prompt_sha12(prompt_for_api)
attempts = 2 if retry_empty else 1
last_in, last_out = 0, 0
for attempt in range(attempts):
response = await bound.ainvoke(prompt_for_api)
last_in, last_out = extract_token_usage(response)
content = (getattr(response, "content", None) or "").strip()
if content:
if attempt > 0:
@@ -189,7 +199,7 @@ async def _invoke_raw_async(
tag,
sha,
)
return content, attempt + 1
return content, attempt + 1, last_in, last_out
if attempt == 0 and retry_empty:
logger.warning(
"json_object 返回空 content，将重试 agent={} attempt={} prompt_sha12={}",
@@ -198,7 +208,7 @@ async def _invoke_raw_async(
sha,
)
logger.warning("json_object 仍为空 agent={} prompt_sha12={}", tag, sha)
return "", attempts
return "", attempts, last_in, last_out
def _parse_and_validate(
@@ -252,6 +262,12 @@ def _emit_meta(
parse_ok: bool,
used_fallback: bool,
error_kind: str | None,
provider: str,
model: str,
prompt_sha12: str,
input_tokens: int = 0,
output_tokens: int = 0,
span: Any | None = None,
) -> None:
meta = LLMCallMeta(
agent=agent,
@@ -263,17 +279,35 @@ def _emit_meta(
used_fallback=used_fallback,
error_kind=error_kind,
)
logger.bind(
event="llm_json_call",
bind = {
"event": "llm_json_call",
"agent": meta.agent,
"schema": meta.schema_name,
"max_tokens": meta.max_tokens,
"duration_ms": round(meta.duration_ms, 2),
"attempts": meta.attempts,
"parse_ok": meta.parse_ok,
"used_fallback": meta.used_fallback,
"error_kind": meta.error_kind,
"provider": provider,
"prompt_sha12": prompt_sha12,
}
logger.bind(**bind).info("llm_json_call_done")
record_llm_call(
agent=meta.agent,
schema=meta.schema_name,
max_tokens=meta.max_tokens,
duration_ms=round(meta.duration_ms, 2),
schema_name=meta.schema_name,
provider=provider,
model=model,
duration_ms=meta.duration_ms,
attempts=meta.attempts,
parse_ok=meta.parse_ok,
used_fallback=meta.used_fallback,
error_kind=meta.error_kind,
).info("llm_json_call_done")
prompt_sha12=prompt_sha12,
input_tokens=input_tokens,
output_tokens=output_tokens,
span=span,
)
def llm_json_call(
@@ -288,13 +322,59 @@ def llm_json_call(
http_error_vendor: LlmHttpErrorVendor = "deepseek",
) -> T:
"""同步：invoke → 解析 JSON → `schema.model_validate`；失败时 `fallback_factory` 或 `LLMCallError`。"""
t0 = time.perf_counter()
schema_name = getattr(schema, "__name__", str(schema))
provider, model = infer_provider_model(llm, http_error_vendor=http_error_vendor)
prompt_sha12 = _prompt_sha12(prompt)
with llm_call_span(
agent=agent,
schema_name=schema_name,
provider=provider,
model=model,
prompt_sha12=prompt_sha12,
max_tokens=max_tokens,
) as span:
return _llm_json_call_sync_body(
llm,
prompt,
schema,
max_tokens=max_tokens,
agent=agent,
fallback_factory=fallback_factory,
retry_empty=retry_empty,
http_error_vendor=http_error_vendor,
schema_name=schema_name,
provider=provider,
model=model,
prompt_sha12=prompt_sha12,
span=span,
)
def _llm_json_call_sync_body(
llm: Any,
prompt: str,
schema: type[T],
*,
max_tokens: int,
agent: str,
fallback_factory: Callable[[], T] | None,
retry_empty: bool,
http_error_vendor: LlmHttpErrorVendor,
schema_name: str,
provider: str,
model: str,
prompt_sha12: str,
span: Any,
) -> T:
t0 = time.perf_counter()
attempts_used = 0
input_tokens = 0
output_tokens = 0
raw = ""
try:
raw, attempts_used = _invoke_raw_sync(
raw, attempts_used, input_tokens, output_tokens = _invoke_raw_sync(
llm,
prompt,
max_tokens=max_tokens,
@@ -311,6 +391,12 @@ def llm_json_call(
parse_ok=True,
used_fallback=False,
error_kind=None,
provider=provider,
model=model,
prompt_sha12=prompt_sha12,
input_tokens=input_tokens,
output_tokens=output_tokens,
span=span,
)
if agent_verbose_enabled():
log_agent_payload(
@@ -331,6 +417,12 @@ def llm_json_call(
parse_ok=False,
used_fallback=used_fb,
error_kind=e.kind,
provider=provider,
model=model,
prompt_sha12=prompt_sha12,
input_tokens=input_tokens,
output_tokens=output_tokens,
span=span,
)
if agent_verbose_enabled():
log_agent_payload(
@@ -354,6 +446,12 @@ def llm_json_call(
parse_ok=False,
used_fallback=used_fb,
error_kind="invoke",
provider=provider,
model=model,
prompt_sha12=prompt_sha12,
input_tokens=input_tokens,
output_tokens=output_tokens,
span=span,
)
if agent_verbose_enabled():
log_agent_payload(
@@ -383,13 +481,59 @@ async def allm_json_call(
http_error_vendor: LlmHttpErrorVendor = "deepseek",
) -> T:
"""异步版，语义与 `llm_json_call` 一致。"""
t0 = time.perf_counter()
schema_name = getattr(schema, "__name__", str(schema))
provider, model = infer_provider_model(llm, http_error_vendor=http_error_vendor)
prompt_sha12 = _prompt_sha12(prompt)
with llm_call_span(
agent=agent,
schema_name=schema_name,
provider=provider,
model=model,
prompt_sha12=prompt_sha12,
max_tokens=max_tokens,
) as span:
return await _allm_json_call_async_body(
llm,
prompt,
schema,
max_tokens=max_tokens,
agent=agent,
fallback_factory=fallback_factory,
retry_empty=retry_empty,
http_error_vendor=http_error_vendor,
schema_name=schema_name,
provider=provider,
model=model,
prompt_sha12=prompt_sha12,
span=span,
)
async def _allm_json_call_async_body(
llm: Any,
prompt: str,
schema: type[T],
*,
max_tokens: int,
agent: str,
fallback_factory: Callable[[], T] | None,
retry_empty: bool,
http_error_vendor: LlmHttpErrorVendor,
schema_name: str,
provider: str,
model: str,
prompt_sha12: str,
span: Any,
) -> T:
t0 = time.perf_counter()
attempts_used = 0
input_tokens = 0
output_tokens = 0
raw = ""
try:
raw, attempts_used = await _invoke_raw_async(
raw, attempts_used, input_tokens, output_tokens = await _invoke_raw_async(
llm,
prompt,
max_tokens=max_tokens,
@@ -406,6 +550,12 @@ async def allm_json_call(
parse_ok=True,
used_fallback=False,
error_kind=None,
provider=provider,
model=model,
prompt_sha12=prompt_sha12,
input_tokens=input_tokens,
output_tokens=output_tokens,
span=span,
)
if agent_verbose_enabled():
log_agent_payload(
@@ -426,6 +576,12 @@ async def allm_json_call(
parse_ok=False,
used_fallback=used_fb,
error_kind=e.kind,
provider=provider,
model=model,
prompt_sha12=prompt_sha12,
input_tokens=input_tokens,
output_tokens=output_tokens,
span=span,
)
if agent_verbose_enabled():
log_agent_payload(
@@ -449,6 +605,12 @@ async def allm_json_call(
parse_ok=False,
used_fallback=used_fb,
error_kind="invoke",
provider=provider,
model=model,
prompt_sha12=prompt_sha12,
input_tokens=input_tokens,
output_tokens=output_tokens,
span=span,
)
if agent_verbose_enabled():
log_agent_payload(