"""统一以 loguru 持久化日志: - 控制台 stderr:中文 + 颜色(人类可读); - ``runtime_YYYY-MM-DD.log``:中文人读 + 耗时,按本地日期切分(``rotation="00:00"``),按天保留; - ``events_YYYY-MM-DD.jsonl``:结构化事件,仅当 ``logger.bind(event=True, ...)`` 时写入, 字段 ``ts / pipeline / step / status / run_id / source / metrics / duration_ms / error``; - 同时把标准 ``logging``(uvicorn / fastapi / starlette)桥接到 loguru。 业务侧用 ``stage(...)`` 上下文管理器写阶段开始 / 结束(自动埋点耗时与状态), 或 ``bind_pipeline(...)`` 在长流程内统一带上 ``run_id / pipeline / source``。 """ from __future__ import annotations import json import logging import os import sys import time import uuid from contextlib import contextmanager from pathlib import Path from typing import Any, Iterator, Mapping, Optional, Union from loguru import logger _loguru_console_configured = False _loguru_file_configured = False _runtime_sink_id: Optional[int] = None _events_sink_id: Optional[int] = None _console_sink_id: Optional[int] = None _events_sink_obj: Optional["_EventsJsonlSink"] = None # --------------------------------------------------------------------------- # JSON helpers # --------------------------------------------------------------------------- def format_json_pretty( data: Any, *, indent: int = 2, max_chars: Optional[int] = 24_000, ) -> str: """将对象格式化为带缩进、保留中文的 JSON 字符串,供 loguru 多行输出。 ``max_chars`` 用于避免单条日志过大;超长时截断并标注。 """ try: s = json.dumps(data, ensure_ascii=False, indent=indent, default=str) except (TypeError, ValueError): return repr(data) if max_chars is not None and len(s) > max_chars: return s[:max_chars] + "\n... (truncated, max_chars=" + str(max_chars) + ")" return s # --------------------------------------------------------------------------- # logging -> loguru bridge # --------------------------------------------------------------------------- class InterceptHandler(logging.Handler): """将标准 logging 记录转发到 loguru。""" def emit(self, record: logging.LogRecord) -> None: try: level = logger.level(record.levelname).name # type: Union[str, int] except ValueError: level = record.levelno logger.opt(depth=6, exception=record.exc_info).log(level, record.getMessage()) # --------------------------------------------------------------------------- # Sink filters & formatters # --------------------------------------------------------------------------- def _is_event_record(record: Mapping[str, Any]) -> bool: return bool(record["extra"].get("event")) def _runtime_filter(record: Mapping[str, Any]) -> bool: """主 runtime 日志:忽略纯结构化事件(避免与 events JSONL 重复)。""" # 即使 event=True 也保留 message(阶段开始/结束的中文描述对人有用), # 真正“仅机器读”的事件由 events sink 收集;为减少噪音这里仅排除 status=start # 的开始事件(结束事件作为完成日志保留),见 stage() 实现。 extra = record["extra"] if extra.get("event") and extra.get("event_silent"): return False return True def _events_filter(record: Mapping[str, Any]) -> bool: return _is_event_record(record) def _runtime_format(record: Mapping[str, Any]) -> str: """中文 runtime 文件 / 控制台格式:``时间 | 级别 | [pipeline] message``。""" extra = record["extra"] pipeline = extra.get("pipeline") or "-" parts = [ "{time:YYYY-MM-DD HH:mm:ss.SSS}", "{level: <8}", f"[{pipeline}]", "{message}", ] fmt = " | ".join(parts) if record["exception"] is not None: fmt += "\n{exception}" return fmt + "\n" def _runtime_format_colored(record: Mapping[str, Any]) -> str: extra = record["extra"] pipeline = extra.get("pipeline") or "-" fmt = ( "{time:YYYY-MM-DD HH:mm:ss.SSS} | " "{level: <8} | " f"[{pipeline}] | " "{message}" ) if record["exception"] is not None: fmt += "\n{exception}" return fmt + "\n" def _build_event_payload(record: Mapping[str, Any]) -> dict: extra = record["extra"] payload: dict = { "ts": record["time"].isoformat(), "level": record["level"].name, "pipeline": extra.get("pipeline") or "", "step": extra.get("step") or "", "status": extra.get("status") or "", "run_id": extra.get("run_id") or "", "source": extra.get("source") or "", "message": record["message"], "metrics": extra.get("metrics") or {}, } if "duration_ms" in extra and extra["duration_ms"] is not None: payload["duration_ms"] = extra["duration_ms"] if extra.get("fish_id") is not None: payload["fish_id"] = extra["fish_id"] if extra.get("error"): payload["error"] = extra["error"] if record["exception"] is not None: payload["exception"] = str(record["exception"]) return payload class _EventsJsonlSink: """以日期滚动写入 ``events_YYYY-MM-DD.jsonl``。 自己负责按本地日期滚动 + 旧文件保留清理;不走 loguru 的 ``format`` 通道, 避免 ``{"ts"}`` 这类 JSON 被当作格式占位符抛出 ``KeyError``。 """ def __init__( self, log_dir: Path, *, retention_days: int, prefix: str = "events_", suffix: str = ".jsonl", ) -> None: self._log_dir = log_dir self._prefix = prefix self._suffix = suffix self._retention_days = max(1, int(retention_days)) self._lock = __import__("threading").Lock() self._current_date: Optional[str] = None self._fp = None # type: Optional[Any] def _open_for_today(self) -> None: today = time.strftime("%Y-%m-%d", time.localtime()) if self._current_date == today and self._fp is not None: return if self._fp is not None: try: self._fp.close() except Exception: pass self._fp = None self._log_dir.mkdir(parents=True, exist_ok=True) path = self._log_dir / f"{self._prefix}{today}{self._suffix}" self._fp = open(path, "a", encoding="utf-8") self._current_date = today self._cleanup_old() def _cleanup_old(self) -> None: try: cutoff = time.time() - self._retention_days * 86400 for f in self._log_dir.glob(f"{self._prefix}*{self._suffix}"): try: if f.stat().st_mtime < cutoff: f.unlink() except OSError: pass except Exception: pass def __call__(self, message: Any) -> None: record = message.record if not record["extra"].get("event"): return payload = _build_event_payload(record) try: line = json.dumps(payload, ensure_ascii=False, default=str) except (TypeError, ValueError): line = json.dumps( {"ts": payload.get("ts", ""), "message": str(payload)}, ensure_ascii=False, ) with self._lock: try: self._open_for_today() if self._fp is not None: self._fp.write(line + "\n") self._fp.flush() except OSError: pass def close(self) -> None: with self._lock: if self._fp is not None: try: self._fp.close() except Exception: pass self._fp = None # --------------------------------------------------------------------------- # Public setup # --------------------------------------------------------------------------- def _default_log_dir() -> Path: here = Path(__file__).resolve() return here.parents[1] / ".data" / "logs" / "fish_api" def setup_logging( settings: Optional[Any] = None, *, force: bool = False, ) -> None: """配置 loguru sink,并把 uvicorn / fastapi 等 logger 接到 loguru。 - 第一次调用(main.py 顶部 import 时):仅装 stderr sink,不依赖 Settings; - 第二次调用(FastAPI lifespan 启动时,``settings=get_settings()``):装 runtime 与 events 文件 sink;``force=True`` 时会先卸载已有同类 sink 再装。 """ global _loguru_console_configured, _loguru_file_configured global _runtime_sink_id, _events_sink_id, _console_sink_id, _events_sink_obj level = (os.environ.get("LOG_LEVEL") or "INFO").upper() if settings is not None: level = (getattr(settings, "log_level", None) or level).upper() if not _loguru_console_configured: # 第一次 setup:清空所有内置 sink,只装 stderr。 logger.remove() _console_sink_id = logger.add( sys.stderr, level=level, colorize=sys.stderr.isatty(), format=( _runtime_format_colored if sys.stderr.isatty() else _runtime_format ), filter=_runtime_filter, backtrace=False, diagnose=False, enqueue=False, ) _loguru_console_configured = True # 桥接标准 logging intercept = InterceptHandler() logging.root.handlers = [intercept] logging.root.setLevel(logging.DEBUG) for name in ( "uvicorn", "uvicorn.error", "uvicorn.access", "uvicorn.asgi", "fastapi", "starlette", "starlette.requests", ): lg = logging.getLogger(name) lg.handlers.clear() lg.propagate = True if settings is None: return # 安装文件 sink(runtime + events) if force or not _loguru_file_configured: if _runtime_sink_id is not None: try: logger.remove(_runtime_sink_id) except ValueError: pass _runtime_sink_id = None if _events_sink_id is not None: try: logger.remove(_events_sink_id) except ValueError: pass _events_sink_id = None if _events_sink_obj is not None: try: _events_sink_obj.close() except Exception: pass _events_sink_obj = None log_dir = Path(getattr(settings, "log_dir", None) or _default_log_dir()) try: log_dir.mkdir(parents=True, exist_ok=True) except OSError as e: logger.warning( "[logging] 无法创建日志目录 {}: {}(仅控制台日志生效)", log_dir, e ) return rotation = getattr(settings, "log_rotation", None) or "00:00" retention_days = int(getattr(settings, "log_retention_days", 14) or 14) retention = f"{retention_days} days" file_level = (getattr(settings, "log_file_level", None) or level).upper() runtime_path = str(log_dir / "runtime_{time:YYYY-MM-DD}.log") _runtime_sink_id = logger.add( runtime_path, level=file_level, format=_runtime_format, filter=_runtime_filter, rotation=rotation, retention=retention, encoding="utf-8", enqueue=True, backtrace=False, diagnose=False, ) _events_sink_obj = _EventsJsonlSink( log_dir, retention_days=retention_days ) _events_sink_id = logger.add( _events_sink_obj, level="DEBUG", filter=_events_filter, enqueue=True, backtrace=False, diagnose=False, ) _loguru_file_configured = True # --------------------------------------------------------------------------- # Stage helpers # --------------------------------------------------------------------------- def new_run_id(pipeline: str = "") -> str: """生成短 run_id,便于跨日志/事件关联。""" prefix = (pipeline or "run").replace(" ", "_")[:24] return f"{prefix}-{uuid.uuid4().hex[:12]}" @contextmanager def stage( name_zh: str, *, pipeline: str, step: str, run_id: Optional[str] = None, source: str = "", fish_id: Optional[Any] = None, metrics: Optional[Mapping[str, Any]] = None, log_start: bool = True, log_success: bool = True, raise_on_error: bool = True, ) -> Iterator[str]: """中文阶段埋点:开始 / 完成 / 失败 自动写 runtime + events。 - ``run_id`` 不传时自动生成;通过 ``yield rid`` 暴露给调用方。 - 失败默认会重新抛出,仅写一条 ``status=fail`` 事件再抛。 - ``raise_on_error=False`` 时吞掉异常(仅记录),调用方需自行恢复。 """ rid = run_id or new_run_id(pipeline) src = str(source) if source else "" base_metrics: dict[str, Any] = dict(metrics or {}) def _bind(extra_metrics: Optional[Mapping[str, Any]] = None, **extra: Any) -> Any: m = dict(base_metrics) if extra_metrics: m.update(extra_metrics) return logger.bind( event=True, pipeline=pipeline, step=step, run_id=rid, source=src, fish_id=fish_id, metrics=m, **extra, ) t0 = time.perf_counter() if log_start: _bind(status="start", event_silent=False).info( "开始: {} | source={}", name_zh, src or "-" ) try: yield rid except Exception as e: dt_ms = (time.perf_counter() - t0) * 1000.0 _bind( status="fail", duration_ms=round(dt_ms, 3), error=f"{type(e).__name__}: {e}", ).error("失败: {} | 耗时={:.2f}s | {}: {}", name_zh, dt_ms / 1000.0, type(e).__name__, e) if raise_on_error: raise return dt_ms = (time.perf_counter() - t0) * 1000.0 if log_success: _bind(status="success", duration_ms=round(dt_ms, 3)).success( "完成: {} | 耗时={:.2f}s", name_zh, dt_ms / 1000.0 ) def bind_pipeline( pipeline: str, *, run_id: Optional[str] = None, source: str = "", fish_id: Optional[Any] = None, **extra: Any, ) -> Any: """在长流程内绑定 ``pipeline / run_id / source / fish_id``,返回 logger 代理。""" rid = run_id or new_run_id(pipeline) return logger.bind( pipeline=pipeline, run_id=rid, source=str(source) if source else "", fish_id=fish_id, **extra, ) def emit_event( name_zh: str, *, pipeline: str, step: str, status: str = "info", run_id: Optional[str] = None, source: str = "", fish_id: Optional[Any] = None, metrics: Optional[Mapping[str, Any]] = None, duration_ms: Optional[float] = None, level: str = "INFO", ) -> None: """写一条带 ``event=True`` 的 INFO 日志(runtime + events 都写)。""" payload = logger.bind( event=True, pipeline=pipeline, step=step, status=status, run_id=run_id or new_run_id(pipeline), source=str(source) if source else "", fish_id=fish_id, metrics=dict(metrics or {}), duration_ms=duration_ms, ) payload.log(level.upper(), name_zh)