"""仅 FastAPI 进程使用 SQLite:落库测量/健康结果与 watch 已处理路径。 FishMeasure / FishAction 子进程不连接、不依赖本库;它们只读写各自文件(如 output 下 weight_prediction.json、临时 pred.json 等),由 fish_api 在子进程结束后读文件并写入本表。 视频仍使用 measure_output_root、media_root 等原路径。 """ from __future__ import annotations import json import sqlite3 from datetime import datetime, timezone from pathlib import Path from typing import Any, Optional, Set, Tuple from app.settings import Settings from app.state import HealthSnapshot, MeasureSnapshot def _connect(path: Path) -> sqlite3.Connection: path.parent.mkdir(parents=True, exist_ok=True) conn = sqlite3.connect(str(path), check_same_thread=False, isolation_level=None) conn.row_factory = sqlite3.Row conn.execute("PRAGMA journal_mode=WAL") conn.execute("PRAGMA foreign_keys=ON") return conn def init_db(settings: Settings) -> None: conn = _connect(settings.sqlite_path) try: conn.executescript( """ CREATE TABLE IF NOT EXISTS measure_snapshots ( id INTEGER PRIMARY KEY AUTOINCREMENT, created_at TEXT NOT NULL, result_json TEXT NOT NULL, video_left TEXT NOT NULL DEFAULT '', video_right TEXT NOT NULL DEFAULT '', error TEXT, raw_prediction_path TEXT, source_path TEXT ); CREATE TABLE IF NOT EXISTS health_snapshots ( id INTEGER PRIMARY KEY AUTOINCREMENT, created_at TEXT NOT NULL, behavior_result TEXT NOT NULL DEFAULT '', health_result TEXT NOT NULL DEFAULT '', raw_class_en TEXT NOT NULL DEFAULT '', error TEXT, source_path TEXT ); CREATE TABLE IF NOT EXISTS watch_processed ( path TEXT NOT NULL, kind TEXT NOT NULL CHECK (kind IN ('measure', 'action')), PRIMARY KEY (path, kind) ); CREATE TABLE IF NOT EXISTS delivery_cursor ( kind TEXT PRIMARY KEY CHECK (kind IN ('measure', 'health')), last_delivered_id INTEGER NOT NULL DEFAULT 0 ); """ ) _ensure_delivery_cursors(conn) finally: conn.close() def _ensure_delivery_cursors(conn: sqlite3.Connection) -> None: """为每条流插入一行游标;首次插入时 last_delivered_id=当前 MAX(id),避免升级后逐条投递历史快照。""" for kind, table in ( ("measure", "measure_snapshots"), ("health", "health_snapshots"), ): row = conn.execute( "SELECT 1 FROM delivery_cursor WHERE kind = ?", (kind,) ).fetchone() if row is None: mid = conn.execute( f"SELECT COALESCE(MAX(id), 0) FROM {table}" ).fetchone()[0] conn.execute( "INSERT INTO delivery_cursor (kind, last_delivered_id) VALUES (?, ?)", (kind, int(mid)), ) conn.commit() def save_measure_snapshot( settings: Settings, snap: MeasureSnapshot, source_path: Optional[str] = None, ) -> None: init_db(settings) conn = _connect(settings.sqlite_path) try: ts = ( snap.updated_at.isoformat() if snap.updated_at else datetime.now(timezone.utc).isoformat() ) conn.execute( """ INSERT INTO measure_snapshots ( created_at, result_json, video_left, video_right, error, raw_prediction_path, source_path ) VALUES (?, ?, ?, ?, ?, ?, ?) """, ( ts, json.dumps(snap.result, ensure_ascii=False), snap.video_left, snap.video_right, snap.error, snap.raw_prediction_path, source_path, ), ) finally: conn.close() def save_health_snapshot( settings: Settings, snap: HealthSnapshot, source_path: Optional[str] = None, ) -> None: init_db(settings) conn = _connect(settings.sqlite_path) try: ts = ( snap.updated_at.isoformat() if snap.updated_at else datetime.now(timezone.utc).isoformat() ) conn.execute( """ INSERT INTO health_snapshots ( created_at, behavior_result, health_result, raw_class_en, error, source_path ) VALUES (?, ?, ?, ?, ?, ?) """, ( ts, snap.behavior_result, snap.health_result, snap.raw_class_en, snap.error, source_path, ), ) finally: conn.close() def _parse_dt(s: Optional[str]) -> Optional[datetime]: if not s: return None try: return datetime.fromisoformat(s.replace("Z", "+00:00")) except ValueError: return None def get_latest_measure(settings: Settings) -> MeasureSnapshot: init_db(settings) conn = _connect(settings.sqlite_path) try: row = conn.execute( """ SELECT created_at, result_json, video_left, video_right, error, raw_prediction_path FROM measure_snapshots ORDER BY id DESC LIMIT 1 """ ).fetchone() if row is None: return MeasureSnapshot(result=[], video_left="", video_right="") data: Any = json.loads(row["result_json"]) if not isinstance(data, list): data = [] return MeasureSnapshot( result=data, video_left=row["video_left"] or "", video_right=row["video_right"] or "", updated_at=_parse_dt(row["created_at"]), error=row["error"], raw_prediction_path=row["raw_prediction_path"], ) finally: conn.close() def get_latest_health(settings: Settings) -> HealthSnapshot: init_db(settings) conn = _connect(settings.sqlite_path) try: row = conn.execute( """ SELECT created_at, behavior_result, health_result, raw_class_en, error FROM health_snapshots ORDER BY id DESC LIMIT 1 """ ).fetchone() if row is None: return HealthSnapshot(behavior_result="", health_result="") return HealthSnapshot( behavior_result=row["behavior_result"] or "", health_result=row["health_result"] or "", updated_at=_parse_dt(row["created_at"]), error=row["error"], raw_class_en=row["raw_class_en"] or "", ) finally: conn.close() def _last_delivered_id( conn: sqlite3.Connection, kind: str, snapshots_table: str ) -> int: row = conn.execute( "SELECT last_delivered_id FROM delivery_cursor WHERE kind = ?", (kind,) ).fetchone() if row is not None: return int(row["last_delivered_id"]) mid = conn.execute( f"SELECT COALESCE(MAX(id), 0) FROM {snapshots_table}" ).fetchone()[0] conn.execute( "INSERT INTO delivery_cursor (kind, last_delivered_id) VALUES (?, ?)", (kind, int(mid)), ) return int(mid) def pop_next_measure( settings: Settings, ) -> Tuple[MeasureSnapshot, bool, Optional[int]]: """取队首未投递的 measure 快照并推进游标;无未投递时 has_new=False。""" init_db(settings) conn = _connect(settings.sqlite_path) try: conn.execute("BEGIN IMMEDIATE") last_id = _last_delivered_id(conn, "measure", "measure_snapshots") next_row = conn.execute( """ SELECT id, created_at, result_json, video_left, video_right, error, raw_prediction_path FROM measure_snapshots WHERE id > ? ORDER BY id ASC LIMIT 1 """, (last_id,), ).fetchone() if next_row is None: conn.commit() return MeasureSnapshot(result=[], video_left="", video_right=""), False, None nid = int(next_row["id"]) conn.execute( "UPDATE delivery_cursor SET last_delivered_id = ? WHERE kind = ?", (nid, "measure"), ) conn.commit() data: Any = json.loads(next_row["result_json"]) if not isinstance(data, list): data = [] snap = MeasureSnapshot( result=data, video_left=next_row["video_left"] or "", video_right=next_row["video_right"] or "", updated_at=_parse_dt(next_row["created_at"]), error=next_row["error"], raw_prediction_path=next_row["raw_prediction_path"], ) return snap, True, nid except Exception: conn.rollback() raise finally: conn.close() def pop_next_health(settings: Settings) -> Tuple[HealthSnapshot, bool, Optional[int]]: """取队首未投递的 health 快照并推进游标;无未投递时 has_new=False。""" init_db(settings) conn = _connect(settings.sqlite_path) try: conn.execute("BEGIN IMMEDIATE") last_id = _last_delivered_id(conn, "health", "health_snapshots") next_row = conn.execute( """ SELECT id, created_at, behavior_result, health_result, raw_class_en, error FROM health_snapshots WHERE id > ? ORDER BY id ASC LIMIT 1 """, (last_id,), ).fetchone() if next_row is None: conn.commit() return HealthSnapshot(behavior_result="", health_result=""), False, None nid = int(next_row["id"]) conn.execute( "UPDATE delivery_cursor SET last_delivered_id = ? WHERE kind = ?", (nid, "health"), ) conn.commit() snap = HealthSnapshot( behavior_result=next_row["behavior_result"] or "", health_result=next_row["health_result"] or "", updated_at=_parse_dt(next_row["created_at"]), error=next_row["error"], raw_class_en=next_row["raw_class_en"] or "", ) return snap, True, nid except Exception: conn.rollback() raise finally: conn.close() def _load_json_processed_set(path: Path) -> Set[str]: if not path.is_file(): return set() try: with open(path, encoding="utf-8") as f: data: Any = json.load(f) if isinstance(data, list): return set(str(x) for x in data) if isinstance(data, dict) and "processed" in data: return set(str(x) for x in data["processed"]) except (json.JSONDecodeError, OSError): pass return set() def load_watch_processed(settings: Settings, state_file: Path, kind: str) -> Set[str]: """从 SQLite 读取已处理路径;若存在旧版 JSON 状态文件则合并导入(幂等)。""" assert kind in ("measure", "action") init_db(settings) conn = _connect(settings.sqlite_path) try: for p in _load_json_processed_set(state_file): conn.execute( "INSERT OR IGNORE INTO watch_processed (path, kind) VALUES (?, ?)", (p, kind), ) conn.commit() cur = conn.execute( "SELECT path FROM watch_processed WHERE kind = ?", (kind,) ) return {r[0] for r in cur} finally: conn.close() def add_watch_processed(settings: Settings, path: str, kind: str) -> None: assert kind in ("measure", "action") init_db(settings) conn = _connect(settings.sqlite_path) try: conn.execute( "INSERT OR IGNORE INTO watch_processed (path, kind) VALUES (?, ?)", (path, kind), ) conn.commit() finally: conn.close() def remove_sqlite_database_files(settings: Settings) -> None: """删除 SQLite 主库及 WAL/SHM 副文件;不存在则忽略。下次 init_db 会重建空库。""" base = settings.sqlite_path.resolve() for p in (base, Path(str(base) + "-wal"), Path(str(base) + "-shm")): try: if p.is_file(): p.unlink() except OSError: pass def clear_watch_cache_and_snapshots(settings: Settings) -> None: """清空 watch 已处理路径与对应快照,便于重新跑推理(与 measure/action_watch 的 use_state_file 开关一致)。""" init_db(settings) conn = _connect(settings.sqlite_path) try: if settings.measure_watch_use_state_file: conn.execute("DELETE FROM watch_processed WHERE kind = ?", ("measure",)) conn.execute("DELETE FROM measure_snapshots") conn.execute( "UPDATE delivery_cursor SET last_delivered_id = 0 WHERE kind = ?", ("measure",), ) if settings.action_watch_use_state_file: conn.execute("DELETE FROM watch_processed WHERE kind = ?", ("action",)) conn.execute("DELETE FROM health_snapshots") conn.execute( "UPDATE delivery_cursor SET last_delivered_id = 0 WHERE kind = ?", ("health",), ) conn.commit() finally: conn.close()