This commit is contained in:
zaiun xu
2026-04-09 15:21:21 +08:00
parent 5e1b2117c1
commit e1b514836e
8 changed files with 129 additions and 96 deletions

5
.gitignore vendored
View File

@@ -14,7 +14,8 @@ build/
.ruff_cache/ .ruff_cache/
*.egg *.egg
# Large model weights (download or copy from backup) # Large model weights — 统一放仓库根目录 models/(见 fish_api 默认 settings
models/
FishMeasure/sam_vit_h_4b8939.pth FishMeasure/sam_vit_h_4b8939.pth
# Local / runtime outputs (regenerate on server) # Local / runtime outputs (regenerate on server)
@@ -41,3 +42,5 @@ mockdata/**
# OS / IDE # OS / IDE
.DS_Store .DS_Store
.cursor/ .cursor/
frp/

View File

@@ -1,13 +1,14 @@
"""仅 FastAPI 进程使用 SQLite落库测量/健康结果与 watch 已处理路径。 """仅 FastAPI 进程使用 SQLite落库测量/健康结果与 watch 已处理路径。
FishMeasure / FishAction 子进程不连接、不依赖本库;它们只读写各自文件(如 output 下 FishMeasure / FishAction 子进程不连接、不依赖本库;它们只读写各自文件(如 measure_output 下
weight_prediction.json、临时 pred.json 等),由 fish_api 在子进程结束后读文件并写入本表。 weight_prediction.json、临时 pred.json 等),由 fish_api 在子进程结束后读文件并写入本表。
视频仍使用 measure_output_root、media_root 等原路径 预览视频在 media_rootstart_fresh 会清空 measure_output、media、ingest 临时目录
""" """
from __future__ import annotations from __future__ import annotations
import json import json
import shutil
import sqlite3 import sqlite3
from datetime import datetime, timezone from datetime import datetime, timezone
from pathlib import Path from pathlib import Path
@@ -386,6 +387,30 @@ def add_watch_processed(settings: Settings, path: str, kind: str) -> None:
conn.close() conn.close()
def clear_runtime_compute_dirs(settings: Settings) -> None:
"""清空 FishMeasure / FishAction 运行时目录、托管预览、ingest 临时文件(保留目录本身)。
与 remove_sqlite_database_files 一起在启动脚本中调用,使两条流水线重启后均重新计算。
"""
for base in (
settings.measure_output_root,
settings.action_output_root,
settings.media_root,
settings.stream_tmp_dir,
):
p = Path(base).resolve()
if not p.is_dir():
continue
for child in p.iterdir():
try:
if child.is_dir():
shutil.rmtree(child)
else:
child.unlink()
except OSError as e:
print(f"[prestart-fresh] skip remove {child}: {e}", flush=True)
def remove_sqlite_database_files(settings: Settings) -> None: def remove_sqlite_database_files(settings: Settings) -> None:
"""删除 SQLite 主库及 WAL/SHM 副文件;不存在则忽略。下次 init_db 会重建空库。""" """删除 SQLite 主库及 WAL/SHM 副文件;不存在则忽略。下次 init_db 会重建空库。"""
base = settings.sqlite_path.resolve() base = settings.sqlite_path.resolve()

View File

@@ -0,0 +1,67 @@
"""启动前清空状态SQLite、watch 旧 JSON、测量/行为运行时目录。
由 start.sh / start_fresh.sh 在 uvicorn 之前调用,使 FishMeasure 与 FishAction 均在无缓存下重新推理。
"""
from __future__ import annotations
from pathlib import Path
from app.db import clear_runtime_compute_dirs, remove_sqlite_database_files
from app.settings import get_settings
def _rm_legacy_json(path: Path | None) -> None:
if path is None:
return
try:
if path.is_file():
path.unlink()
print(f"[prestart-fresh] removed legacy JSON {path}", flush=True)
except OSError as e:
print(f"[prestart-fresh] skip {path}: {e}", flush=True)
def run_prestart_fresh() -> None:
s = get_settings()
remove_sqlite_database_files(s)
print(
f"[prestart-fresh] removed SQLite at {s.sqlite_path} (and -wal/-shm if present).",
flush=True,
)
clear_runtime_compute_dirs(s)
print(
"[prestart-fresh] cleared compute dirs: "
f"measure_output={s.measure_output_root}, "
f"action_output={s.action_output_root}, "
f"media={s.media_root}, stream_tmp={s.stream_tmp_dir}",
flush=True,
)
if s.measure_watch_use_state_file:
if s.measure_watch_state_file is not None:
_rm_legacy_json(s.measure_watch_state_file)
elif s.measure_watch_dir is not None:
_rm_legacy_json(
s.measure_watch_dir / ".fishmeasure_watch_processed.json"
)
if s.action_watch_use_state_file:
if s.action_watch_state_file is not None:
_rm_legacy_json(s.action_watch_state_file)
elif s.action_watch_dir is not None:
_rm_legacy_json(
s.action_watch_dir / ".fishaction_watch_processed.json"
)
print("[prestart-fresh] done.", flush=True)
def main() -> None:
run_prestart_fresh()
if __name__ == "__main__":
main()

View File

@@ -13,6 +13,11 @@ def fish_repo_root() -> Path:
return Path(__file__).resolve().parents[2] return Path(__file__).resolve().parents[2]
def models_dir() -> Path:
"""仓库内统一权重目录YOLO / DGCNN / PointNet / X3D / SAM 等),与 FishMeasure 代码目录解耦。"""
return fish_repo_root() / "models"
def _default_stream_tmp() -> Path: def _default_stream_tmp() -> Path:
return fish_repo_root() / "fish_api" / ".data" / "ingest" return fish_repo_root() / "fish_api" / ".data" / "ingest"
@@ -25,6 +30,10 @@ def _default_sqlite_path() -> Path:
return fish_repo_root() / "fish_api" / ".data" / "app.db" return fish_repo_root() / "fish_api" / ".data" / "app.db"
def _default_action_output_root() -> Path:
return fish_repo_root() / "fish_api" / ".data" / "action_output"
class Settings(BaseSettings): class Settings(BaseSettings):
model_config = SettingsConfigDict( model_config = SettingsConfigDict(
env_file=".env", env_file=".env",
@@ -43,7 +52,10 @@ class Settings(BaseSettings):
fish_measure_root: Path = fish_repo_root() / "FishMeasure" fish_measure_root: Path = fish_repo_root() / "FishMeasure"
fish_action_root: Path = fish_repo_root() / "FishAction" fish_action_root: Path = fish_repo_root() / "FishAction"
measure_output_root: Path = fish_repo_root() / "FishMeasure" / "output_weight_estimator" #: FishMeasure 推理输出(与 SQLite、媒体缓存同属 fish_api/.data启动脚本会清空
measure_output_root: Path = fish_repo_root() / "fish_api" / ".data" / "measure_output"
#: FishAction 侧预留目录(与 measure 对称;当前推理多为临时文件,仍随启动清空)
action_output_root: Path = Field(default_factory=_default_action_output_root)
python_fish_measure: str = "" python_fish_measure: str = ""
python_fish_action: str = "" python_fish_action: str = ""
@@ -114,41 +126,19 @@ class Settings(BaseSettings):
@model_validator(mode="after") @model_validator(mode="after")
def _default_paths(self) -> "Settings": def _default_paths(self) -> "Settings":
md = models_dir()
if not self.yolo_model: if not self.yolo_model:
object.__setattr__( object.__setattr__(self, "yolo_model", str(md / "yolo" / "best.pt"))
self,
"yolo_model",
str(
self.fish_measure_root
/ "runs/train/fish_detection_20251127_104658/weights/best.pt"
),
)
if not self.weight_checkpoint: if not self.weight_checkpoint:
object.__setattr__( object.__setattr__(
self, self, "weight_checkpoint", str(md / "weight_estimator" / "best.pt")
"weight_checkpoint",
str(
self.fish_measure_root
/ "weight_estimator/runs/dgcnn_20260312_171043/best.pt"
),
) )
if not self.action_checkpoint: if not self.action_checkpoint:
object.__setattr__( object.__setattr__(
self, self, "action_checkpoint", str(md / "action_x3d" / "checkpoint_best.pt")
"action_checkpoint",
str(self.fish_action_root / "checkpoints/ptv_x3d_m/checkpoint_best.pt"),
) )
if not self.predict_pointcloud_classifier: if not self.predict_pointcloud_classifier:
_pc = ( _pc = md / "pointcloud" / "best_model.pth"
self.fish_measure_root
/ "pointcloud_classifier"
/ "Pointnet_Pointnet2_pytorch"
/ "log"
/ "classification"
/ "fish_pointnet2_finetune"
/ "checkpoints"
/ "best_model.pth"
)
if _pc.is_file(): if _pc.is_file():
object.__setattr__(self, "predict_pointcloud_classifier", str(_pc)) object.__setattr__(self, "predict_pointcloud_classifier", str(_pc))
return self return self

View File

@@ -1,5 +1,5 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# 一键启动 Fish API(在 fish_api 目录下执行 uvicorn读取同目录 .env # 一键启动 Fish API:先清空 SQLite、watch 状态与运行时目录(与 start_fresh.sh 相同),再启动 uvicorn。
# #
# bash fish_api/start.sh # bash fish_api/start.sh
# PORT=8001 HOST=0.0.0.0 bash fish_api/start.sh # PORT=8001 HOST=0.0.0.0 bash fish_api/start.sh
@@ -14,6 +14,14 @@ cd "$DIR"
export PUBLIC_BASE_URL="${PUBLIC_BASE_URL:-http://127.0.0.1:8000}" export PUBLIC_BASE_URL="${PUBLIC_BASE_URL:-http://127.0.0.1:8000}"
unset PYTHON_FISH_MEASURE PYTHON_FISH_ACTION 2>/dev/null || true unset PYTHON_FISH_MEASURE PYTHON_FISH_ACTION 2>/dev/null || true
if command -v uv >/dev/null 2>&1; then
PY=(uv run python)
else
PY=(python3)
fi
"${PY[@]}" -m app.prestart_fresh
PORT="${PORT:-8000}" PORT="${PORT:-8000}"
HOST="${HOST:-0.0.0.0}" HOST="${HOST:-0.0.0.0}"

View File

@@ -1,69 +1,9 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# 一键启动 Fish API删除整个 SQLite 库文件(含 -wal/-shm并删除旧版 watch JSON 状态文件,再启动服务 # 与 start.sh 相同(历史名称保留):清空状态后启动 Fish API
# #
# bash fish_api/start_fresh.sh # bash fish_api/start_fresh.sh
# PORT=8001 HOST=0.0.0.0 bash fish_api/start_fresh.sh # PORT=8001 HOST=0.0.0.0 bash fish_api/start_fresh.sh
# #
# 首次使用请先cd fish_api && uv sync
#
set -euo pipefail set -euo pipefail
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "$DIR" exec bash "$DIR/start.sh"
export PUBLIC_BASE_URL="${PUBLIC_BASE_URL:-http://127.0.0.1:8000}"
unset PYTHON_FISH_MEASURE PYTHON_FISH_ACTION 2>/dev/null || true
if command -v uv >/dev/null 2>&1; then
PY=(uv run python)
else
PY=(python3)
fi
"${PY[@]}" - <<'PY'
from pathlib import Path
from app.db import remove_sqlite_database_files
from app.settings import get_settings
s = get_settings()
def _rm(path: Path | None) -> None:
if path is None:
return
try:
if path.is_file():
path.unlink()
print(f"[start-fresh] removed legacy JSON {path}", flush=True)
except OSError as e:
print(f"[start-fresh] skip {path}: {e}", flush=True)
remove_sqlite_database_files(s)
print(f"[start-fresh] removed SQLite database at {s.sqlite_path} (and -wal/-shm if present).", flush=True)
# 旧版 JSON 若仍存在,启动时会被 load_watch_processed 合并进 SQLite必须一并删除
if s.measure_watch_use_state_file:
if s.measure_watch_state_file is not None:
_rm(s.measure_watch_state_file)
elif s.measure_watch_dir is not None:
_rm(s.measure_watch_dir / ".fishmeasure_watch_processed.json")
if s.action_watch_use_state_file:
if s.action_watch_state_file is not None:
_rm(s.action_watch_state_file)
elif s.action_watch_dir is not None:
_rm(s.action_watch_dir / ".fishaction_watch_processed.json")
print("[start-fresh] done.", flush=True)
PY
PORT="${PORT:-8000}"
HOST="${HOST:-0.0.0.0}"
if command -v uv >/dev/null 2>&1; then
exec uv run uvicorn app.main:app --host "$HOST" --port "$PORT"
else
exec uvicorn app.main:app --host "$HOST" --port "$PORT"
fi

View File

@@ -1,5 +1,5 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# 仓库根目录入口:与 fish_api/start.sh 等价 # 仓库根目录入口:与 fish_api/start.sh 等价(启动前清空 SQLite、watch 状态、measure/action 输出目录等)
# #
# conda activate fishserver # 若不用 uv # conda activate fishserver # 若不用 uv
# export PUBLIC_BASE_URL=http://<本机对外IP>:8001 # export PUBLIC_BASE_URL=http://<本机对外IP>:8001

View File

@@ -1,5 +1,5 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# 仓库根目录入口:删除 SQLite 库文件(及旧 JSON 状态文件)后启动 Fish API # 与 scripts/run_fishserver.sh 等价(历史名称保留);均会先执行 app.prestart_fresh 再启动 uvicorn
# #
# bash scripts/start_fishapi_fresh.sh # bash scripts/start_fishapi_fresh.sh
# PORT=8001 bash scripts/start_fishapi_fresh.sh # PORT=8001 bash scripts/start_fishapi_fresh.sh