From 941c71e991007ef7e3b412eb7abe9f3eabb0d840 Mon Sep 17 00:00:00 2001 From: Kevin Date: Fri, 22 May 2026 11:15:22 +0800 Subject: [PATCH] Improve speed --- backend/Dockerfile | 5 +- .../infer_doctor_from_video.py | 24 +++++-- .../5.15/src/orchestrator.py | 6 +- backend/app/algo_host/batch_service.py | 20 +++--- backend/app/algo_host/job_workspace.py | 16 ++--- backend/app/algo_host/subprocess_runner.py | 3 + backend/app/algo_host/transcode.py | 13 +++- backend/app/routers/recording_demo.py | 4 +- backend/tests/test_algo_host_batch.py | 64 ++++++++++++++++--- .../test_fastapi_algorithm_subprocess.py | 3 +- backend/tests/test_video_batch_cleanup.py | 6 +- clients/demo-client/app.js | 9 ++- clients/demo-client/server.py | 7 +- docs/Docker部署.md | 1 + 14 files changed, 132 insertions(+), 49 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index 10a8565..1f9eebc 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -12,14 +12,17 @@ RUN sed -i \ -e 's|http://deb.debian.org/debian|https://mirrors.aliyun.com/debian|g' \ /etc/apt/sources.list.d/debian.sources -# OpenCV (pulled in by ultralytics) links against X11 client libs; slim images omit them. +# OpenCV / MediaPipe (doctor pose) need GL + GLES in slim images; omit X11/GUI stack. RUN apt-get update && apt-get install -y --no-install-recommends \ docker.io \ ffmpeg \ fontconfig \ fonts-noto-cjk \ fonts-wqy-microhei \ + libegl1 \ + libgbm1 \ libgl1 \ + libgles2 \ libglib2.0-0 \ libgomp1 \ libxcb1 \ diff --git a/backend/algorithm_subprocesses/5.15/doctor_identity_package/infer_doctor_from_video.py b/backend/algorithm_subprocesses/5.15/doctor_identity_package/infer_doctor_from_video.py index 22f9b3b..b42a3c4 100755 --- a/backend/algorithm_subprocesses/5.15/doctor_identity_package/infer_doctor_from_video.py +++ b/backend/algorithm_subprocesses/5.15/doctor_identity_package/infer_doctor_from_video.py @@ -43,6 +43,23 @@ POSE_LITE_URL = ( POSE_LITE_NAME = "pose_landmarker_lite.task" +def build_pose_landmarker( + model_path: Path, + *, + min_pose_detection_confidence: float = 0.3, +) -> PoseLandmarker: + """Headless/Docker-safe: force CPU delegate (avoids libGLESv2 GPU path).""" + opts = PoseLandmarkerOptions( + base_options=BaseOptions( + model_asset_path=str(model_path), + delegate=BaseOptions.Delegate.CPU, + ), + running_mode=VisionRunningMode.IMAGE, + min_pose_detection_confidence=min_pose_detection_confidence, + ) + return PoseLandmarker.create_from_options(opts) + + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description="Input mp4 -> middle 10s pose crop -> doctor identity", @@ -282,12 +299,7 @@ def main() -> int: try: model_path = _ensure_pose_lite_model(THIS_DIR / ".mediapipe_models") - opts = PoseLandmarkerOptions( - base_options=BaseOptions(model_asset_path=str(model_path)), - running_mode=VisionRunningMode.IMAGE, - min_pose_detection_confidence=0.3, - ) - landmarker = PoseLandmarker.create_from_options(opts) + landmarker = build_pose_landmarker(model_path, min_pose_detection_confidence=0.3) try: best_crop = pick_best_person_crop( video_path=args.video, diff --git a/backend/algorithm_subprocesses/5.15/src/orchestrator.py b/backend/algorithm_subprocesses/5.15/src/orchestrator.py index 736705a..1c71fcf 100755 --- a/backend/algorithm_subprocesses/5.15/src/orchestrator.py +++ b/backend/algorithm_subprocesses/5.15/src/orchestrator.py @@ -57,14 +57,12 @@ def _infer_doctor_text(args: Namespace, video_path: Path) -> str: try: doctor_mod = _load_doctor_module(script_path) model_path = doctor_mod._ensure_pose_lite_model(script_path.parent / ".mediapipe_models") - opts = doctor_mod.PoseLandmarkerOptions( - base_options=doctor_mod.BaseOptions(model_asset_path=str(model_path)), - running_mode=doctor_mod.VisionRunningMode.IMAGE, + landmarker = doctor_mod.build_pose_landmarker( + model_path, min_pose_detection_confidence=float( args.doctor_identity_pose_min_detection_confidence ), ) - landmarker = doctor_mod.PoseLandmarker.create_from_options(opts) try: best_crop = doctor_mod.pick_best_person_crop( video_path=video_path, diff --git a/backend/app/algo_host/batch_service.py b/backend/app/algo_host/batch_service.py index da4f844..e2fd4a5 100644 --- a/backend/app/algo_host/batch_service.py +++ b/backend/app/algo_host/batch_service.py @@ -20,9 +20,9 @@ from app.algo_host.result_adapter import ( ) from app.algo_host.subprocess_runner import run_batch_main, run_visualization_script from app.algo_host.transcode import ( - ensure_batch_pipeline_input_video, is_browser_compatible_mp4, is_readable_mp4, + stage_batch_pipeline_input, transcode_visualization_for_browser, ) from app.domain.consumption import SurgeryConsumptionStored @@ -189,19 +189,23 @@ class BatchAlgorithmService: candidates = resolve_reference_candidates(candidate_consumables) candidate_key = candidate_cache_key(candidates) - surgery_input_dir = self._root_dir / surgery_id / "input" - surgery_input_dir.mkdir(parents=True, exist_ok=True) - surgery_input = surgery_input_dir / f"{digest[:12]}.mp4" - ensure_batch_pipeline_input_video( + pipeline_video = ( + self._root_dir + / "cache" + / digest + / "input" + / f"pipeline{uploaded_video_path.suffix or '.mp4'}" + ) + stage_batch_pipeline_input( source_path=uploaded_video_path, - dest_path=surgery_input, + dest_path=pipeline_video, ) cache_dir = self._root_dir / "cache" / digest / candidate_key job = prepare_batch_job( bundle_dir=self._bundle_dir_override, cache_dir=cache_dir, - uploaded_video_path=uploaded_video_path, + pipeline_video_path=pipeline_video, candidate_consumables=candidates, ) @@ -240,7 +244,7 @@ class BatchAlgorithmService: return BatchRunResult( video_sha256=digest, candidate_cache_key=candidate_key, - input_path=surgery_input, + input_path=pipeline_video, work_dir=job.work_dir, output_path=job.output_path, details=details, diff --git a/backend/app/algo_host/job_workspace.py b/backend/app/algo_host/job_workspace.py index 06b32bb..1f13560 100644 --- a/backend/app/algo_host/job_workspace.py +++ b/backend/app/algo_host/job_workspace.py @@ -10,7 +10,6 @@ from pathlib import Path import yaml from app.algo_host.bundle import load_reference_default_config, resolve_reference_bundle_dir -from app.algo_host.transcode import ensure_batch_pipeline_input_video from app.consumable_catalog import build_name_mapping @@ -75,22 +74,17 @@ def prepare_batch_job( *, bundle_dir: Path | None, cache_dir: Path, - uploaded_video_path: Path, + pipeline_video_path: Path, candidate_consumables: list[str], ) -> BatchJobFiles: root = resolve_reference_bundle_dir(bundle_dir) - cache_input_dir = cache_dir / "input" cache_output_dir = cache_dir / "output" cache_work_dir = cache_dir / "work" cache_config_dir = cache_dir / "config" - for d in (cache_input_dir, cache_output_dir, cache_work_dir, cache_config_dir): + for d in (cache_output_dir, cache_work_dir, cache_config_dir): d.mkdir(parents=True, exist_ok=True) - cache_input = cache_input_dir / "input.mp4" - ensure_batch_pipeline_input_video( - source_path=uploaded_video_path, - dest_path=cache_input, - ) + pipeline_video = pipeline_video_path.resolve() output_path = cache_output_dir / "result.tsv" excel_path = cache_config_dir / "商品信息表.xlsx" whitelist_path = cache_config_dir / "whitelist.json" @@ -100,7 +94,7 @@ def prepare_batch_job( write_reference_whitelist_json(whitelist_path, candidate_consumables=candidate_consumables) config = build_job_config( bundle_dir=root, - video_path=cache_input.resolve(), + video_path=pipeline_video, output_path=output_path.resolve(), work_dir=cache_work_dir.resolve(), excel_path=excel_path.resolve(), @@ -116,5 +110,5 @@ def prepare_batch_job( whitelist_path=whitelist_path, output_path=output_path, work_dir=cache_work_dir, - input_video_path=cache_input, + input_video_path=pipeline_video, ) diff --git a/backend/app/algo_host/subprocess_runner.py b/backend/app/algo_host/subprocess_runner.py index dcf9bd3..819cc10 100644 --- a/backend/app/algo_host/subprocess_runner.py +++ b/backend/app/algo_host/subprocess_runner.py @@ -19,6 +19,9 @@ def build_reference_env() -> dict[str, str]: env = os.environ.copy() env["PYTHONFAULTHANDLER"] = "1" env["PYTHONUNBUFFERED"] = "1" + # Headless containers: avoid OpenGL/EGL init noise from CV/MediaPipe defaults. + env.setdefault("OPENCV_OPENCL_RUNTIME", "") + env.setdefault("QT_QPA_PLATFORM", "offscreen") return env diff --git a/backend/app/algo_host/transcode.py b/backend/app/algo_host/transcode.py index 3283059..f353224 100644 --- a/backend/app/algo_host/transcode.py +++ b/backend/app/algo_host/transcode.py @@ -190,8 +190,17 @@ def normalize_batch_input_video(source_path: Path, output_path: Path) -> bool: return True +def stage_batch_pipeline_input(*, source_path: Path, dest_path: Path) -> None: + """Copy upload to digest-level pipeline input without browser normalize/transcode.""" + + dest_path.parent.mkdir(parents=True, exist_ok=True) + if dest_path.is_file() and dest_path.stat().st_size > 0: + return + shutil.copy2(source_path, dest_path) + + def ensure_batch_pipeline_input_video(*, source_path: Path, dest_path: Path) -> None: - import shutil as sh + """Browser-compatible normalize/copy; use only for visualization inputs, not batch inference.""" dest_path.parent.mkdir(parents=True, exist_ok=True) if dest_path.is_file() and dest_path.stat().st_size > 0 and not batch_input_needs_normalize(dest_path): @@ -205,7 +214,7 @@ def ensure_batch_pipeline_input_video(*, source_path: Path, dest_path: Path) -> dest_path, ) if not dest_path.is_file(): - sh.copy2(source_path, dest_path) + shutil.copy2(source_path, dest_path) def transcode_visualization_for_browser(source_path: Path, output_path: Path) -> bool: diff --git a/backend/app/routers/recording_demo.py b/backend/app/routers/recording_demo.py index 2cd53e6..01e68fc 100644 --- a/backend/app/routers/recording_demo.py +++ b/backend/app/routers/recording_demo.py @@ -173,12 +173,12 @@ async def offline_batch( len(result.details), ) - cache_input = result.output_path.parent.parent / "input" / "input.mp4" + cache_input = result.input_path if include_visualization: stage_visualization_pending( runner.root_dir, surgery_id, - source_mp4=cache_input if cache_input.is_file() else result.input_path, + source_mp4=cache_input, result_tsv=result.output_path, ) background_tasks.add_task(_background_finalize_visualization, runner, surgery_id) diff --git a/backend/tests/test_algo_host_batch.py b/backend/tests/test_algo_host_batch.py index 262de26..9c13b86 100644 --- a/backend/tests/test_algo_host_batch.py +++ b/backend/tests/test_algo_host_batch.py @@ -35,6 +35,7 @@ from app.algo_host.transcode import ( browser_transcode_tmp_path, ensure_batch_pipeline_input_video, is_browser_compatible_mp4, + stage_batch_pipeline_input, transcode_visualization_for_browser, ) from app.api import router as api_router @@ -169,6 +170,42 @@ def test_browser_transcode_tmp_path_keeps_mp4_extension() -> None: assert str(tmp).endswith(".mp4") +@pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg not installed") +def test_stage_batch_pipeline_input_copies_without_normalize(tmp_path: Path) -> None: + ffmpeg = shutil.which("ffmpeg") + assert ffmpeg is not None + source = tmp_path / "upload.mp4" + dest = tmp_path / "pipeline.mp4" + proc = subprocess.run( + [ + ffmpeg, + "-y", + "-f", + "lavfi", + "-i", + "testsrc=size=640x360:rate=10", + "-t", + "0.5", + "-c:v", + "mpeg4", + "-pix_fmt", + "yuv420p", + str(source), + ], + check=False, + capture_output=True, + text=True, + ) + assert proc.returncode == 0, proc.stderr + assert batch_input_needs_normalize(source) + stage_batch_pipeline_input(source_path=source, dest_path=dest) + assert dest.is_file() + assert dest.read_bytes() == source.read_bytes() + assert batch_input_needs_normalize(dest) + stage_batch_pipeline_input(source_path=tmp_path / "other.mp4", dest_path=dest) + assert dest.read_bytes() == source.read_bytes() + + @pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg not installed") def test_ensure_batch_pipeline_input_video_normalizes_non_h264(tmp_path: Path) -> None: ffmpeg = shutil.which("ffmpeg") @@ -342,6 +379,10 @@ def test_batch_service_respects_reference_bundle_relative_env( assert runner.bundle_dir == bundle.resolve() assert calls[0][5] == str(bundle.resolve() / "main.py") assert result.details[0].item_name == "耗材1" + config = yaml.safe_load(Path(calls[0][calls[0].index("--config") + 1]).read_text(encoding="utf-8")) + assert Path(config["io"]["video"]).name == "pipeline.mp4" + assert "/cache/" in config["io"]["video"] + assert "/input/" in config["io"]["video"] def test_batch_service_reuses_cache_on_repeat_run( @@ -449,6 +490,8 @@ def test_batch_service_reuses_cache_on_repeat_run( config_path = Path(calls[0][calls[0].index("--config") + 1]) config = yaml.safe_load(config_path.read_text(encoding="utf-8")) assert Path(config["io"]["video"]).is_file() + assert Path(config["io"]["video"]).name == "pipeline.mp4" + assert "/input/" in str(Path(config["io"]["video"])) assert Path(config["io"]["excel"]).is_file() whitelist = json.loads(Path(config["io"]["whitelist_json"]).read_text(encoding="utf-8")) assert whitelist == {"allowed_names": ["耗材1"]} @@ -563,16 +606,16 @@ def test_demo_video_batch_endpoint_writes_queryable_result( assert kwargs["candidate_consumables"] == ["耗材1"] assert kwargs.get("include_visualization") is False cache_dir = root_dir / "cache" / ("a" * 64) / "c1" - cache_input = cache_dir / "input" / "input.mp4" - cache_input.parent.mkdir(parents=True) - cache_input.write_bytes(b"pipeline-input") + pipeline_input = root_dir / "cache" / ("a" * 64) / "input" / "pipeline.mp4" + pipeline_input.parent.mkdir(parents=True) + pipeline_input.write_bytes(b"pipeline-input") output_path = cache_dir / "output" / "result.tsv" output_path.parent.mkdir(parents=True) output_path.write_text(complete_result_tsv_body(), encoding="utf-8") return BatchRunResult( video_sha256="a" * 64, candidate_cache_key="c1", - input_path=root_dir / "100001" / "input" / "saved.mp4", + input_path=pipeline_input, work_dir=cache_dir / "work", output_path=output_path, details=[detail], @@ -630,7 +673,7 @@ def test_demo_video_batch_endpoint_writes_queryable_result( assert body["status"] == "accepted" assert body["visualization_url"] is None assert vis_calls == [] - assert not (root_dir / "cache" / ("a" * 64)).exists() + assert not (root_dir / "cache" / ("a" * 64) / "c1").exists() assert not (root_dir / "100001").exists() got = client.get("/client/surgeries/100001/result") @@ -663,16 +706,16 @@ def test_demo_video_batch_endpoint_stages_vis_and_purges_cache_when_requested( def run(self, **kwargs: Any) -> BatchRunResult: cache_dir = root_dir / "cache" / ("b" * 64) / "c1" - cache_input = cache_dir / "input" / "input.mp4" - cache_input.parent.mkdir(parents=True) - cache_input.write_bytes(b"pipeline-input") + pipeline_input = root_dir / "cache" / ("b" * 64) / "input" / "pipeline.mp4" + pipeline_input.parent.mkdir(parents=True) + pipeline_input.write_bytes(b"pipeline-input") output_path = cache_dir / "output" / "result.tsv" output_path.parent.mkdir(parents=True) output_path.write_text(complete_result_tsv_body(), encoding="utf-8") return BatchRunResult( video_sha256="b" * 64, candidate_cache_key="c1", - input_path=root_dir / "100001" / "input" / "saved.mp4", + input_path=pipeline_input, work_dir=cache_dir / "work", output_path=output_path, details=[detail], @@ -709,7 +752,8 @@ def test_demo_video_batch_endpoint_stages_vis_and_purges_cache_when_requested( body = res.json() assert body["visualization_url"] == "/internal/demo/offline-batch/100001/visualization" assert vis_calls == ["100001"] - assert not (root_dir / "cache" / ("b" * 64)).exists() + assert not (root_dir / "cache" / ("b" * 64) / "c1").exists() + assert (root_dir / "cache" / ("b" * 64) / "input" / "pipeline.mp4").is_file() pending_input = root_dir / "vis_pending" / "100001" / "input.mp4" pending_tsv = root_dir / "vis_pending" / "100001" / "result.tsv" assert pending_input.read_bytes() == b"pipeline-input" diff --git a/backend/tests/test_fastapi_algorithm_subprocess.py b/backend/tests/test_fastapi_algorithm_subprocess.py index 6bd82e9..73aac19 100644 --- a/backend/tests/test_fastapi_algorithm_subprocess.py +++ b/backend/tests/test_fastapi_algorithm_subprocess.py @@ -117,7 +117,8 @@ def test_video_batch_endpoint_invokes_reference_bundle_subprocess( assert kwargs.get("env", {}).get("PYTHONFAULTHANDLER") == "1" config = call["config"] - assert Path(config["io"]["video"]).name == "input.mp4" + assert Path(config["io"]["video"]).name == "pipeline.mp4" + assert "/input/" in config["io"]["video"] assert str(config["io"]["excel"]).endswith("商品信息表.xlsx") assert str(config["io"]["whitelist_json"]).endswith("whitelist.json") assert config["runtime"]["keep_work_dir"] is False diff --git a/backend/tests/test_video_batch_cleanup.py b/backend/tests/test_video_batch_cleanup.py index 110dc89..c2fb10d 100644 --- a/backend/tests/test_video_batch_cleanup.py +++ b/backend/tests/test_video_batch_cleanup.py @@ -37,8 +37,9 @@ def test_purge_batch_artifacts_removes_cache_and_uploads(tmp_path: Path) -> None digest = "d" * 64 candidate_key = "c1" cache_entry = root / "cache" / digest / candidate_key - (cache_entry / "input").mkdir(parents=True) - (cache_entry / "input" / "input.mp4").write_bytes(b"x" * 100) + pipeline_input = root / "cache" / digest / "input" / "pipeline.mp4" + pipeline_input.parent.mkdir(parents=True) + pipeline_input.write_bytes(b"x" * 100) (cache_entry / "output").mkdir(parents=True) (cache_entry / "output" / "result.tsv").write_text("ok\n", encoding="utf-8") (cache_entry / "work").mkdir(parents=True) @@ -53,6 +54,7 @@ def test_purge_batch_artifacts_removes_cache_and_uploads(tmp_path: Path) -> None purge_surgery_batch_tree(root, surgery_id) assert not cache_entry.exists() + assert pipeline_input.is_file() assert not (root / surgery_id).exists() diff --git a/clients/demo-client/app.js b/clients/demo-client/app.js index 8349525..572f179 100644 --- a/clients/demo-client/app.js +++ b/clients/demo-client/app.js @@ -458,6 +458,13 @@ } } + function pickDoctorBannerText(details, fallbackDisplay) { + const rowDoctor = (details[0]?.doctor_id || "").trim(); + const apiDisplay = (fallbackDisplay || "").trim(); + if (apiDisplay && (!rowDoctor || rowDoctor === "vision")) return apiDisplay; + return rowDoctor || apiDisplay; + } + function showVideoBatchDoctorInfo(displayText) { const el = $("video-batch-doctor-info"); if (!el) return; @@ -754,7 +761,7 @@ } const { details = [], summary = [] } = body; if (getRunMode() === "offline-batch") { - showVideoBatchDoctorInfo(details[0]?.doctor_id || lastVideoBatchDoctorDisplay); + showVideoBatchDoctorInfo(pickDoctorBannerText(details, lastVideoBatchDoctorDisplay)); } const renderTable = (title, rows, cols) => { const h = document.createElement("h3"); diff --git a/clients/demo-client/server.py b/clients/demo-client/server.py index 325d0c3..4bba9a3 100755 --- a/clients/demo-client/server.py +++ b/clients/demo-client/server.py @@ -88,6 +88,12 @@ class DemoHandler(SimpleHTTPRequestHandler): def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, directory=str(SCRIPT_DIR), **kwargs) + def end_headers(self) -> None: + path = self.path.split("?", 1)[0] + if path == "/labels.json" or path.endswith((".html", ".js", ".css")): + self.send_header("Cache-Control", "no-store") + super().end_headers() + def do_GET(self) -> None: # noqa: N802 (stdlib override) if self.path.split("?", 1)[0] == "/labels.json": self._send_labels() @@ -99,7 +105,6 @@ class DemoHandler(SimpleHTTPRequestHandler): self.send_response(HTTPStatus.OK) self.send_header("Content-Type", "application/json; charset=utf-8") self.send_header("Content-Length", str(len(body))) - self.send_header("Cache-Control", "no-store") self.end_headers() self.wfile.write(body) diff --git a/docs/Docker部署.md b/docs/Docker部署.md index d530222..7f1462e 100644 --- a/docs/Docker部署.md +++ b/docs/Docker部署.md @@ -27,6 +27,7 @@ operation-room-monitor/ - 复制 `backend/.env.example` 为 `backend/.env` 并填写 - 算法子进程包：`backend/algorithm_subprocesses/5.15/`（含 `main.py` 与 `weights/`；镜像构建时会 `COPY` 进容器，勿在 `.dockerignore` 中整目录排除） - 标注视频中文字体：镜像内已安装 `fonts-noto-cjk`、`fonts-wqy-microhei`（供 `visualize_result_video.py` 绘制耗材标签） +- 医生识别（MediaPipe Pose）：镜像内已安装 `libgles2`、`libegl1`；子进程强制 CPU delegate，避免无 GPU 图形栈时出现 `libGLESv2.so.2` 错误 - 可选备用权重：`backend/app/resources/actionformer_epoch_045.pth.tar` ---