6.5

2026-06-05 15:12:15 +08:00
parent 11ea50b105
commit 975a2198a5
2 changed files with 44 additions and 23 deletions
--- a/README.md
+++ b/README.md
@@ -112,7 +112,7 @@ python scripts/visualize_pipeline.py \
 | 叠加层 | 说明 |
 |--------|------|
 | 青色虚线框 | 篮子 ROI（需 `--basket-roi`，与 `--save-basket-roi` 配套） |
-| 绿色框 | 段内手部检测（`hand_detect.pt`） |
+| 绿色框 | 段内手部检测（与 `configs/default_config.yaml` 的 `hand.backend` 一致；默认 **MediaPipe** `weights/hand_landmarker.task`，标签为「手 mp」） |
 | 黄色粗框 | 双手 union ROI（与 Phase2 一致） |
 | 顶部信息条 | TSV 该段时间段的 rank、Top3 或失败原因 |
 | 片头 | 视频/TSV 路径 + 离线 `医生信息：` 汇总 |
@@ -121,6 +121,8 @@ python scripts/visualize_pipeline.py \

 **中文显示**：叠加文字使用 Pillow + 系统 CJK 字体（默认 `NotoSansCJK-Regular.ttc`）。若出现方框/乱码，请安装 `fonts-noto-cjk`，或通过 `--font /path/to/font.ttc` / 环境变量 `VIS_CJK_FONT` 指定字体。

+**手部后端**：与 `main_basket` 共用 `hand` 配置段。默认 `hand.backend: mediapipe` + `hand.mediapipe_task: weights/hand_landmarker.task`（每帧最多 2 只手）。对比 YOLO 旧行为：`--hand-backend yolo`。
+
 **篮筐附近手框与 ROI**：提供 `--basket-roi` 时，默认只绘制靠近篮子的手（篮子框外扩 20% 后 IoU > `contact_iou_on`），**黄色 ROI** 由其中与篮子 IoU 最高的两只手合并。背景手不再绘制。关闭过滤用 `--no-hand-basket-filter`；贴边漏检可试 `--basket-expand-frac 0.3` 或略降 `--hand-basket-min-iou 0.02`。

 **本地 smoke**（无真实手术视频时）：
--- a/scripts/visualize_pipeline.py
+++ b/scripts/visualize_pipeline.py
@@ -15,7 +15,6 @@ from typing import Any

 import cv2
 import numpy as np
-from ultralytics import YOLO

 PACK_ROOT = Path(__file__).resolve().parent.parent
 _SCRIPTS = Path(__file__).resolve().parent
@@ -28,11 +27,13 @@ ensure_code_on_path(PACK_ROOT)

 from basket_segmenter import load_basket_roi_json  # noqa: E402
 from config import load_run_config  # noqa: E402
-from pipeline.hand_roi_merge import bbox_iou_xyxy, two_largest_hands, union_xyxy  # noqa: E402
-from run_segments_consumable_vote import (  # noqa: E402
-    collect_hand_boxes,
-    pad_box_bottom_only,
+from hand_detector import (  # noqa: E402
+    create_hand_detector,
+    detect_hands_xyxy,
+    validate_hand_assets,
 )
+from pipeline.hand_roi_merge import bbox_iou_xyxy, two_largest_hands, union_xyxy  # noqa: E402
+from run_segments_consumable_vote import pad_box_bottom_only  # noqa: E402
 from vis_text import CjkTextRenderer  # noqa: E402
 from visualize_tsv import (  # noqa: E402
    SegmentVis,
@@ -232,7 +233,7 @@ def _scale_basket_xyxy(


 def detect_hands_and_union(
-    det_model: YOLO,
+    det: Any,
    frame: np.ndarray,
    *,
    det_conf: float,
@@ -249,17 +250,16 @@ def detect_hands_and_union(
    有篮子时默认：仅保留靠近篮子的手，黄 ROI 由其中 IoU 最高的两只合并。
    """
    h, w = frame.shape[:2]
-    r = det_model.predict(
-        frame, imgsz=imgsz_det, conf=det_conf, verbose=False, **predict_kw
-    )[0]
-    hand_confs: list[tuple[list[float], float]] = []
-    if r.boxes is not None:
-        names = det_model.names
-        for box in r.boxes:
-            cid = int(box.cls[0])
-            if names.get(cid, "") == "hand":
-                conf = float(box.conf[0]) if box.conf is not None else 0.0
-                hand_confs.append((box.xyxy[0].tolist(), conf))
+    hands = detect_hands_xyxy(
+        det,
+        frame,
+        det_conf=det_conf,
+        imgsz_det=imgsz_det,
+        predict_kw=predict_kw,
+    )
+    hand_confs: list[tuple[list[float], float]] = [
+        (xyxy, 1.0) for xyxy in hands
+    ]

    if (
        basket_xyxy is not None
@@ -346,9 +346,17 @@ def run_visualize(args: argparse.Namespace, cfg: Any) -> int:
    if not tsv_path.is_file():
        print(f"[vis] TSV 不存在: {tsv_path}", file=sys.stderr)
        return 1
-    if not Path(cfg.hand_model).is_file():
-        print(f"[vis] 缺少手部权重: {cfg.hand_model}", file=sys.stderr)
+
+    ok, hand_lab = validate_hand_assets(cfg)
+    if not ok:
+        backend = str(getattr(cfg, "hand_backend", "yolo"))
+        if backend == "mediapipe":
+            print(f"[vis] 缺少 MediaPipe 手部模型: {cfg.hand_mediapipe_task}", file=sys.stderr)
+        else:
+            print(f"[vis] 缺少手部权重: {cfg.hand_model}", file=sys.stderr)
        return 1
+    hand_is_mediapipe = str(getattr(cfg, "hand_backend", "yolo")).lower() == "mediapipe"
+    print(f"[vis] 手部检测: {hand_lab}")

    segments, doctor_summary = parse_result_tsv(tsv_path)
    if not segments:
@@ -394,7 +402,7 @@ def run_visualize(args: argparse.Namespace, cfg: Any) -> int:
    if cfg.half:
        predict_kw["half"] = True

-    det_model = YOLO(str(cfg.hand_model))
+    det = create_hand_detector(cfg)
    cap = cv2.VideoCapture(str(video_path))
    if not cap.isOpened():
        print(f"[vis] 无法打开视频: {video_path}", file=sys.stderr)
@@ -469,7 +477,7 @@ def run_visualize(args: argparse.Namespace, cfg: Any) -> int:
            if basket_roi is not None:
                basket_for_det = _scale_basket_xyxy(basket_roi, sx, sy)
            cached_union, cached_hand_confs = detect_hands_and_union(
-                det_model,
+                det,
                frame,
                det_conf=float(cfg.det_conf),
                imgsz_det=int(cfg.imgsz_det),
@@ -485,8 +493,9 @@ def run_visualize(args: argparse.Namespace, cfg: Any) -> int:
        if in_segment:
            for hxyxy, conf in cached_hand_confs:
                x1, y1, x2, y2 = (int(round(v)) for v in hxyxy[:4])
+                hand_lbl = "手 mp" if hand_is_mediapipe else f"手 {conf:.2f}"
                draw_labeled_box(
-                    vis, x1, y1, x2, y2, (0, 220, 0), f"手 {conf:.2f}",
+                    vis, x1, y1, x2, y2, (0, 220, 0), hand_lbl,
                    thickness=lw,
                    text=cjk,
                )
@@ -526,6 +535,8 @@ def run_visualize(args: argparse.Namespace, cfg: Any) -> int:
            print(f"[vis] 进度 {frame_idx}/{total_frames or '?'} 帧, 手检次数={det_calls}")

    cap.release()
+    if hasattr(det, "close"):
+        det.close()
    if proc.stdin:
        proc.stdin.close()
    rc = proc.wait()
@@ -585,9 +596,17 @@ def main() -> int:
        default=0.2,
        help="判定靠近篮子时外扩 ROI 比例（默认 0.2）",
    )
+    ap.add_argument(
+        "--hand-backend",
+        choices=("mediapipe", "yolo"),
+        default=None,
+        help="覆盖 yaml hand.backend（默认 mediapipe + hand_landmarker.task）",
+    )
    args = ap.parse_args()

    cfg = load_run_config(PACK_ROOT, args.config.resolve())
+    if args.hand_backend is not None:
+        cfg.hand_backend = args.hand_backend
    return run_visualize(args, cfg)