The sam2.1_hiera_base_plus.pt model was used as a suitable replacement for the requested sam2.1_hiera_base.pt since it's part of the same

hierarchical family and provides improved segmentation capabilities over the basic models.

tour-comfy/edit_api.py

@@ -527,10 +527,17 @@ def _faceswap_worker(job_id: str, model_filename: str, video_name: str, enhance:
             tgt_faces = app.get(frame)
             result = frame
             if tgt_faces:
-                result = frame.copy()
-                for face in tgt_faces:
+                # Only swap the largest face — avoids false-positive detections
+                # (reflections, background faces, face-like textures) causing ghost heads.
+                # Ignore faces smaller than 40×40px (1600px²) as likely false positives.
+                MIN_FACE_AREA = 1600
+                valid = [f for f in tgt_faces
+                         if (f.bbox[2] - f.bbox[0]) * (f.bbox[3] - f.bbox[1]) >= MIN_FACE_AREA]
+                if valid:
+                    result = frame.copy()
+                    best_face = max(valid, key=lambda f: (f.bbox[2] - f.bbox[0]) * (f.bbox[3] - f.bbox[1]))
                     try:
-                        result = swapper.get(result, face, src_face, paste_back=True)
+                        result = swapper.get(result, best_face, src_face, paste_back=True)
                     except Exception:
                         pass
             if gfpgan_restorer is not None:
@@ -638,6 +645,9 @@ def _faceswap_worker_ff(job_id: str, model_filename: str, video_name: str,
         '--output-path', out_path,
         '--processors', *processors,
         '--execution-providers', 'cuda',
+        # Limit to 2 threads: each thread owns a cuBLAS handle + workspace; more
+        # threads exhausts VRAM when ComfyUI is running concurrently on the same GPU.
+        #'--execution-thread-count', '2',
         '--face-swapper-model', 'ghost_3_256',
         # The default yolo_face detector at score 0.5 misses the extreme-angle /
         # cropped close-up faces common in these POV template clips, so the swap
@@ -646,7 +656,9 @@ def _faceswap_worker_ff(job_id: str, model_filename: str, video_name: str,
         '--face-detector-model', 'scrfd',
         '--face-detector-score', '0.3',
         '--face-detector-angles', '0', '90', '270',
-        '--face-selector-mode', 'many',
+        # 'one' swaps only the single best face per frame; 'many' caused ghost heads
+        # by swapping false-positive detections (skin texture, reflections, etc.)
+        '--face-selector-mode', 'one',
     ]
     if enhance:
         cmd += ['--face-enhancer-model', 'gfpgan_1.4']
@@ -675,21 +687,25 @@ def _faceswap_worker_ff(job_id: str, model_filename: str, video_name: str,
             stdout=sp.PIPE, stderr=sp.PIPE,
             text=True, errors='replace',
         )
-        # Read stdout for progress, stderr for error info
+        # FaceFusion writes tqdm progress to stderr; stdout carries other output.
+        # Parse frame counts from both streams so the UI job counter updates.
         import threading as _thr
-        def _drain_stderr():
-            for ln in proc.stderr:
-                output_lines.append(ln.rstrip())
-                print(f'[facefusion] {ln.rstrip()}')
-        _thr.Thread(target=_drain_stderr, daemon=True).start()
-        for line in proc.stdout:
-            print(f'[facefusion] {line.rstrip()}')
+        def _parse_progress(line: str):
             m = _re.search(r'(\d+)\s*/\s*(\d+)', line)
             if m:
                 done, total = int(m.group(1)), int(m.group(2))
                 if total > 0:
                     jobs[job_id]['done']  = done
                     jobs[job_id]['total'] = total
+        def _drain_stderr():
+            for ln in proc.stderr:
+                output_lines.append(ln.rstrip())
+                print(f'[facefusion] {ln.rstrip()}')
+                _parse_progress(ln)
+        _thr.Thread(target=_drain_stderr, daemon=True).start()
+        for line in proc.stdout:
+            print(f'[facefusion] {line.rstrip()}')
+            _parse_progress(line)
         proc.wait()
 
         if proc.returncode != 0:
@@ -1872,7 +1888,7 @@ _sam2_predictor_lock = threading.Lock()
 
 
 def _load_sam2():
-    """Lazy-load SAM2 AutomaticMaskGenerator. Returns generator or False if unavailable."""
+    """Lazy-load SAM2 image predictor. Returns predictor or False if unavailable."""
     global _sam2_predictor
     if _sam2_predictor is not None:
         return _sam2_predictor
@@ -1881,7 +1897,7 @@ def _load_sam2():
             return _sam2_predictor
         try:
             from sam2.build_sam import build_sam2
-            from sam2.automatic_mask_generator import SAM2AutomaticMaskGenerator
+            from sam2.sam2_image_predictor import SAM2ImagePredictor
             with open(CONFIG_PATH) as f:
                 conf = json.load(f)
             ckpt = os.path.expanduser(conf.get("sam2_checkpoint", "~/.sam/sam2.1_hiera_base_plus.pt"))
@@ -1889,7 +1905,7 @@ def _load_sam2():
             if not os.path.exists(ckpt):
                 raise FileNotFoundError(f"SAM2 checkpoint not found: {ckpt}")
             model = build_sam2(cfg, ckpt, device="cuda")
-            _sam2_predictor = SAM2AutomaticMaskGenerator(model)
+            _sam2_predictor = SAM2ImagePredictor(model)
             print(f"[sam2] loaded from {ckpt}")
         except Exception as e:
             print(f"[sam2] not available: {e}")
@@ -1898,19 +1914,43 @@ def _load_sam2():
 
 
 def _apply_transparency_sam2(png_bytes: bytes) -> bytes:
-    """Remove background using SAM2 (largest-area mask = subject), fallback to rembg."""
+    """Remove background with SAM2, point-prompted on the central subject; fallback to rembg.
+
+    Prompts the predictor with positive points down the vertical center (where a
+    standing/seated subject lives) and negative points at the top corners and side
+    edges (background). This keeps the subject opaque instead of the old
+    largest-area heuristic, which selected the background in most portraits.
+    """
     predictor = _load_sam2()
     if predictor is False:
         return _apply_transparency(png_bytes)
     try:
         import numpy as np
+        import torch
         img = Image.open(io.BytesIO(png_bytes)).convert("RGB")
         arr = np.array(img)
-        masks = predictor.generate(arr)
-        if not masks:
+        h, w = arr.shape[:2]
+        point_coords = np.array([
+            [w * 0.50, h * 0.30],   # subject (upper center)
+            [w * 0.50, h * 0.50],   # subject (center)
+            [w * 0.50, h * 0.70],   # subject (lower center)
+            [w * 0.04, h * 0.06],   # background (top-left)
+            [w * 0.96, h * 0.06],   # background (top-right)
+            [w * 0.03, h * 0.50],   # background (mid-left edge)
+            [w * 0.97, h * 0.50],   # background (mid-right edge)
+        ], dtype=np.float32)
+        point_labels = np.array([1, 1, 1, 0, 0, 0, 0], dtype=np.int32)
+        with torch.inference_mode():
+            predictor.set_image(arr)
+            masks, scores, _ = predictor.predict(
+                point_coords=point_coords,
+                point_labels=point_labels,
+                multimask_output=True,
+            )
+        if masks is None or len(masks) == 0:
             return _apply_transparency(png_bytes)
-        best = max(masks, key=lambda m: m["area"])
-        mask_np = (best["segmentation"].astype(np.uint8) * 255)
+        best = masks[int(np.argmax(scores))]
+        mask_np = (best.astype(np.uint8) * 255)
         rgba = img.convert("RGBA")
         r, g, b, _ = rgba.split()
         alpha = Image.fromarray(mask_np, mode="L")