aa

optimize_clips.py

@@ -0,0 +1,152 @@
+import os
+import glob
+import shutil
+import subprocess
+import tempfile
+import threading
+import time
+CLIPS_DIR = "/data/events/clips"
+ARCHIVE_DIR = "/mnt/tour-big/clips"
+SIZE_THRESHOLD_MB = 200
+SIZE_THRESHOLD_BYTES = SIZE_THRESHOLD_MB * 1024 * 1024
+MAX_PER_RUN = 20
+
+
+def has_nvenc():
+    r = subprocess.run(["ffmpeg", "-hide_banner", "-encoders"],
+                       stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
+    return b"hevc_nvenc" in r.stdout
+
+
+def _ffmpeg(cmd):
+    return subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
+
+
+def _proxy_cmd(src, dst, hw):
+    """10fps H.265 proxy for quick local browsing."""
+    if hw:
+        return ["ffmpeg", "-y",
+                "-hwaccel", "cuda", "-hwaccel_output_format", "cuda",
+                "-i", src,
+                "-vcodec", "hevc_nvenc", "-rc", "constqp", "-qp", "28", "-preset", "p4",
+                "-r", "10", "-tag:v", "hvc1", "-acodec", "copy", dst]
+    return ["ffmpeg", "-y", "-i", src,
+            "-vcodec", "libx265", "-crf", "28", "-preset", "medium",
+            "-r", "10", "-tag:v", "hvc1", "-acodec", "copy", dst]
+
+
+def _archive_cmd(src, dst, hw):
+    """Full-fps high-quality H.265 for archive — small enough to pull back fast over slow SMB."""
+    if hw:
+        return ["ffmpeg", "-y",
+                "-hwaccel", "cuda", "-hwaccel_output_format", "cuda",
+                "-i", src,
+                "-vcodec", "hevc_nvenc", "-rc", "vbr", "-cq", "19", "-b:v", "0",
+                "-preset", "p6", "-tune", "hq",
+                "-tag:v", "hvc1", "-acodec", "copy", dst]
+    return ["ffmpeg", "-y", "-i", src,
+            "-vcodec", "libx265", "-crf", "19", "-preset", "slow",
+            "-tag:v", "hvc1", "-acodec", "copy", dst]
+
+
+def _encode_worker(label, cmd, hw, fallback_cmd, out):
+    proc = _ffmpeg(cmd)
+    if proc.returncode != 0 and hw:
+        print(f"  [{label}] NVENC failed, retrying in software...")
+        proc = _ffmpeg(fallback_cmd)
+    out[label] = proc
+
+
+def process(src, orig_size, hw):
+    name = os.path.basename(src)
+    archive_dst = os.path.join(ARCHIVE_DIR, name)
+
+    if os.path.exists(archive_dst):
+        print(f"  Skip {name} — already archived")
+        return 0
+
+    proxy_tmp   = src + ".proxy.mp4"
+    # Encode archive to local /tmp first — never write directly to a slow SMB share
+    archive_tmp = os.path.join(tempfile.gettempdir(), name + ".archive.mp4")
+
+    print(f"[{name}]  {orig_size/(1024*1024):.0f} MB  ({'NVENC' if hw else 'software'})")
+    t0 = time.time()
+
+    results = {}
+    threads = [
+        threading.Thread(target=_encode_worker, args=(
+            "proxy",
+            _proxy_cmd(src, proxy_tmp, hw), hw,
+            _proxy_cmd(src, proxy_tmp, False),
+            results,
+        )),
+        threading.Thread(target=_encode_worker, args=(
+            "archive",
+            _archive_cmd(src, archive_tmp, hw), hw,
+            _archive_cmd(src, archive_tmp, False),
+            results,
+        )),
+    ]
+    for t in threads: t.start()
+    for t in threads: t.join()
+
+    failed = {k: v for k, v in results.items() if v.returncode != 0}
+    if failed:
+        for label, proc in failed.items():
+            print(f"  [{label}] error: {proc.stderr.decode()[-300:]}")
+        for f in (proxy_tmp, archive_tmp):
+            if os.path.exists(f): os.remove(f)
+        return 0
+
+    proxy_mb   = os.path.getsize(proxy_tmp)   / (1024*1024)
+    archive_mb = os.path.getsize(archive_tmp) / (1024*1024)
+    orig_mb    = orig_size / (1024*1024)
+    print(f"  proxy {orig_mb:.0f}->{proxy_mb:.0f} MB  "
+          f"archive {orig_mb:.0f}->{archive_mb:.0f} MB  "
+          f"({time.time()-t0:.1f}s) — pushing archive to share...")
+
+    try:
+        shutil.move(archive_tmp, archive_dst)
+    except Exception as e:
+        print(f"  Archive move failed: {e} — aborting, original intact")
+        for f in (proxy_tmp, archive_tmp):
+            if os.path.exists(f): os.remove(f)
+        return 0
+
+    # Atomic swap: original -> proxy (same filesystem)
+    os.replace(proxy_tmp, src)
+    freed = orig_size - os.path.getsize(src)
+    print(f"  Done  freed locally: {freed/(1024*1024):.0f} MB")
+    return freed
+
+
+if __name__ == "__main__":
+    hw = has_nvenc()
+    print(f"Encoder:  {'hevc_nvenc (A6000, 2x parallel)' if hw else 'libx265 (software)'}")
+    print(f"Clips:    {CLIPS_DIR}")
+    print(f"Archive:  {ARCHIVE_DIR}\n")
+
+    if not os.path.isdir(ARCHIVE_DIR):
+        print(f"ERROR: archive not reachable: {ARCHIVE_DIR}")
+        raise SystemExit(1)
+
+    files = sorted(
+        [(fp, os.path.getsize(fp))
+         for fp in glob.glob(os.path.join(CLIPS_DIR, "*.mp4"))
+         if os.path.isfile(fp) and os.path.getsize(fp) > SIZE_THRESHOLD_BYTES],
+        key=lambda x: x[1], reverse=True,
+    )
+    print(f"Found {len(files)} files > {SIZE_THRESHOLD_MB} MB\n")
+
+    total_freed = 0
+    done = 0
+    for path, size in files:
+        freed = process(path, size, hw)
+        total_freed += freed
+        if freed:
+            done += 1
+        if done >= MAX_PER_RUN:
+            print(f"Reached {MAX_PER_RUN}-file limit.")
+            break
+
+    print(f"\nDone.  Processed: {done}  Freed locally: {total_freed/(1024*1024):.0f} MB")