680 lines
31 KiB
Python
680 lines
31 KiB
Python
"""
|
|
orbit_qwen.py — near-real actor turntable using Qwen-Image-Edit.
|
|
|
|
Unlike orbit_module.py (fake 2.5D depth-card parallax), this actually asks the
|
|
generative model to RE-RENDER the subject at each yaw angle. Each view is
|
|
anchored to the original front image with a fixed seed so identity, body, hair
|
|
and lighting stay consistent while only the viewpoint rotates.
|
|
|
|
Pipeline:
|
|
1. build a yaw-angle prompt per frame (turntable or swing)
|
|
2. _run_pipeline (Qwen via ComfyUI) → one re-rendered view per angle
|
|
3. bottom-center align onto a common canvas
|
|
4. stitch to a looping MP4
|
|
|
|
Validated finding (2026-06-25): 2D blending between independently-generated
|
|
views (optical-flow morph OR crossfade) always ghosts — the bodies don't
|
|
overlap, so any in-between frame shows a double exposure. The cure is DENSITY,
|
|
not blending: ~24 crisp keyframes (15° steps) played with NO interpolation at
|
|
~12fps reads as a smooth turntable, exactly like classic 3D turntable GIFs.
|
|
Interpolation is kept available (interp_factor>1) but defaults OFF.
|
|
|
|
Reuses edit_api._run_pipeline, so it talks to the same running ComfyUI server.
|
|
|
|
Usage:
|
|
from orbit_qwen import run_qwen_orbit
|
|
result = run_qwen_orbit("/path/to/front.png", "/out/dir", n_views=12)
|
|
|
|
CLI: see orbit_qwen_poc.py
|
|
"""
|
|
|
|
import os
|
|
import io
|
|
import sys
|
|
import math
|
|
import subprocess
|
|
import tempfile
|
|
|
|
import cv2
|
|
import numpy as np
|
|
from PIL import Image
|
|
|
|
# Reuse the real Qwen pipeline from the API service (no server round-trip needed;
|
|
# _run_pipeline queues directly to ComfyUI). Import is cheap — only loads the
|
|
# workflow JSON; models load lazily and the uvicorn startup hook does not fire.
|
|
_HERE = os.path.dirname(os.path.abspath(__file__))
|
|
if _HERE not in sys.path:
|
|
sys.path.insert(0, _HERE)
|
|
|
|
from edit_api import _run_pipeline, _load_output_dir, MAX_AREA # noqa: E402
|
|
|
|
__all__ = [
|
|
"is_front_view",
|
|
"is_face_visible",
|
|
"yaw_prompt",
|
|
"generate_views",
|
|
"interpolate_views",
|
|
"build_video",
|
|
"run_qwen_orbit",
|
|
]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 1. Prompt construction
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def is_front_view(pil_image: Image.Image) -> bool:
|
|
"""Detect if the image is a clear front view where nose and both eyes or ears are visible."""
|
|
try:
|
|
from edit_api import _load_pose_estimator
|
|
estimator = _load_pose_estimator()
|
|
if not estimator:
|
|
return True
|
|
infer_fn, _ = estimator
|
|
people = infer_fn(pil_image)
|
|
if not people:
|
|
return True
|
|
kpts = people[0]
|
|
# kpts format: 17 joints, each is [x, y, score]
|
|
# 0: nose, 1: left_eye, 2: right_eye, 3: left_ear, 4: right_ear
|
|
nose_score = kpts[0][2]
|
|
l_eye_score = kpts[1][2]
|
|
r_eye_score = kpts[2][2]
|
|
l_ear_score = kpts[3][2]
|
|
r_ear_score = kpts[4][2]
|
|
|
|
# Symmetrical front view detection:
|
|
if nose_score > 0.4:
|
|
if l_eye_score > 0.4 and r_eye_score > 0.4:
|
|
return True
|
|
if l_ear_score > 0.4 and r_ear_score > 0.4:
|
|
return True
|
|
return False
|
|
except Exception as e:
|
|
print(f"[orbit-qwen] is_front_view check failed: {e}. Defaulting to True.")
|
|
return True
|
|
|
|
|
|
def is_face_visible(deg: float) -> bool:
|
|
"""True if face/nose is visible at this yaw angle, False for rear views."""
|
|
d = deg % 360
|
|
return d <= 97.5 or d >= 262.5
|
|
|
|
|
|
# Identity lock appended to every angle — keeps face/body/hair consistent across views.
|
|
# For front/side views where the face is visible:
|
|
_IDENTITY_FRONT = (
|
|
"same person, identical face, identical hair style and color, identical body shape and proportions, "
|
|
"same skin tone, same clothing, same lighting, photorealistic, sharp focus, "
|
|
"full body visible head to toe, centered, transparent background "
|
|
)
|
|
|
|
# For rear/back views where the face is hidden (omits "face" keyword to avoid contradiction/hallucination):
|
|
_IDENTITY_BACK = (
|
|
"same person, identical hair style and color from behind, identical body shape and proportions, "
|
|
"same skin tone, same clothing, same lighting, photorealistic, sharp focus, "
|
|
"full body visible head to toe from behind, centered, transparent background "
|
|
)
|
|
|
|
|
|
def _angle_phrase(deg: float) -> str:
|
|
"""
|
|
24 distinct buckets, each 15° wide, boundaries at 7.5°/22.5°/37.5°…352.5°.
|
|
Works correctly for n_views=12 (30° steps) AND n_views=24 (15° steps).
|
|
|
|
Convention (confirmed by test):
|
|
• 90° → face/nose points LEFT in the output image (camera to subject's right).
|
|
• 270° → face/nose points RIGHT in the output image (camera to subject's left).
|
|
• Rear views: viewed from behind, anatomical right appears on image LEFT.
|
|
• Profile and rear-view anchors use explicit image-coordinate phrases to
|
|
prevent Qwen from swapping sides.
|
|
"""
|
|
d = deg % 360
|
|
|
|
# ── front ──────────────────────────────────────────────────────────────────
|
|
if d < 7.5 or d >= 352.5: # 0° — full front
|
|
return (
|
|
"showing her full front directly toward the camera: "
|
|
"her face, both breasts, navel, and the fronts of both legs are fully visible, "
|
|
"her back is completely hidden"
|
|
)
|
|
|
|
# ── right-front quadrant ───────────────────────────────────────────────────
|
|
elif d < 22.5: # 15° — barely perceptible right-front tilt
|
|
return (
|
|
"facing almost directly toward the camera — just the subtlest hint of a right-front turn. "
|
|
"Both eyes and her full face are visible. "
|
|
"In the output image her face is nearly perfectly centered, "
|
|
"with only the tiniest tilt toward the LEFT edge. "
|
|
"Her left shoulder is just a hair closer to the camera than her right. "
|
|
"This looks almost identical to a pure front view"
|
|
)
|
|
elif d < 37.5: # 30° — slight right-front turn
|
|
return (
|
|
"turned slightly to her right — a subtle right-front view. "
|
|
"Both eyes visible, face still mostly toward the camera. "
|
|
"In the output image her face is nearly centered but noticeably shifted toward the LEFT side. "
|
|
"Her left shoulder is clearly closer to the camera than her right"
|
|
)
|
|
elif d < 52.5: # 45° — gentle three-quarter right-front
|
|
return (
|
|
"turned about 45° to her right. "
|
|
"Her face is partly toward the camera, left cheek and jaw more visible than right. "
|
|
"In the output image her face appears on the LEFT half, nose angled toward the left edge. "
|
|
"Her left shoulder, left breast and left hip are angled toward the camera. "
|
|
"Her right side is starting to turn away"
|
|
)
|
|
elif d < 67.5: # 60° — clear three-quarter right-front
|
|
return (
|
|
"turned so the camera sees a clear three-quarter right-front view. "
|
|
"In the output image her face is partially visible on the LEFT side, nose pointing left. "
|
|
"Her left breast, left shoulder and left hip are angled toward the camera. "
|
|
"Her right breast, right hip and right side are turned away from camera"
|
|
)
|
|
elif d < 82.5: # 75° — strong right-front, almost profile
|
|
return (
|
|
"turned strongly to her right — almost a pure side profile, but the face is still slightly visible. "
|
|
"In the output image her face is on the LEFT side with nose pointing toward the left edge. "
|
|
"Her left ear, left cheek and left shoulder are the main visible features. "
|
|
"Her right breast and right side are mostly hidden"
|
|
)
|
|
|
|
# ── right profile ──────────────────────────────────────────────────────────
|
|
elif d < 97.5: # 90° — pure right profile
|
|
return (
|
|
"in a pure side profile. "
|
|
"IMPORTANT: In the output image her nose and face point toward the LEFT edge of the frame — "
|
|
"she is NOT facing right. "
|
|
"Her chest and front of her body are on the LEFT side of the image; "
|
|
"her back (spine, shoulder blade) is on the RIGHT side of the image. "
|
|
"Her left side is facing the camera, and her right side is completely hidden behind her body"
|
|
)
|
|
|
|
# ── right-rear quadrant ────────────────────────────────────────────────────
|
|
elif d < 112.5: # 105° — just past right profile, back turning
|
|
return (
|
|
"turned just past a pure right-side profile — she is starting to show her back. "
|
|
"THIS IS A BACK-TURNING VIEW: her back is starting to face the camera. "
|
|
"Her spine is on the RIGHT side of the image. "
|
|
"Her left shoulder blade (on the left half of the image) is becoming more visible. "
|
|
"Her face is almost completely hidden — only the very edge of her profile is barely visible on the far left edge of the image. "
|
|
"Her spine and left shoulder blade are the main features. Her right side is hidden"
|
|
)
|
|
elif d < 127.5: # 120° — three-quarter rear-right
|
|
return (
|
|
"THIS IS A BACK VIEW — her back faces the camera. "
|
|
"Three-quarter rear-right: her left shoulder blade and left hip (on the left half of the image) are most prominent. "
|
|
"Her spine is on the RIGHT half of the image. "
|
|
"In the output image her left shoulder blade appears on the LEFT half of the image, "
|
|
"with her back turning towards the camera. "
|
|
"Her face is completely hidden. No breasts visible"
|
|
)
|
|
elif d < 142.5: # 135° — rear-right, heading toward full back
|
|
return (
|
|
"THIS IS A BACK VIEW — her back faces the camera. "
|
|
"Rear-right view, closer to a full back than to a side profile. "
|
|
"Her spine is on the RIGHT half of the image. "
|
|
"Her left shoulder blade is somewhat LEFT of center in the image. "
|
|
"Her right shoulder blade is also visible but less prominent. "
|
|
"Face completely hidden. Buttocks and backs of legs visible"
|
|
)
|
|
elif d < 157.5: # 150° — mostly back, subtle right lean
|
|
return (
|
|
"THIS IS A BACK VIEW — her back faces the camera. "
|
|
"Nearly a full back view with a very subtle lean. "
|
|
"Her spine is slightly to the RIGHT of center in the image. "
|
|
# "Both shoulder blades are visible, with her left shoulder blade slightly more prominent. "
|
|
"Both shoulder blades are visible. "
|
|
"Face completely hidden"
|
|
)
|
|
elif d < 172.5: # 165° — almost full back (right side)
|
|
return (
|
|
"THIS IS A BACK VIEW — almost exactly a full back view, the tiniest lean from the right. "
|
|
"Her spine is just barely to the RIGHT of center in the image. "
|
|
"Both shoulder blades, buttocks and backs of both legs are visible. "
|
|
# "Her left shoulder blade is just barely more prominent. Face completely hidden"
|
|
"Face completely hidden"
|
|
)
|
|
|
|
# ── full back ──────────────────────────────────────────────────────────────
|
|
elif d < 187.5: # 180° — pure full back
|
|
return (
|
|
"showing her full back to the camera: "
|
|
"the back of her head, her spine, both shoulder blades equally, "
|
|
"her buttocks, and the backs of both legs are fully visible. "
|
|
"Her face and both breasts are completely hidden"
|
|
)
|
|
|
|
# ── left-rear quadrant ─────────────────────────────────────────────────────
|
|
elif d < 202.5: # 195° — almost full back (left side)
|
|
return (
|
|
"THIS IS A BACK VIEW — almost exactly a full back view, the tiniest lean from the left. "
|
|
"Her spine is just barely to the LEFT of center in the image. "
|
|
"Both shoulder blades, buttocks and backs of both legs are visible. "
|
|
"Her right shoulder blade is just barely more prominent. Face completely hidden"
|
|
)
|
|
elif d < 217.5: # 210° — mostly back, subtle left lean
|
|
return (
|
|
"THIS IS A BACK VIEW — her back faces the camera. "
|
|
"Nearly a full back view with a very subtle lean from the left side. "
|
|
"Her spine is slightly to the LEFT of center in the image. "
|
|
# "Both shoulder blades are visible, with her right shoulder blade slightly more prominent. "
|
|
"Both shoulder blades are visible. "
|
|
"Face completely hidden"
|
|
)
|
|
elif d < 232.5: # 225° — rear-left, heading toward full back
|
|
return (
|
|
"THIS IS A BACK VIEW — her back faces the camera. "
|
|
"Rear-left view, closer to a full back than to a side profile. "
|
|
"Her spine is on the LEFT half of the image. "
|
|
"Her right shoulder blade is somewhat RIGHT of center in the image. "
|
|
"Her left shoulder blade is also visible but less prominent. "
|
|
"Face completely hidden. Buttocks and backs of legs visible"
|
|
)
|
|
elif d < 247.5: # 240° — three-quarter rear-left
|
|
return (
|
|
"THIS IS A BACK VIEW — her back faces the camera. "
|
|
# "Three-quarter rear-left: her right shoulder blade and right hip (on the right half of the image) are most prominent. "
|
|
"Three-quarter rear-left: her right hip (on the right half of the image) are most prominent. "
|
|
"Her spine is on the LEFT half of the image. "
|
|
"In the output image her right shoulder blade appears on the RIGHT half of the image, "
|
|
"with her back turning towards the camera. "
|
|
"Her face is completely hidden. No breasts visible"
|
|
)
|
|
elif d < 262.5: # 255° — just past left profile, back turning
|
|
return (
|
|
"turned just past a pure left-side profile — she is starting to show her back. "
|
|
"THIS IS A BACK-TURNING VIEW: her back is starting to face the camera. "
|
|
"Her spine is on the LEFT side of the image. "
|
|
"Her right shoulder blade is becoming visible. "
|
|
"Her face is almost completely hidden — only the very edge of her profile is barely visible on the far right edge of the image. "
|
|
"Her spine and right shoulder blade are the main features. Her left side is hidden"
|
|
)
|
|
|
|
# ── left profile ───────────────────────────────────────────────────────────
|
|
elif d < 277.5: # 270° — pure left profile
|
|
return (
|
|
"in a pure side profile. "
|
|
"IMPORTANT: In the output image her nose and face point toward the RIGHT edge of the frame — "
|
|
"she is NOT facing left. "
|
|
"Her chest and front of her body are on the RIGHT side of the image; "
|
|
"her back (spine, shoulder blade) is on the LEFT side of the image. "
|
|
"Her right side is facing the camera, and her left side is completely hidden behind her body"
|
|
)
|
|
|
|
# ── left-front quadrant ────────────────────────────────────────────────────
|
|
elif d < 292.5: # 285° — strong left-front, almost profile
|
|
return (
|
|
"turned strongly to her left — almost a pure side profile, but the face is still slightly visible. "
|
|
"In the output image her face is on the RIGHT side with nose pointing toward the right edge. "
|
|
"Her right ear, right cheek and right shoulder are the main visible features. "
|
|
"Her left breast and left side are mostly hidden"
|
|
)
|
|
elif d < 307.5: # 300° — clear three-quarter left-front
|
|
return (
|
|
"turned so the camera sees a clear three-quarter left-front view. "
|
|
"In the output image her face is partially visible on the RIGHT side, nose pointing right. "
|
|
"Her right breast, right shoulder and right hip are angled toward the camera. "
|
|
"Her left breast, left hip and left side are turned away from camera"
|
|
)
|
|
elif d < 322.5: # 315° — gentle three-quarter left-front
|
|
return (
|
|
"turned about 45° to her left. "
|
|
"Her face is partly toward the camera, right cheek and jaw more visible than left. "
|
|
"In the output image her face appears on the RIGHT half, nose angled toward the right edge. "
|
|
"Her right shoulder, right breast and right hip are angled toward the camera. "
|
|
"Her left side is starting to turn away"
|
|
)
|
|
elif d < 337.5: # 330° — slight left-front turn
|
|
return (
|
|
"turned slightly to her left — a subtle left-front view. "
|
|
"Both eyes visible, face still mostly toward the camera. "
|
|
"In the output image her face is nearly centered but noticeably shifted toward the RIGHT side. "
|
|
"Her right shoulder is clearly closer to the camera than her left"
|
|
)
|
|
else: # 345° — barely perceptible left-front tilt
|
|
return (
|
|
"facing almost directly toward the camera — just the subtlest hint of a left-front turn. "
|
|
"Both eyes and her full face are visible. "
|
|
"In the output image her face is nearly perfectly centered, "
|
|
"with only the tiniest tilt toward the RIGHT edge. "
|
|
"Her right shoulder is just a hair closer to the camera than her left. "
|
|
"This looks almost identical to a pure front view"
|
|
)
|
|
|
|
|
|
def yaw_prompt(deg: float) -> str:
|
|
"""Full prompt for one turntable angle."""
|
|
view = _angle_phrase(deg)
|
|
identity = _IDENTITY_FRONT if is_face_visible(deg) else _IDENTITY_BACK
|
|
return (
|
|
f"Redraw this person {view}. "
|
|
f"Keep everything identical — same person, same hair, same body, same lighting — "
|
|
f"only the camera viewing angle changes. {identity}."
|
|
)
|
|
|
|
|
|
def _angles_for(mode: str, n_views: int, sweep_deg: float) -> list:
|
|
"""Return the list of yaw angles to render."""
|
|
if mode == "turntable":
|
|
# Full 360, evenly spaced, loops cleanly
|
|
return [360.0 * i / n_views for i in range(n_views)]
|
|
elif mode == "swing":
|
|
# -sweep/2 .. +sweep/2 .. back (front-facing arc only — most reliable)
|
|
half = sweep_deg / 2.0
|
|
fwd = [(-half + sweep_deg * i / (n_views - 1)) for i in range(n_views)]
|
|
# map negatives into 0..360 turntable space (e.g. -45 -> 315)
|
|
return [a % 360 for a in fwd]
|
|
raise ValueError(f"Unknown mode: {mode!r}")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 2. View generation (Qwen)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _autocrop_alpha(pil: Image.Image, pad: int = 8) -> Image.Image:
|
|
"""Crop to the alpha bounding box (+pad) so every view is framed on the body."""
|
|
if pil.mode != "RGBA":
|
|
return pil
|
|
alpha = np.array(pil)[:, :, 3]
|
|
ys, xs = np.where(alpha > 16)
|
|
if len(xs) == 0:
|
|
return pil
|
|
x0, x1 = max(0, xs.min() - pad), min(pil.width, xs.max() + pad)
|
|
y0, y1 = max(0, ys.min() - pad), min(pil.height, ys.max() + pad)
|
|
return pil.crop((x0, y0, x1, y1))
|
|
|
|
|
|
def generate_views(
|
|
image_path: str,
|
|
output_dir: str,
|
|
n_views: int = 12,
|
|
seed: int = 42,
|
|
mode: str = "turntable",
|
|
sweep_deg: float = 180.0,
|
|
anchor: str = "original",
|
|
max_area: int = 0,
|
|
steps: int = 8,
|
|
on_progress=None,
|
|
) -> list:
|
|
"""
|
|
Render one Qwen view per yaw angle.
|
|
|
|
anchor='original' — every view edits the SAME front image (stable identity)
|
|
anchor='chain' — each view edits the previous result (smoother transitions,
|
|
but identity can drift over a full turn)
|
|
|
|
Returns list of dicts: {deg, path, pil}.
|
|
"""
|
|
os.makedirs(output_dir, exist_ok=True)
|
|
views_dir = os.path.join(output_dir, "views")
|
|
os.makedirs(views_dir, exist_ok=True)
|
|
|
|
start_pil = Image.open(image_path).convert("RGB")
|
|
is_front = is_front_view(start_pil)
|
|
|
|
if not is_front:
|
|
print(f"[orbit-qwen] Input image is NOT a representative front view. Generating a full front-view first...")
|
|
front_png = _run_pipeline(
|
|
start_pil, yaw_prompt(0.0), seed,
|
|
max_area or MAX_AREA,
|
|
steps=steps
|
|
)
|
|
base_pil = Image.open(io.BytesIO(front_png)).convert("RGB")
|
|
else:
|
|
base_pil = start_pil
|
|
|
|
angles = _angles_for(mode, n_views, sweep_deg)
|
|
|
|
results = []
|
|
prev_pil = None
|
|
completed_views_uncropped: dict[float, Image.Image] = {} # deg -> uncropped RGBA pil
|
|
for i, deg in enumerate(angles):
|
|
# If we pre-generated the front view and this is the 0° view, use it directly!
|
|
if not is_front and abs(deg) < 1e-3:
|
|
view_pil = base_pil.convert("RGBA")
|
|
completed_views_uncropped[deg] = view_pil
|
|
|
|
cropped_pil = _autocrop_alpha(view_pil)
|
|
path = os.path.join(views_dir, f"view_{i:03d}_{int(deg):03d}deg.png")
|
|
cropped_pil.save(path)
|
|
results.append({"deg": deg, "path": path, "pil": cropped_pil})
|
|
|
|
if anchor == "chain":
|
|
prev_pil = base_pil
|
|
continue
|
|
|
|
# Hybrid anchor strategy:
|
|
# Front/side views use the original front view.
|
|
# Back/rear views use the immediately preceding completed view.
|
|
if anchor == "chain":
|
|
src_pil = prev_pil if prev_pil is not None else base_pil
|
|
else:
|
|
# "original" anchor, but with our hybrid back-view chain:
|
|
if not is_face_visible(deg) and i > 0:
|
|
prev_angle = angles[i - 1]
|
|
src_pil = completed_views_uncropped[prev_angle].convert("RGB")
|
|
else:
|
|
src_pil = base_pil
|
|
|
|
prompt = yaw_prompt(deg)
|
|
if on_progress:
|
|
on_progress(i, len(angles), deg)
|
|
|
|
# Pass up to 2 already-generated views as extra references so Qwen can
|
|
# maintain identity/hair/clothing consistency across the full rotation.
|
|
extra_refs = None
|
|
if completed_views_uncropped:
|
|
def _angular_dist(a, b):
|
|
d = abs(a - b) % 360
|
|
return min(d, 360 - d)
|
|
target_visible = is_face_visible(deg)
|
|
eligible_views = {
|
|
a: pil for a, pil in completed_views_uncropped.items()
|
|
if is_face_visible(a) == target_visible
|
|
}
|
|
if eligible_views:
|
|
sorted_done = sorted(eligible_views.keys(),
|
|
key=lambda a: _angular_dist(a, deg))
|
|
extra_refs = [eligible_views[a].convert("RGB") for a in sorted_done[:2]]
|
|
|
|
png = _run_pipeline(
|
|
src_pil, prompt, seed,
|
|
max_area or MAX_AREA,
|
|
steps=steps,
|
|
extra_images=extra_refs,
|
|
)
|
|
view_pil = Image.open(io.BytesIO(png)).convert("RGBA")
|
|
completed_views_uncropped[deg] = view_pil
|
|
|
|
cropped_pil = _autocrop_alpha(view_pil)
|
|
path = os.path.join(views_dir, f"view_{i:03d}_{int(deg):03d}deg.png")
|
|
cropped_pil.save(path)
|
|
results.append({"deg": deg, "path": path, "pil": cropped_pil})
|
|
|
|
if anchor == "chain":
|
|
# Feed an RGB version forward (pipeline wants RGB anyway)
|
|
prev_pil = view_pil.convert("RGB")
|
|
|
|
return results
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 3. Smoothing — canvas-align + optical-flow interpolation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _to_common_canvas(views: list, pad_frac: float = 0.12) -> list:
|
|
"""
|
|
Place every view on one fixed-size RGBA canvas, bottom-centered (feet anchored),
|
|
so the body doesn't jump frame-to-frame. Returns list of HxWx4 uint8 arrays.
|
|
"""
|
|
H = max(v["pil"].height for v in views)
|
|
W = max(v["pil"].width for v in views)
|
|
padH, padW = int(H * pad_frac), int(W * pad_frac)
|
|
CH, CW = H + 2 * padH, W + 2 * padW
|
|
|
|
out = []
|
|
for v in views:
|
|
p = v["pil"]
|
|
canvas = Image.new("RGBA", (CW, CH), (0, 0, 0, 0))
|
|
# bottom-centered: feet sit on a common baseline
|
|
x = (CW - p.width) // 2
|
|
y = CH - padH - p.height
|
|
canvas.paste(p, (x, y), p)
|
|
out.append(np.array(canvas))
|
|
return out
|
|
|
|
|
|
def _flow_morph_rgb(a: np.ndarray, b: np.ndarray, t: float) -> np.ndarray:
|
|
"""
|
|
Optical-flow morph between two SOLID RGB frames (3-channel) at fraction t.
|
|
Operates on composited-over-bg images so there is no alpha halo/ghost.
|
|
Warps a→mid and b→mid, then blends.
|
|
"""
|
|
ag = cv2.cvtColor(a, cv2.COLOR_RGB2GRAY)
|
|
bg = cv2.cvtColor(b, cv2.COLOR_RGB2GRAY)
|
|
flow_ab = cv2.calcOpticalFlowFarneback(ag, bg, None, 0.5, 5, 31, 5, 7, 1.5, 0)
|
|
flow_ba = cv2.calcOpticalFlowFarneback(bg, ag, None, 0.5, 5, 31, 5, 7, 1.5, 0)
|
|
|
|
H, W = ag.shape
|
|
yc, xc = np.mgrid[0:H, 0:W].astype(np.float32)
|
|
wa = cv2.remap(a, (xc + flow_ab[..., 0] * t), (yc + flow_ab[..., 1] * t),
|
|
cv2.INTER_LINEAR, borderMode=cv2.BORDER_REPLICATE)
|
|
wb = cv2.remap(b, (xc + flow_ba[..., 0] * (1 - t)), (yc + flow_ba[..., 1] * (1 - t)),
|
|
cv2.INTER_LINEAR, borderMode=cv2.BORDER_REPLICATE)
|
|
return (wa.astype(np.float32) * (1 - t) + wb.astype(np.float32) * t).clip(0, 255).astype(np.uint8)
|
|
|
|
|
|
def interpolate_views(
|
|
views: list,
|
|
factor: int = 4,
|
|
loop: bool = True,
|
|
smooth: bool = True,
|
|
bg: tuple = (18, 18, 18),
|
|
) -> list:
|
|
"""
|
|
Expand keyframes into a smooth sequence.
|
|
|
|
Keyframes are first composited over the solid bg, so all blending happens
|
|
in opaque RGB space — this removes the transparent-alpha ghosting that
|
|
plagued earlier flow morphs.
|
|
|
|
factor — intermediate frames per keyframe pair (1 = keyframes only)
|
|
loop — also blend last→first (seamless turntable)
|
|
smooth — optical-flow morph (True) vs simple crossfade (False)
|
|
|
|
Returns list of HxWx3 uint8 RGB frames.
|
|
"""
|
|
canvases = _to_common_canvas(views)
|
|
bg_arr = np.array(bg, dtype=np.float32)
|
|
|
|
def _flatten(rgba):
|
|
a = rgba[:, :, 3:4].astype(np.float32) / 255.0
|
|
return (rgba[:, :, :3].astype(np.float32) * a + bg_arr * (1 - a)).clip(0, 255).astype(np.uint8)
|
|
|
|
solid = [_flatten(c) for c in canvases]
|
|
if factor <= 1:
|
|
return solid
|
|
|
|
n = len(solid)
|
|
pairs = n if loop else n - 1
|
|
frames = []
|
|
for i in range(pairs):
|
|
a, b = solid[i], solid[(i + 1) % n]
|
|
frames.append(a)
|
|
for k in range(1, factor):
|
|
t = k / factor
|
|
if smooth:
|
|
frames.append(_flow_morph_rgb(a, b, t))
|
|
else:
|
|
frames.append((a.astype(np.float32) * (1 - t) +
|
|
b.astype(np.float32) * t).astype(np.uint8))
|
|
if not loop:
|
|
frames.append(solid[-1])
|
|
return frames
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 4. Video
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _composite_solid(frame: np.ndarray, bg=(18, 18, 18)) -> np.ndarray:
|
|
"""Accept RGB (already flattened) or RGBA; return BGR for ffmpeg."""
|
|
if frame.shape[2] == 3:
|
|
return cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
|
|
rgb = frame[:, :, :3].astype(np.float32)
|
|
a = frame[:, :, 3:4].astype(np.float32) / 255.0
|
|
bg_f = np.array(bg, dtype=np.float32)
|
|
out = (rgb * a + bg_f * (1 - a)).clip(0, 255).astype(np.uint8)
|
|
return cv2.cvtColor(out, cv2.COLOR_RGB2BGR)
|
|
|
|
|
|
def build_video(frames: list, output_path: str, fps: int = 24, bg=(18, 18, 18)) -> None:
|
|
if not frames:
|
|
return
|
|
with tempfile.TemporaryDirectory(prefix="orbit_qwen_") as tmp:
|
|
for i, fr in enumerate(frames):
|
|
cv2.imwrite(os.path.join(tmp, f"f_{i:04d}.jpg"),
|
|
_composite_solid(fr, bg), [cv2.IMWRITE_JPEG_QUALITY, 95])
|
|
H, W = frames[0].shape[:2]
|
|
W2, H2 = W - (W % 2), H - (H % 2)
|
|
cmd = [
|
|
"ffmpeg", "-y", "-framerate", str(fps),
|
|
"-i", os.path.join(tmp, "f_%04d.jpg"),
|
|
"-vf", f"crop={W2}:{H2}:0:0",
|
|
"-c:v", "libx264", "-pix_fmt", "yuv420p",
|
|
"-crf", "18", "-movflags", "+faststart", output_path,
|
|
]
|
|
r = subprocess.run(cmd, capture_output=True, text=True)
|
|
if r.returncode != 0:
|
|
raise RuntimeError(f"ffmpeg failed: {r.stderr[-600:]}")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 5. Orchestration
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def run_qwen_orbit(
|
|
image_path: str,
|
|
output_dir: str,
|
|
n_views: int = 24,
|
|
seed: int = 42,
|
|
mode: str = "turntable",
|
|
sweep_deg: float = 180.0,
|
|
anchor: str = "original",
|
|
interp_factor: int = 1,
|
|
smooth: bool = False,
|
|
fps: int = 12,
|
|
max_area: int = 0,
|
|
steps: int = 8,
|
|
on_progress=None,
|
|
) -> dict:
|
|
"""
|
|
Full near-real turntable: generate Qwen views → align → MP4.
|
|
|
|
Defaults reflect the validated recipe: 24 crisp keyframes, NO blending,
|
|
12fps. Raise interp_factor only if you accept morph ghosting.
|
|
|
|
Returns dict: views (list), n_views, n_frames, video_path, views_dir.
|
|
"""
|
|
os.makedirs(output_dir, exist_ok=True)
|
|
|
|
views = generate_views(
|
|
image_path, output_dir,
|
|
n_views=n_views, seed=seed, mode=mode, sweep_deg=sweep_deg,
|
|
anchor=anchor, max_area=max_area, steps=steps, on_progress=on_progress,
|
|
)
|
|
|
|
loop = (mode == "turntable")
|
|
frames = interpolate_views(views, factor=interp_factor, loop=loop, smooth=smooth)
|
|
|
|
video_path = "" # MP4 not wanted, custom frame-loop used instead
|
|
|
|
return {
|
|
"views": [{"deg": v["deg"], "path": v["path"]} for v in views],
|
|
"n_views": len(views),
|
|
"n_frames": len(frames),
|
|
"video_path": "",
|
|
"views_dir": os.path.join(output_dir, "views"),
|
|
}
|