This commit is contained in:
mike
2026-06-25 19:05:21 +02:00
parent 27e8a09bc1
commit ce7b0e52e5
3 changed files with 681 additions and 7 deletions

View File

@@ -20,6 +20,7 @@ Pose tools (rtmlib / RTMPose, 2D):
- ✅ Draggable joints to refine/explore a pose
- ✅ Find similar poses across the library (translation/scale/mirror-invariant; `poses_index.json`)
- ✅ Gesture presets in pose toolbar (T-pose, wave, arms raised, hands on hips, turn ±45°)
- ✅ "Generate ▸" in pose toolbar — renders skeleton as OpenPose image → Qwen image2 guide → new generation
- ✅ "Orbit all" button in Generate tab selects all 6 absolute camera angles at once
Scenery:

View File

@@ -1895,6 +1895,7 @@
<button class="sb-btn" id="lbRotateLeftBtn" onclick="lbRotate(-90)" title="Rotate 90° counter-clockwise">⟲ 90°</button>
<button class="sb-btn" id="lbRotateRightBtn" onclick="lbRotate(90)" title="Rotate 90° clockwise">⟳ 90°</button>
<button class="sb-btn" id="lbPoseBtn" onclick="lbTogglePose()" title="Preview body pose skeleton">Pose</button>
<button class="sb-btn" id="lbMakeVideoBtn" onclick="lbMakeVideo()" title="Stitch group images (or Shift+Click selection) into a short MP4">Make video</button>
<a class="sb-btn" id="lbDownloadBtn" download style="text-decoration:none">Download</a>
<button class="sb-btn" id="lbArchiveBtn" onclick="lbArchive()" style="color:#f59e0b" title="Move to archive (recoverable)">Archive</button>
<button class="sb-btn danger" id="lbDeleteBtn" onclick="lbDeleteArm()" title="Click once to arm, then again to confirm">Delete</button>
@@ -2926,7 +2927,9 @@
+ '<div style="position:relative;display:inline-block">'
+ ' <button class="sb-btn" id="gestureMenuBtn" onclick="toggleGestureMenu(event)" title="Apply a gesture preset to the skeleton">Gesture ▾</button>'
+ ' <div id="gestureMenuPanel" style="display:none;position:absolute;top:100%;left:0;z-index:200;background:#18181b;border:1px solid #2a2a2a;border-radius:6px;padding:4px;box-shadow:0 6px 20px rgba(0,0,0,.7);min-width:150px;margin-top:4px"></div>'
+ '</div>';
+ '</div>'
+ '<button class="sb-btn" onclick="lbPoseFromVideo()" title="Pick a pose from a wireframe video frame">From video</button>'
+ '<button class="sb-btn" id="poseGenBtn" onclick="lbGenerateWithPose()" title="Generate the person in this pose (skeleton → Qwen image2 guide)">Generate ▸</button>';
viewer.appendChild(bar);
_buildGestureMenu();
@@ -2997,9 +3000,10 @@
const g = GESTURE_PRESETS[idx];
const W = st.width, H = st.height;
st.people[0] = g.kpts.map(([nx, ny, c]) => [nx * W, ny * H, c]);
st._gestureName = g.name; // remember for generation
_drawPose();
document.getElementById('gestureMenuPanel').style.display = 'none';
showToast(`Gesture: ${g.name}`, 'success');
showToast(`Gesture: ${g.name} — click Generate ▸ to create`, 'success');
}
function _drawPose() {
@@ -3098,6 +3102,154 @@
if (btn) btn.disabled = false;
}
async function lbGenerateWithPose() {
const st = window._poseState;
if (!st || !st.people.length) { showToast('No pose overlay active — click Pose first', 'info'); return; }
const fname = st.fname;
if (!fname) { showToast('No image active', 'info'); return; }
const btn = document.getElementById('poseGenBtn');
if (btn) { btn.disabled = true; btn.textContent = '…'; }
try {
const payload = {
model_filename: fname,
keypoints: st.people[0],
width: st.width,
height: st.height,
gesture_name: st._gestureName || null,
seed: -1,
};
const r = await fetch(`${API}/generate-with-pose`, {
method: 'POST', headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(payload),
});
if (!r.ok) { showToast('Generate failed: ' + await r.text(), 'error'); return; }
const { job_id } = await r.json();
showToast('Pose generation started…', 'info');
if (btn) btn.textContent = 'Generating…';
const poll = () => setTimeout(async () => {
try {
const jr = await fetch(`${API}/batch/${job_id}`);
if (!jr.ok) { poll(); return; }
const job = await jr.json();
if (job.status === 'running') { poll(); return; }
if (btn) { btn.disabled = false; btn.textContent = 'Generate ▸'; }
if (job.status === 'done') {
showToast('Pose generation done!', 'success');
_followLatestGid = lbCurrentGid;
refreshNow();
} else {
showToast('Pose generation failed: ' + (job.error || 'unknown'), 'error');
}
} catch (e) { poll(); }
}, 2500);
poll();
} catch (e) {
showToast('Generate error: ' + e, 'error');
if (btn) { btn.disabled = false; btn.textContent = 'Generate ▸'; }
}
}
// ─── Pose from wireframe video ───────────────────────────────────────────
// Opens a modal that lists wireframe videos for the current group,
// lets the user scrub to a timestamp, then extracts keypoints from that
// frame and loads them into the draggable pose overlay.
async function lbPoseFromVideo() {
const st = window._poseState;
if (!st) { showToast('Open Pose first', 'info'); return; }
// Gather wireframe videos from the current group's images.
const videos = (lbNames || []).filter(n => /\.(mp4|webm|mov)$/i.test(n));
if (!videos.length) {
showToast('No videos in this group — generate a wireframe video first', 'info');
return;
}
// Build modal
const overlay = document.createElement('div');
overlay.id = 'poseVideoModal';
overlay.style.cssText = 'position:fixed;inset:0;z-index:900;background:rgba(0,0,0,.8);display:flex;align-items:center;justify-content:center;';
overlay.innerHTML = `
<div style="background:#18181b;border:1px solid #2a2a2a;border-radius:10px;padding:20px;width:480px;max-width:95vw;box-shadow:0 8px 40px rgba(0,0,0,.8)">
<div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:14px">
<span style="font-size:14px;font-weight:600">Pick pose from wireframe video</span>
<button class="sb-btn" onclick="document.getElementById('poseVideoModal').remove()">✕</button>
</div>
<div style="margin-bottom:10px">
<label style="font-size:11px;color:#aaa;display:block;margin-bottom:4px">Video</label>
<select id="pvmVideoSelect" style="width:100%;background:#0f0f0f;color:#e0e0e0;border:1px solid #2a2a2a;border-radius:5px;padding:5px 8px;font-size:12px">
${videos.map(v => `<option value="${escHtml(v)}">${escHtml(v)}</option>`).join('')}
</select>
</div>
<div style="margin-bottom:10px">
<video id="pvmVideo" style="width:100%;border-radius:6px;background:#000;max-height:260px" preload="metadata"></video>
</div>
<div style="margin-bottom:14px">
<label style="font-size:11px;color:#aaa;display:block;margin-bottom:4px">Scrub to pose frame</label>
<input type="range" id="pvmScrubber" min="0" max="100" step="0.1" value="0"
style="width:100%;accent-color:#a78bfa">
<span id="pvmTimeLabel" style="font-size:11px;color:#888">0.0 s</span>
</div>
<div style="display:flex;gap:8px;justify-content:flex-end">
<button class="sb-btn" onclick="document.getElementById('poseVideoModal').remove()">Cancel</button>
<button class="sb-btn primary" id="pvmLoadBtn" onclick="pvmLoadPose()">Load pose</button>
</div>
</div>`;
document.body.appendChild(overlay);
const vidEl = document.getElementById('pvmVideo');
const select = document.getElementById('pvmVideoSelect');
const scrubber = document.getElementById('pvmScrubber');
const timeLabel = document.getElementById('pvmTimeLabel');
function pvmSetVideo(filename) {
vidEl.src = `${API}/output/${encodeURIComponent(filename)}`;
vidEl.load();
vidEl.addEventListener('loadedmetadata', () => {
const dur = vidEl.duration || 10;
scrubber.max = dur;
scrubber.step = Math.max(0.033, dur / 500);
scrubber.value = 0;
timeLabel.textContent = '0.0 s';
}, { once: true });
}
scrubber.addEventListener('input', () => {
const t = parseFloat(scrubber.value);
vidEl.currentTime = t;
timeLabel.textContent = t.toFixed(2) + ' s';
});
select.addEventListener('change', () => pvmSetVideo(select.value));
pvmSetVideo(videos[0]);
window.pvmLoadPose = async function() {
const video = select.value;
const time_s = parseFloat(scrubber.value);
const btn = document.getElementById('pvmLoadBtn');
btn.disabled = true;
btn.textContent = 'Extracting…';
try {
const r = await fetch(`${API}/pose/from-wireframe`, {
method: 'POST', headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ video, time_s }),
});
if (!r.ok) { showToast('Pose extract failed: ' + await r.text(), 'error'); btn.disabled = false; btn.textContent = 'Load pose'; return; }
const d = await r.json();
// Rescale keypoints from video dimensions to current image dimensions
const scaleX = st.width / d.width;
const scaleY = st.height / d.height;
st.people[0] = d.keypoints.map(([x, y, c]) => [x * scaleX, y * scaleY, c]);
st._gestureName = null; // came from real video, not a named preset
_drawPose();
document.getElementById('poseVideoModal').remove();
showToast(`Pose loaded from ${video} @ ${time_s.toFixed(2)}s`, 'success');
} catch (e) {
showToast('Pose extract error: ' + e, 'error');
btn.disabled = false; btn.textContent = 'Load pose';
}
};
}
async function lbFindSimilarFaces() {
if (!lbCurrentGid) return;
try {
@@ -3257,13 +3409,14 @@
const bar = document.createElement('div');
bar.id = 'cropBar';
// Anchored at the TOP so the buttons never block cropping to the bottom edge.
bar.style.cssText = 'position:absolute;top:0;left:0;right:0;display:flex;gap:8px;padding:8px;background:rgba(0,0,0,0.75);z-index:101;justify-content:flex-end;align-items:center;';
// pointer-events:none on the container so the canvas underneath captures drags
// from the very top of the image. Individual interactive controls opt back in.
bar.style.cssText = 'position:absolute;top:0;left:0;right:0;display:flex;gap:8px;padding:8px;background:rgba(0,0,0,0.75);z-index:101;justify-content:flex-end;align-items:center;pointer-events:none;';
bar.innerHTML = '<span style="flex:1;font-size:11px;color:#aaa">Drag to select crop area</span>'
+ '<label style="display:flex;align-items:center;gap:4px;font-size:11px;color:#ccc;cursor:pointer" title="Keep the original and crop a copy instead">'
+ '<label style="display:flex;align-items:center;gap:4px;font-size:11px;color:#ccc;cursor:pointer;pointer-events:auto" title="Keep the original and crop a copy instead">'
+ '<input type="checkbox" id="cropAsCopy" checked style="cursor:pointer">Save as copy</label>'
+ '<button class="sb-btn" onclick="cancelManualCrop()">Cancel</button>'
+ '<button class="sb-btn primary" id="cropConfirmBtn" disabled onclick="confirmManualCrop()">Crop</button>';
+ '<button class="sb-btn" style="pointer-events:auto" onclick="cancelManualCrop()">Cancel</button>'
+ '<button class="sb-btn primary" style="pointer-events:auto" id="cropConfirmBtn" disabled onclick="confirmManualCrop()">Crop</button>';
viewer.appendChild(bar);
const st = window._cropState = { canvas, viewer, img, fname, x1: 0, y1: 0, x2: 0, y2: 0, dragging: false };
@@ -3435,6 +3588,32 @@
if (btn) btn.disabled = false;
}
async function lbMakeVideo() {
// Use Shift+Click selected refs if any, otherwise all images in the current group.
let filenames;
if (_sbRefIndices.length >= 2) {
filenames = _sbRefIndices.map(i => lbNames[i]).filter(n => n && !isVideo(n));
if (filenames.length < 2) { showToast('Need at least 2 non-video images selected (Shift+Click filmstrip)', 'info'); return; }
} else {
filenames = lbNames.filter(n => n && !isVideo(n));
if (filenames.length < 2) { showToast('Need at least 2 images in this group', 'info'); return; }
}
const btn = document.getElementById('lbMakeVideoBtn');
if (btn) { btn.disabled = true; btn.textContent = 'Creating…'; }
try {
const r = await fetch(`${API}/generate-video`, {
method: 'POST', headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ filenames, fps: 8, loop: true }),
});
if (!r.ok) { showToast('Video failed: ' + await r.text(), 'error'); return; }
const d = await r.json();
showToast(`Video created: ${d.output}`, 'success');
_followLatestGid = lbCurrentGid;
refreshNow();
} catch (e) { showToast('Video error: ' + e, 'error'); }
if (btn) { btn.disabled = false; btn.textContent = 'Make video'; }
}
// Force the studio image (filmstrip + main viewer) to reload from disk.
function _bustStudioImage(fname) {
const t = Date.now();

View File

@@ -1576,6 +1576,88 @@ def trim_wireframe(req: TrimRequest):
return {'output_name': out_name, 'start': req.start, 'end': req.end}
class VideoStitchRequest(BaseModel):
filenames: list[str] # images from output_dir, in desired frame order
fps: float = 8.0
loop: bool = True # append reversed sequence for a ping-pong loop
output_name: str | None = None
@app.post("/generate-video")
def generate_video(req: VideoStitchRequest):
"""Stitch a list of output images into a short looping MP4 using ffmpeg."""
import subprocess, tempfile, textwrap
if len(req.filenames) < 2:
raise HTTPException(400, "Need at least 2 images")
if not (0.1 <= req.fps <= 60):
raise HTTPException(400, "fps must be between 0.1 and 60")
output_dir = _load_output_dir()
# Validate + collect paths
paths = []
for fn in req.filenames:
p = os.path.join(output_dir, fn)
if not os.path.exists(p):
raise HTTPException(404, f"Image not found: {fn}")
paths.append(p)
# Ping-pong loop: forward + reversed (drop first/last to avoid duplicate frames at seam)
if req.loop and len(paths) > 1:
paths = paths + list(reversed(paths[1:-1])) if len(paths) > 2 else paths + list(reversed(paths))
ts = time.strftime("%Y%m%d_%H%M%S")
base = naming.get_base_name(req.filenames[0])
stem = os.path.splitext(base)[0]
out_name = req.output_name or f"{ts}_vid_{stem}.mp4"
if not out_name.lower().endswith(".mp4"):
out_name += ".mp4"
out_path = os.path.join(output_dir, out_name)
# Build ffmpeg concat list in a temp file
duration = 1.0 / req.fps
with tempfile.NamedTemporaryFile("w", suffix=".txt", delete=False) as tf:
for p in paths:
tf.write(f"file '{p}'\nduration {duration:.6f}\n")
# ffmpeg concat needs the last entry without a duration
tf.write(f"file '{paths[-1]}'\n")
concat_path = tf.name
try:
r = subprocess.run(
["ffmpeg", "-y",
"-f", "concat", "-safe", "0", "-i", concat_path,
"-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", # ensure even dimensions
"-c:v", "libx264", "-pix_fmt", "yuv420p",
"-movflags", "+faststart",
out_path],
capture_output=True, timeout=120,
)
finally:
os.unlink(concat_path)
if r.returncode != 0:
raise HTTPException(500, f"ffmpeg error: {r.stderr.decode(errors='replace')[:600]}")
# Register in DB so it shows up in the gallery
person = database.get_person(req.filenames[0])
group_id = person[1] if person and person[1] else naming.get_base_name(req.filenames[0])
try:
next_order = database.get_next_sort_order(group_id)
database.upsert_person(
out_name, filepath=out_path,
group_id=group_id,
sort_order=next_order,
content_type="video",
source_refs=json.dumps(req.filenames),
)
_invalidate_static()
except Exception as e:
print(f"[generate-video] DB error: {e}")
return {"output": out_name, "frames": len(paths), "fps": req.fps}
class FrameExtractRequest(BaseModel):
video_name: str
time: float = 0.0
@@ -3287,6 +3369,418 @@ def similar_pose_from_keypoints(req: PoseSimilarRequest):
return {"similar": _rank_similar_poses(query, req.limit)}
# --- pose-guided generation ---------------------------------------------------
# Renders COCO-17 keypoints as an OpenPose-style skeleton image and passes it as
# image2 to Qwen so the VLM can match the body pose while preserving appearance.
# Color per POSE_SKELETON entry (18 bones in POSE_SKELETON order).
_OPENPOSE_BONE_COLORS = [
(0, 80, 255), # 5→7 L upper arm
(0, 130, 255), # 7→9 L lower arm
(255, 50, 0), # 6→8 R upper arm
(255, 110, 0), # 8→10 R lower arm
(255, 200, 0), # 11→13 L upper leg
(240, 240, 0), # 13→15 L lower leg
(100, 255, 0), # 12→14 R upper leg
( 50, 220, 0), # 14→16 R lower leg
(200, 0, 220), # 5→6 shoulders
(200, 0, 220), # 11→12 hips
(180, 60, 180), # 5→11 L torso side
( 60, 180, 180), # 6→12 R torso side
( 0, 230, 230), # 0→1 noseL eye
( 0, 230, 230), # 0→2 noseR eye
( 0, 180, 180), # 1→3 L eyeear
( 0, 180, 180), # 2→4 R eyeear
( 80, 80, 255), # 0→5 headL shoulder
(255, 80, 80), # 0→6 headR shoulder
]
# Per-joint hue (HSV-like, converted to RGB for PIL):
def _joint_color(idx: int) -> tuple[int, int, int]:
h = idx / 17.0 * 6.0
x = 1 - abs(h % 2 - 1)
if h < 1: r, g, b = 1, x, 0
elif h < 2: r, g, b = x, 1, 0
elif h < 3: r, g, b = 0, 1, x
elif h < 4: r, g, b = 0, x, 1
elif h < 5: r, g, b = x, 0, 1
else: r, g, b = 1, 0, x
return (int(r * 255), int(g * 255), int(b * 255))
def _render_openpose_image(keypoints: list, width: int, height: int) -> Image.Image:
"""Draw COCO-17 keypoints as a white stick-figure on a vivid magenta background.
Vivid background (not black/dark) prevents Qwen from treating the skeleton as
a real scene element and from blending the bone colors into the generated output.
White lines on magenta read unambiguously as a technical reference diagram.
"""
from PIL import ImageDraw
MIN_CONF = 0.25
cap_w = min(width, 896)
cap_h = min(height, 1152)
scale = min(cap_w / max(width, 1), cap_h / max(height, 1))
out_w, out_h = int(width * scale), int(height * scale)
img = Image.new("RGB", (out_w, out_h), color=(200, 0, 180)) # vivid magenta
draw = ImageDraw.Draw(img)
lw = max(4, out_w // 80) # line width scales with image
jr = max(5, out_w // 70) # joint radius
kpts = keypoints # [[x, y, conf], ...]
# bones — white so they stand out on the magenta background without
# adding any colors that Qwen might reproduce in the generated image
for a, b in POSE_SKELETON:
if a >= len(kpts) or b >= len(kpts):
continue
pa, pb = kpts[a], kpts[b]
if pa[2] < MIN_CONF or pb[2] < MIN_CONF:
continue
ax, ay = int(pa[0] * scale), int(pa[1] * scale)
bx, by = int(pb[0] * scale), int(pb[1] * scale)
draw.line([(ax, ay), (bx, by)], fill=(255, 255, 255), width=lw)
# joints — light yellow circles with dark outline for visibility
for kp in kpts:
if kp[2] < MIN_CONF:
continue
x, y = int(kp[0] * scale), int(kp[1] * scale)
draw.ellipse([(x - jr, y - jr), (x + jr, y + jr)],
fill=(255, 240, 100), outline=(60, 0, 60), width=max(1, lw // 2))
return img
class PoseFromWireframeRequest(BaseModel):
video: str # filename of a wireframe video in the output dir
time_s: float = 0.0 # timestamp in seconds to extract the frame from
@app.post("/pose/from-wireframe")
def pose_from_wireframe(req: PoseFromWireframeRequest):
"""Extract a frame from a wireframe video, run pose estimation, return COCO-17 keypoints.
This is the pose-discovery tool: scrub through a wireframe video to find a frame
that shows the desired pose, then load those keypoints into the pose overlay without
ever passing the wireframe image to Qwen.
"""
import tempfile
import subprocess as sp
output_dir = _load_output_dir()
video_path = os.path.join(output_dir, req.video)
if not os.path.exists(video_path):
raise HTTPException(404, f"Video not found: {req.video}")
est = _load_pose_estimator()
if not est:
raise HTTPException(501, "No pose estimator installed. Try: pip install rtmlib onnxruntime")
infer, backend = est
# Extract one frame at the requested timestamp with ffmpeg.
with tempfile.NamedTemporaryFile(suffix=".jpg", delete=False) as tf:
frame_path = tf.name
try:
result = sp.run(
[
"/usr/bin/ffmpeg", "-y",
"-ss", str(req.time_s),
"-i", video_path,
"-frames:v", "1",
"-q:v", "2",
frame_path,
],
capture_output=True, timeout=30,
)
if result.returncode != 0:
raise HTTPException(500, f"ffmpeg frame extract failed: {result.stderr.decode()[:300]}")
pil = Image.open(frame_path).convert("RGB")
finally:
try:
os.unlink(frame_path)
except OSError:
pass
try:
people = infer(pil)
except Exception as e:
raise HTTPException(500, f"Pose estimation failed: {e}")
best = _best_person(people)
if best is None:
raise HTTPException(422, "No person detected in the extracted frame")
return {
"status": "success",
"backend": backend,
"width": pil.width,
"height": pil.height,
"names": POSE_KEYPOINT_NAMES,
"skeleton": POSE_SKELETON,
"keypoints": best,
"time_s": req.time_s,
}
class PoseGenRequest(BaseModel):
model_filename: str
keypoints: list[list[float]] # 17 × [x, y, conf] in original image pixels
width: int # original image dimensions the keypoints came from
height: int
gesture_name: str | None = None # preset name for prompt enrichment and tagging
prompt: str | None = None # full override; auto-built if None
seed: int = -1
extra_filename: str | None = None # optional image3 extra reference
def _keypoints_to_pose_text(kpts: list, width: int, height: int) -> str:
"""Convert COCO-17 keypoints to a natural-language pose description for Qwen."""
MIN_CONF = 0.25
W, H = max(width, 1), max(height, 1)
def vis(idx):
return idx < len(kpts) and kpts[idx][2] >= MIN_CONF
def pt(idx):
return (kpts[idx][0] / W, kpts[idx][1] / H) if vis(idx) else None
parts = []
# --- head orientation ---
nose, l_ear, r_ear = pt(0), pt(3), pt(4)
if l_ear and r_ear:
ear_mid_x = (l_ear[0] + r_ear[0]) / 2
if nose:
dx = nose[0] - ear_mid_x
if dx < -0.05:
parts.append("head turned to the left")
elif dx > 0.05:
parts.append("head turned to the right")
else:
parts.append("head facing forward")
elif l_ear and not r_ear:
parts.append("head turned strongly to the right")
elif r_ear and not l_ear:
parts.append("head turned strongly to the left")
# --- body rotation (shoulders vs hips) ---
l_sh, r_sh = pt(5), pt(6)
l_hip, r_hip = pt(11), pt(12)
if l_sh and r_sh:
sh_w = abs(r_sh[0] - l_sh[0])
if l_hip and r_hip:
hip_w = abs(r_hip[0] - l_hip[0])
ratio = sh_w / max(hip_w, 0.01)
if ratio < 0.5:
parts.append("body rotated ~90° (side profile)")
elif ratio < 0.75:
parts.append("body rotated ~45° (three-quarter view)")
else:
parts.append("body facing forward")
# --- arm positions ---
for side, sh_i, el_i, wr_i in [("left", 5, 7, 9), ("right", 6, 8, 10)]:
sh, el, wr = pt(sh_i), pt(el_i), pt(wr_i)
if not sh:
continue
if wr:
dy = sh[1] - wr[1] # positive = wrist above shoulder
dx = abs(wr[0] - sh[0])
if dy > 0.15:
if el and pt(el_i):
el_dy = sh[1] - el[1]
if el_dy > 0.05:
parts.append(f"{side} arm raised straight up")
else:
parts.append(f"{side} arm raised with elbow bent")
else:
parts.append(f"{side} arm raised above shoulder")
elif dx > 0.25:
parts.append(f"{side} arm extended sideways")
elif el:
el_dx = abs(el[0] - sh[0])
if el_dx > 0.15 and abs(wr[1] - sh[1]) < 0.1:
parts.append(f"{side} arm extended forward")
else:
parts.append(f"{side} arm at side")
else:
parts.append(f"{side} arm at side")
elif el:
el_dy = sh[1] - el[1]
if el_dy > 0.1:
parts.append(f"{side} elbow raised")
# --- leg positions ---
for side, hip_i, kn_i, an_i in [("left", 11, 13, 15), ("right", 12, 14, 16)]:
hip, kn, an = pt(hip_i), pt(kn_i), pt(an_i)
if not hip:
continue
if an:
dx = abs(an[0] - hip[0])
dy = an[1] - hip[1] # positive = ankle below hip (normal standing)
if dy < 0.1:
parts.append(f"{side} leg raised / knee up")
elif dx > 0.2:
parts.append(f"{side} leg stepped out to the side")
elif kn:
kn_dx = abs(kn[0] - hip[0])
if kn_dx > 0.15:
parts.append(f"{side} leg bent / lunging")
else:
parts.append(f"{side} leg straight, standing")
else:
parts.append(f"{side} leg straight")
if not parts:
return "standing neutral pose"
return ", ".join(parts)
def _pad_for_pose(pil: Image.Image, keypoints: list, margin: float = 0.15) -> tuple[Image.Image, tuple[int,int,int,int]]:
"""Expand the image so target keypoints fit within the canvas.
Only expands when keypoints are actually outside [0,W]×[0,H]. Expansion is
symmetric on each axis (equal left/right, equal top/bottom) so the person
stays centred on the new canvas. Fill is black to match PIL's RGBA→RGB fill.
"""
W, H = pil.size
all_kps = [kp for kp in keypoints if len(kp) >= 2]
if not all_kps:
return pil, (0, 0, 0, 0)
xs = [kp[0] for kp in all_kps]
ys = [kp[1] for kp in all_kps]
# How far each side actually exceeds the image bounds (0 if within bounds).
over_l = max(0.0, -min(xs))
over_r = max(0.0, max(xs) - W)
over_t = max(0.0, -min(ys))
over_b = max(0.0, max(ys) - H)
# If nothing is outside, no padding needed at all.
if over_l == over_r == over_t == over_b == 0:
return pil, (0, 0, 0, 0)
# Symmetric expansion per axis: take the larger overhang and add margin.
px = int(max(over_l, over_r) + margin * W)
py = int(max(over_t, over_b) + margin * H)
new_w, new_h = W + 2 * px, H + 2 * py
canvas = Image.new("RGB", (new_w, new_h), color=(0, 0, 0))
canvas.paste(pil, (px, py))
print(f"[pose-gen] padded {W}×{H}{new_w}×{new_h}{px}px x, ±{py}px y)")
return canvas, (px, py, px, py)
def _pose_gen_worker(job_id: str, model_filename: str, prompt: str, seed: int,
keypoints: list | None = None,
gesture_name: str | None = None,
extra_filename: str | None = None):
output_dir = _load_output_dir()
try:
model_path = os.path.join(output_dir, model_filename)
model_pil = Image.open(model_path).convert("RGB")
# Auto-pad if the target pose would extend outside the current image bounds.
if keypoints:
model_pil, _padding = _pad_for_pose(model_pil, keypoints)
# No skeleton image — pose is encoded entirely in the text prompt.
extra_images = None
if extra_filename:
ep = os.path.join(output_dir, extra_filename)
extra_images = [Image.open(ep).convert("RGB")]
png_bytes = _run_pipeline(model_pil, prompt, seed, MAX_AREA, extra_images=extra_images)
ts = time.strftime("%Y%m%d_%H%M%S")
base_name = naming.get_base_name(model_filename)
if not base_name.lower().endswith(".png"):
base_name = os.path.splitext(base_name)[0] + ".png"
out_name = f"{ts}_pose_{base_name}"
out_path = os.path.join(output_dir, out_name)
with open(out_path, "wb") as f:
f.write(png_bytes)
person = database.get_person(model_filename)
group_id = person[1] if person and person[1] else naming.get_base_name(model_filename)
embedding = embeddings.generate_embedding(out_path)
next_order = database.get_next_sort_order(group_id)
refs = [model_filename]
if gesture_name:
refs.append(f"gesture:{gesture_name}")
database.upsert_person(
out_name, filepath=out_path, embedding=embedding,
group_id=group_id, prompt=prompt,
pose=gesture_name or "custom",
sort_order=next_order,
source_refs=json.dumps(refs),
)
jobs[job_id]["status"] = "done"
jobs[job_id]["output"] = out_name
_invalidate_static()
except Exception as e:
print(f"[pose-gen] error: {e}")
jobs[job_id]["status"] = "error"
jobs[job_id]["error"] = str(e)
@app.post("/pose/render")
def render_pose_image(req: PoseSimilarRequest):
"""Render the supplied keypoints as an OpenPose-style skeleton PNG (base64) — for preview only."""
import base64
skeleton = _render_openpose_image(req.keypoints, req.width or 512, req.height or 768)
buf = io.BytesIO()
skeleton.save(buf, format="PNG")
return {"image": base64.b64encode(buf.getvalue()).decode()}
@app.post("/generate-with-pose")
def generate_with_pose(req: PoseGenRequest):
"""Generate the person in a specific body pose. Pose is encoded as text — no skeleton image
is passed to Qwen to avoid wireframe bleed-through in the output."""
output_dir = _load_output_dir()
model_path = os.path.join(output_dir, req.model_filename)
if not os.path.exists(model_path):
raise HTTPException(404, f"Model image not found: {req.model_filename}")
if req.extra_filename:
ep = os.path.join(output_dir, req.extra_filename)
if not os.path.exists(ep):
raise HTTPException(404, f"Extra reference not found: {req.extra_filename}")
if req.prompt:
prompt = req.prompt
else:
# Convert skeleton keypoints to natural-language description
pose_text = _keypoints_to_pose_text(req.keypoints, req.width, req.height)
if req.gesture_name:
pose_text = f'{req.gesture_name} gesture: {pose_text}'
prompt = (
f"Change the body pose of the person in image 1 to: {pose_text}. "
"Keep the person's face, hair, skin tone, clothing, and body proportions "
"exactly as they are — only the limb positions and body orientation change. "
"Transparent background."
)
job_id = uuid.uuid4().hex[:8]
jobs[job_id] = {"status": "running", "type": "pose_gen", "total": 1, "done": 0, "failed": 0}
threading.Thread(
target=_pose_gen_worker,
args=(job_id, req.model_filename, prompt, req.seed),
kwargs={
"keypoints": req.keypoints,
"gesture_name": req.gesture_name,
"extra_filename": req.extra_filename,
},
daemon=True,
).start()
return {"job_id": job_id, "model": req.model_filename}
if __name__ == "__main__":
import uvicorn
uvicorn.run(app, host=os.environ.get("HOST", "0.0.0.0"),