aa
This commit is contained in:
@@ -20,6 +20,7 @@ Pose tools (rtmlib / RTMPose, 2D):
|
||||
- ✅ Draggable joints to refine/explore a pose
|
||||
- ✅ Find similar poses across the library (translation/scale/mirror-invariant; `poses_index.json`)
|
||||
- ✅ Gesture presets in pose toolbar (T-pose, wave, arms raised, hands on hips, turn ±45°)
|
||||
- ✅ "Generate ▸" in pose toolbar — renders skeleton as OpenPose image → Qwen image2 guide → new generation
|
||||
- ✅ "Orbit all" button in Generate tab selects all 6 absolute camera angles at once
|
||||
|
||||
Scenery:
|
||||
|
||||
@@ -1895,6 +1895,7 @@
|
||||
<button class="sb-btn" id="lbRotateLeftBtn" onclick="lbRotate(-90)" title="Rotate 90° counter-clockwise">⟲ 90°</button>
|
||||
<button class="sb-btn" id="lbRotateRightBtn" onclick="lbRotate(90)" title="Rotate 90° clockwise">⟳ 90°</button>
|
||||
<button class="sb-btn" id="lbPoseBtn" onclick="lbTogglePose()" title="Preview body pose skeleton">Pose</button>
|
||||
<button class="sb-btn" id="lbMakeVideoBtn" onclick="lbMakeVideo()" title="Stitch group images (or Shift+Click selection) into a short MP4">Make video</button>
|
||||
<a class="sb-btn" id="lbDownloadBtn" download style="text-decoration:none">Download</a>
|
||||
<button class="sb-btn" id="lbArchiveBtn" onclick="lbArchive()" style="color:#f59e0b" title="Move to archive (recoverable)">Archive</button>
|
||||
<button class="sb-btn danger" id="lbDeleteBtn" onclick="lbDeleteArm()" title="Click once to arm, then again to confirm">Delete</button>
|
||||
@@ -2926,7 +2927,9 @@
|
||||
+ '<div style="position:relative;display:inline-block">'
|
||||
+ ' <button class="sb-btn" id="gestureMenuBtn" onclick="toggleGestureMenu(event)" title="Apply a gesture preset to the skeleton">Gesture ▾</button>'
|
||||
+ ' <div id="gestureMenuPanel" style="display:none;position:absolute;top:100%;left:0;z-index:200;background:#18181b;border:1px solid #2a2a2a;border-radius:6px;padding:4px;box-shadow:0 6px 20px rgba(0,0,0,.7);min-width:150px;margin-top:4px"></div>'
|
||||
+ '</div>';
|
||||
+ '</div>'
|
||||
+ '<button class="sb-btn" onclick="lbPoseFromVideo()" title="Pick a pose from a wireframe video frame">From video</button>'
|
||||
+ '<button class="sb-btn" id="poseGenBtn" onclick="lbGenerateWithPose()" title="Generate the person in this pose (skeleton → Qwen image2 guide)">Generate ▸</button>';
|
||||
viewer.appendChild(bar);
|
||||
_buildGestureMenu();
|
||||
|
||||
@@ -2997,9 +3000,10 @@
|
||||
const g = GESTURE_PRESETS[idx];
|
||||
const W = st.width, H = st.height;
|
||||
st.people[0] = g.kpts.map(([nx, ny, c]) => [nx * W, ny * H, c]);
|
||||
st._gestureName = g.name; // remember for generation
|
||||
_drawPose();
|
||||
document.getElementById('gestureMenuPanel').style.display = 'none';
|
||||
showToast(`Gesture: ${g.name}`, 'success');
|
||||
showToast(`Gesture: ${g.name} — click Generate ▸ to create`, 'success');
|
||||
}
|
||||
|
||||
function _drawPose() {
|
||||
@@ -3098,6 +3102,154 @@
|
||||
if (btn) btn.disabled = false;
|
||||
}
|
||||
|
||||
async function lbGenerateWithPose() {
|
||||
const st = window._poseState;
|
||||
if (!st || !st.people.length) { showToast('No pose overlay active — click Pose first', 'info'); return; }
|
||||
const fname = st.fname;
|
||||
if (!fname) { showToast('No image active', 'info'); return; }
|
||||
const btn = document.getElementById('poseGenBtn');
|
||||
if (btn) { btn.disabled = true; btn.textContent = '…'; }
|
||||
try {
|
||||
const payload = {
|
||||
model_filename: fname,
|
||||
keypoints: st.people[0],
|
||||
width: st.width,
|
||||
height: st.height,
|
||||
gesture_name: st._gestureName || null,
|
||||
seed: -1,
|
||||
};
|
||||
const r = await fetch(`${API}/generate-with-pose`, {
|
||||
method: 'POST', headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
if (!r.ok) { showToast('Generate failed: ' + await r.text(), 'error'); return; }
|
||||
const { job_id } = await r.json();
|
||||
showToast('Pose generation started…', 'info');
|
||||
if (btn) btn.textContent = 'Generating…';
|
||||
const poll = () => setTimeout(async () => {
|
||||
try {
|
||||
const jr = await fetch(`${API}/batch/${job_id}`);
|
||||
if (!jr.ok) { poll(); return; }
|
||||
const job = await jr.json();
|
||||
if (job.status === 'running') { poll(); return; }
|
||||
if (btn) { btn.disabled = false; btn.textContent = 'Generate ▸'; }
|
||||
if (job.status === 'done') {
|
||||
showToast('Pose generation done!', 'success');
|
||||
_followLatestGid = lbCurrentGid;
|
||||
refreshNow();
|
||||
} else {
|
||||
showToast('Pose generation failed: ' + (job.error || 'unknown'), 'error');
|
||||
}
|
||||
} catch (e) { poll(); }
|
||||
}, 2500);
|
||||
poll();
|
||||
} catch (e) {
|
||||
showToast('Generate error: ' + e, 'error');
|
||||
if (btn) { btn.disabled = false; btn.textContent = 'Generate ▸'; }
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Pose from wireframe video ───────────────────────────────────────────
|
||||
// Opens a modal that lists wireframe videos for the current group,
|
||||
// lets the user scrub to a timestamp, then extracts keypoints from that
|
||||
// frame and loads them into the draggable pose overlay.
|
||||
async function lbPoseFromVideo() {
|
||||
const st = window._poseState;
|
||||
if (!st) { showToast('Open Pose first', 'info'); return; }
|
||||
|
||||
// Gather wireframe videos from the current group's images.
|
||||
const videos = (lbNames || []).filter(n => /\.(mp4|webm|mov)$/i.test(n));
|
||||
if (!videos.length) {
|
||||
showToast('No videos in this group — generate a wireframe video first', 'info');
|
||||
return;
|
||||
}
|
||||
|
||||
// Build modal
|
||||
const overlay = document.createElement('div');
|
||||
overlay.id = 'poseVideoModal';
|
||||
overlay.style.cssText = 'position:fixed;inset:0;z-index:900;background:rgba(0,0,0,.8);display:flex;align-items:center;justify-content:center;';
|
||||
overlay.innerHTML = `
|
||||
<div style="background:#18181b;border:1px solid #2a2a2a;border-radius:10px;padding:20px;width:480px;max-width:95vw;box-shadow:0 8px 40px rgba(0,0,0,.8)">
|
||||
<div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:14px">
|
||||
<span style="font-size:14px;font-weight:600">Pick pose from wireframe video</span>
|
||||
<button class="sb-btn" onclick="document.getElementById('poseVideoModal').remove()">✕</button>
|
||||
</div>
|
||||
<div style="margin-bottom:10px">
|
||||
<label style="font-size:11px;color:#aaa;display:block;margin-bottom:4px">Video</label>
|
||||
<select id="pvmVideoSelect" style="width:100%;background:#0f0f0f;color:#e0e0e0;border:1px solid #2a2a2a;border-radius:5px;padding:5px 8px;font-size:12px">
|
||||
${videos.map(v => `<option value="${escHtml(v)}">${escHtml(v)}</option>`).join('')}
|
||||
</select>
|
||||
</div>
|
||||
<div style="margin-bottom:10px">
|
||||
<video id="pvmVideo" style="width:100%;border-radius:6px;background:#000;max-height:260px" preload="metadata"></video>
|
||||
</div>
|
||||
<div style="margin-bottom:14px">
|
||||
<label style="font-size:11px;color:#aaa;display:block;margin-bottom:4px">Scrub to pose frame</label>
|
||||
<input type="range" id="pvmScrubber" min="0" max="100" step="0.1" value="0"
|
||||
style="width:100%;accent-color:#a78bfa">
|
||||
<span id="pvmTimeLabel" style="font-size:11px;color:#888">0.0 s</span>
|
||||
</div>
|
||||
<div style="display:flex;gap:8px;justify-content:flex-end">
|
||||
<button class="sb-btn" onclick="document.getElementById('poseVideoModal').remove()">Cancel</button>
|
||||
<button class="sb-btn primary" id="pvmLoadBtn" onclick="pvmLoadPose()">Load pose</button>
|
||||
</div>
|
||||
</div>`;
|
||||
document.body.appendChild(overlay);
|
||||
|
||||
const vidEl = document.getElementById('pvmVideo');
|
||||
const select = document.getElementById('pvmVideoSelect');
|
||||
const scrubber = document.getElementById('pvmScrubber');
|
||||
const timeLabel = document.getElementById('pvmTimeLabel');
|
||||
|
||||
function pvmSetVideo(filename) {
|
||||
vidEl.src = `${API}/output/${encodeURIComponent(filename)}`;
|
||||
vidEl.load();
|
||||
vidEl.addEventListener('loadedmetadata', () => {
|
||||
const dur = vidEl.duration || 10;
|
||||
scrubber.max = dur;
|
||||
scrubber.step = Math.max(0.033, dur / 500);
|
||||
scrubber.value = 0;
|
||||
timeLabel.textContent = '0.0 s';
|
||||
}, { once: true });
|
||||
}
|
||||
|
||||
scrubber.addEventListener('input', () => {
|
||||
const t = parseFloat(scrubber.value);
|
||||
vidEl.currentTime = t;
|
||||
timeLabel.textContent = t.toFixed(2) + ' s';
|
||||
});
|
||||
|
||||
select.addEventListener('change', () => pvmSetVideo(select.value));
|
||||
pvmSetVideo(videos[0]);
|
||||
|
||||
window.pvmLoadPose = async function() {
|
||||
const video = select.value;
|
||||
const time_s = parseFloat(scrubber.value);
|
||||
const btn = document.getElementById('pvmLoadBtn');
|
||||
btn.disabled = true;
|
||||
btn.textContent = 'Extracting…';
|
||||
try {
|
||||
const r = await fetch(`${API}/pose/from-wireframe`, {
|
||||
method: 'POST', headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ video, time_s }),
|
||||
});
|
||||
if (!r.ok) { showToast('Pose extract failed: ' + await r.text(), 'error'); btn.disabled = false; btn.textContent = 'Load pose'; return; }
|
||||
const d = await r.json();
|
||||
// Rescale keypoints from video dimensions to current image dimensions
|
||||
const scaleX = st.width / d.width;
|
||||
const scaleY = st.height / d.height;
|
||||
st.people[0] = d.keypoints.map(([x, y, c]) => [x * scaleX, y * scaleY, c]);
|
||||
st._gestureName = null; // came from real video, not a named preset
|
||||
_drawPose();
|
||||
document.getElementById('poseVideoModal').remove();
|
||||
showToast(`Pose loaded from ${video} @ ${time_s.toFixed(2)}s`, 'success');
|
||||
} catch (e) {
|
||||
showToast('Pose extract error: ' + e, 'error');
|
||||
btn.disabled = false; btn.textContent = 'Load pose';
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async function lbFindSimilarFaces() {
|
||||
if (!lbCurrentGid) return;
|
||||
try {
|
||||
@@ -3257,13 +3409,14 @@
|
||||
|
||||
const bar = document.createElement('div');
|
||||
bar.id = 'cropBar';
|
||||
// Anchored at the TOP so the buttons never block cropping to the bottom edge.
|
||||
bar.style.cssText = 'position:absolute;top:0;left:0;right:0;display:flex;gap:8px;padding:8px;background:rgba(0,0,0,0.75);z-index:101;justify-content:flex-end;align-items:center;';
|
||||
// pointer-events:none on the container so the canvas underneath captures drags
|
||||
// from the very top of the image. Individual interactive controls opt back in.
|
||||
bar.style.cssText = 'position:absolute;top:0;left:0;right:0;display:flex;gap:8px;padding:8px;background:rgba(0,0,0,0.75);z-index:101;justify-content:flex-end;align-items:center;pointer-events:none;';
|
||||
bar.innerHTML = '<span style="flex:1;font-size:11px;color:#aaa">Drag to select crop area</span>'
|
||||
+ '<label style="display:flex;align-items:center;gap:4px;font-size:11px;color:#ccc;cursor:pointer" title="Keep the original and crop a copy instead">'
|
||||
+ '<label style="display:flex;align-items:center;gap:4px;font-size:11px;color:#ccc;cursor:pointer;pointer-events:auto" title="Keep the original and crop a copy instead">'
|
||||
+ '<input type="checkbox" id="cropAsCopy" checked style="cursor:pointer">Save as copy</label>'
|
||||
+ '<button class="sb-btn" onclick="cancelManualCrop()">Cancel</button>'
|
||||
+ '<button class="sb-btn primary" id="cropConfirmBtn" disabled onclick="confirmManualCrop()">Crop</button>';
|
||||
+ '<button class="sb-btn" style="pointer-events:auto" onclick="cancelManualCrop()">Cancel</button>'
|
||||
+ '<button class="sb-btn primary" style="pointer-events:auto" id="cropConfirmBtn" disabled onclick="confirmManualCrop()">Crop</button>';
|
||||
viewer.appendChild(bar);
|
||||
|
||||
const st = window._cropState = { canvas, viewer, img, fname, x1: 0, y1: 0, x2: 0, y2: 0, dragging: false };
|
||||
@@ -3435,6 +3588,32 @@
|
||||
if (btn) btn.disabled = false;
|
||||
}
|
||||
|
||||
async function lbMakeVideo() {
|
||||
// Use Shift+Click selected refs if any, otherwise all images in the current group.
|
||||
let filenames;
|
||||
if (_sbRefIndices.length >= 2) {
|
||||
filenames = _sbRefIndices.map(i => lbNames[i]).filter(n => n && !isVideo(n));
|
||||
if (filenames.length < 2) { showToast('Need at least 2 non-video images selected (Shift+Click filmstrip)', 'info'); return; }
|
||||
} else {
|
||||
filenames = lbNames.filter(n => n && !isVideo(n));
|
||||
if (filenames.length < 2) { showToast('Need at least 2 images in this group', 'info'); return; }
|
||||
}
|
||||
const btn = document.getElementById('lbMakeVideoBtn');
|
||||
if (btn) { btn.disabled = true; btn.textContent = 'Creating…'; }
|
||||
try {
|
||||
const r = await fetch(`${API}/generate-video`, {
|
||||
method: 'POST', headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ filenames, fps: 8, loop: true }),
|
||||
});
|
||||
if (!r.ok) { showToast('Video failed: ' + await r.text(), 'error'); return; }
|
||||
const d = await r.json();
|
||||
showToast(`Video created: ${d.output}`, 'success');
|
||||
_followLatestGid = lbCurrentGid;
|
||||
refreshNow();
|
||||
} catch (e) { showToast('Video error: ' + e, 'error'); }
|
||||
if (btn) { btn.disabled = false; btn.textContent = 'Make video'; }
|
||||
}
|
||||
|
||||
// Force the studio image (filmstrip + main viewer) to reload from disk.
|
||||
function _bustStudioImage(fname) {
|
||||
const t = Date.now();
|
||||
|
||||
@@ -1576,6 +1576,88 @@ def trim_wireframe(req: TrimRequest):
|
||||
return {'output_name': out_name, 'start': req.start, 'end': req.end}
|
||||
|
||||
|
||||
class VideoStitchRequest(BaseModel):
|
||||
filenames: list[str] # images from output_dir, in desired frame order
|
||||
fps: float = 8.0
|
||||
loop: bool = True # append reversed sequence for a ping-pong loop
|
||||
output_name: str | None = None
|
||||
|
||||
|
||||
@app.post("/generate-video")
|
||||
def generate_video(req: VideoStitchRequest):
|
||||
"""Stitch a list of output images into a short looping MP4 using ffmpeg."""
|
||||
import subprocess, tempfile, textwrap
|
||||
if len(req.filenames) < 2:
|
||||
raise HTTPException(400, "Need at least 2 images")
|
||||
if not (0.1 <= req.fps <= 60):
|
||||
raise HTTPException(400, "fps must be between 0.1 and 60")
|
||||
|
||||
output_dir = _load_output_dir()
|
||||
|
||||
# Validate + collect paths
|
||||
paths = []
|
||||
for fn in req.filenames:
|
||||
p = os.path.join(output_dir, fn)
|
||||
if not os.path.exists(p):
|
||||
raise HTTPException(404, f"Image not found: {fn}")
|
||||
paths.append(p)
|
||||
|
||||
# Ping-pong loop: forward + reversed (drop first/last to avoid duplicate frames at seam)
|
||||
if req.loop and len(paths) > 1:
|
||||
paths = paths + list(reversed(paths[1:-1])) if len(paths) > 2 else paths + list(reversed(paths))
|
||||
|
||||
ts = time.strftime("%Y%m%d_%H%M%S")
|
||||
base = naming.get_base_name(req.filenames[0])
|
||||
stem = os.path.splitext(base)[0]
|
||||
out_name = req.output_name or f"{ts}_vid_{stem}.mp4"
|
||||
if not out_name.lower().endswith(".mp4"):
|
||||
out_name += ".mp4"
|
||||
out_path = os.path.join(output_dir, out_name)
|
||||
|
||||
# Build ffmpeg concat list in a temp file
|
||||
duration = 1.0 / req.fps
|
||||
with tempfile.NamedTemporaryFile("w", suffix=".txt", delete=False) as tf:
|
||||
for p in paths:
|
||||
tf.write(f"file '{p}'\nduration {duration:.6f}\n")
|
||||
# ffmpeg concat needs the last entry without a duration
|
||||
tf.write(f"file '{paths[-1]}'\n")
|
||||
concat_path = tf.name
|
||||
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["ffmpeg", "-y",
|
||||
"-f", "concat", "-safe", "0", "-i", concat_path,
|
||||
"-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", # ensure even dimensions
|
||||
"-c:v", "libx264", "-pix_fmt", "yuv420p",
|
||||
"-movflags", "+faststart",
|
||||
out_path],
|
||||
capture_output=True, timeout=120,
|
||||
)
|
||||
finally:
|
||||
os.unlink(concat_path)
|
||||
|
||||
if r.returncode != 0:
|
||||
raise HTTPException(500, f"ffmpeg error: {r.stderr.decode(errors='replace')[:600]}")
|
||||
|
||||
# Register in DB so it shows up in the gallery
|
||||
person = database.get_person(req.filenames[0])
|
||||
group_id = person[1] if person and person[1] else naming.get_base_name(req.filenames[0])
|
||||
try:
|
||||
next_order = database.get_next_sort_order(group_id)
|
||||
database.upsert_person(
|
||||
out_name, filepath=out_path,
|
||||
group_id=group_id,
|
||||
sort_order=next_order,
|
||||
content_type="video",
|
||||
source_refs=json.dumps(req.filenames),
|
||||
)
|
||||
_invalidate_static()
|
||||
except Exception as e:
|
||||
print(f"[generate-video] DB error: {e}")
|
||||
|
||||
return {"output": out_name, "frames": len(paths), "fps": req.fps}
|
||||
|
||||
|
||||
class FrameExtractRequest(BaseModel):
|
||||
video_name: str
|
||||
time: float = 0.0
|
||||
@@ -3287,6 +3369,418 @@ def similar_pose_from_keypoints(req: PoseSimilarRequest):
|
||||
return {"similar": _rank_similar_poses(query, req.limit)}
|
||||
|
||||
|
||||
# --- pose-guided generation ---------------------------------------------------
|
||||
# Renders COCO-17 keypoints as an OpenPose-style skeleton image and passes it as
|
||||
# image2 to Qwen so the VLM can match the body pose while preserving appearance.
|
||||
|
||||
# Color per POSE_SKELETON entry (18 bones in POSE_SKELETON order).
|
||||
_OPENPOSE_BONE_COLORS = [
|
||||
(0, 80, 255), # 5→7 L upper arm
|
||||
(0, 130, 255), # 7→9 L lower arm
|
||||
(255, 50, 0), # 6→8 R upper arm
|
||||
(255, 110, 0), # 8→10 R lower arm
|
||||
(255, 200, 0), # 11→13 L upper leg
|
||||
(240, 240, 0), # 13→15 L lower leg
|
||||
(100, 255, 0), # 12→14 R upper leg
|
||||
( 50, 220, 0), # 14→16 R lower leg
|
||||
(200, 0, 220), # 5→6 shoulders
|
||||
(200, 0, 220), # 11→12 hips
|
||||
(180, 60, 180), # 5→11 L torso side
|
||||
( 60, 180, 180), # 6→12 R torso side
|
||||
( 0, 230, 230), # 0→1 nose–L eye
|
||||
( 0, 230, 230), # 0→2 nose–R eye
|
||||
( 0, 180, 180), # 1→3 L eye–ear
|
||||
( 0, 180, 180), # 2→4 R eye–ear
|
||||
( 80, 80, 255), # 0→5 head–L shoulder
|
||||
(255, 80, 80), # 0→6 head–R shoulder
|
||||
]
|
||||
|
||||
# Per-joint hue (HSV-like, converted to RGB for PIL):
|
||||
def _joint_color(idx: int) -> tuple[int, int, int]:
|
||||
h = idx / 17.0 * 6.0
|
||||
x = 1 - abs(h % 2 - 1)
|
||||
if h < 1: r, g, b = 1, x, 0
|
||||
elif h < 2: r, g, b = x, 1, 0
|
||||
elif h < 3: r, g, b = 0, 1, x
|
||||
elif h < 4: r, g, b = 0, x, 1
|
||||
elif h < 5: r, g, b = x, 0, 1
|
||||
else: r, g, b = 1, 0, x
|
||||
return (int(r * 255), int(g * 255), int(b * 255))
|
||||
|
||||
|
||||
def _render_openpose_image(keypoints: list, width: int, height: int) -> Image.Image:
|
||||
"""Draw COCO-17 keypoints as a white stick-figure on a vivid magenta background.
|
||||
|
||||
Vivid background (not black/dark) prevents Qwen from treating the skeleton as
|
||||
a real scene element and from blending the bone colors into the generated output.
|
||||
White lines on magenta read unambiguously as a technical reference diagram.
|
||||
"""
|
||||
from PIL import ImageDraw
|
||||
MIN_CONF = 0.25
|
||||
cap_w = min(width, 896)
|
||||
cap_h = min(height, 1152)
|
||||
scale = min(cap_w / max(width, 1), cap_h / max(height, 1))
|
||||
out_w, out_h = int(width * scale), int(height * scale)
|
||||
|
||||
img = Image.new("RGB", (out_w, out_h), color=(200, 0, 180)) # vivid magenta
|
||||
draw = ImageDraw.Draw(img)
|
||||
|
||||
lw = max(4, out_w // 80) # line width scales with image
|
||||
jr = max(5, out_w // 70) # joint radius
|
||||
|
||||
kpts = keypoints # [[x, y, conf], ...]
|
||||
|
||||
# bones — white so they stand out on the magenta background without
|
||||
# adding any colors that Qwen might reproduce in the generated image
|
||||
for a, b in POSE_SKELETON:
|
||||
if a >= len(kpts) or b >= len(kpts):
|
||||
continue
|
||||
pa, pb = kpts[a], kpts[b]
|
||||
if pa[2] < MIN_CONF or pb[2] < MIN_CONF:
|
||||
continue
|
||||
ax, ay = int(pa[0] * scale), int(pa[1] * scale)
|
||||
bx, by = int(pb[0] * scale), int(pb[1] * scale)
|
||||
draw.line([(ax, ay), (bx, by)], fill=(255, 255, 255), width=lw)
|
||||
|
||||
# joints — light yellow circles with dark outline for visibility
|
||||
for kp in kpts:
|
||||
if kp[2] < MIN_CONF:
|
||||
continue
|
||||
x, y = int(kp[0] * scale), int(kp[1] * scale)
|
||||
draw.ellipse([(x - jr, y - jr), (x + jr, y + jr)],
|
||||
fill=(255, 240, 100), outline=(60, 0, 60), width=max(1, lw // 2))
|
||||
|
||||
return img
|
||||
|
||||
|
||||
class PoseFromWireframeRequest(BaseModel):
|
||||
video: str # filename of a wireframe video in the output dir
|
||||
time_s: float = 0.0 # timestamp in seconds to extract the frame from
|
||||
|
||||
|
||||
@app.post("/pose/from-wireframe")
|
||||
def pose_from_wireframe(req: PoseFromWireframeRequest):
|
||||
"""Extract a frame from a wireframe video, run pose estimation, return COCO-17 keypoints.
|
||||
|
||||
This is the pose-discovery tool: scrub through a wireframe video to find a frame
|
||||
that shows the desired pose, then load those keypoints into the pose overlay without
|
||||
ever passing the wireframe image to Qwen.
|
||||
"""
|
||||
import tempfile
|
||||
import subprocess as sp
|
||||
|
||||
output_dir = _load_output_dir()
|
||||
video_path = os.path.join(output_dir, req.video)
|
||||
if not os.path.exists(video_path):
|
||||
raise HTTPException(404, f"Video not found: {req.video}")
|
||||
|
||||
est = _load_pose_estimator()
|
||||
if not est:
|
||||
raise HTTPException(501, "No pose estimator installed. Try: pip install rtmlib onnxruntime")
|
||||
infer, backend = est
|
||||
|
||||
# Extract one frame at the requested timestamp with ffmpeg.
|
||||
with tempfile.NamedTemporaryFile(suffix=".jpg", delete=False) as tf:
|
||||
frame_path = tf.name
|
||||
try:
|
||||
result = sp.run(
|
||||
[
|
||||
"/usr/bin/ffmpeg", "-y",
|
||||
"-ss", str(req.time_s),
|
||||
"-i", video_path,
|
||||
"-frames:v", "1",
|
||||
"-q:v", "2",
|
||||
frame_path,
|
||||
],
|
||||
capture_output=True, timeout=30,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise HTTPException(500, f"ffmpeg frame extract failed: {result.stderr.decode()[:300]}")
|
||||
|
||||
pil = Image.open(frame_path).convert("RGB")
|
||||
finally:
|
||||
try:
|
||||
os.unlink(frame_path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
try:
|
||||
people = infer(pil)
|
||||
except Exception as e:
|
||||
raise HTTPException(500, f"Pose estimation failed: {e}")
|
||||
|
||||
best = _best_person(people)
|
||||
if best is None:
|
||||
raise HTTPException(422, "No person detected in the extracted frame")
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"backend": backend,
|
||||
"width": pil.width,
|
||||
"height": pil.height,
|
||||
"names": POSE_KEYPOINT_NAMES,
|
||||
"skeleton": POSE_SKELETON,
|
||||
"keypoints": best,
|
||||
"time_s": req.time_s,
|
||||
}
|
||||
|
||||
|
||||
class PoseGenRequest(BaseModel):
|
||||
model_filename: str
|
||||
keypoints: list[list[float]] # 17 × [x, y, conf] in original image pixels
|
||||
width: int # original image dimensions the keypoints came from
|
||||
height: int
|
||||
gesture_name: str | None = None # preset name for prompt enrichment and tagging
|
||||
prompt: str | None = None # full override; auto-built if None
|
||||
seed: int = -1
|
||||
extra_filename: str | None = None # optional image3 extra reference
|
||||
|
||||
|
||||
def _keypoints_to_pose_text(kpts: list, width: int, height: int) -> str:
|
||||
"""Convert COCO-17 keypoints to a natural-language pose description for Qwen."""
|
||||
MIN_CONF = 0.25
|
||||
W, H = max(width, 1), max(height, 1)
|
||||
|
||||
def vis(idx):
|
||||
return idx < len(kpts) and kpts[idx][2] >= MIN_CONF
|
||||
|
||||
def pt(idx):
|
||||
return (kpts[idx][0] / W, kpts[idx][1] / H) if vis(idx) else None
|
||||
|
||||
parts = []
|
||||
|
||||
# --- head orientation ---
|
||||
nose, l_ear, r_ear = pt(0), pt(3), pt(4)
|
||||
if l_ear and r_ear:
|
||||
ear_mid_x = (l_ear[0] + r_ear[0]) / 2
|
||||
if nose:
|
||||
dx = nose[0] - ear_mid_x
|
||||
if dx < -0.05:
|
||||
parts.append("head turned to the left")
|
||||
elif dx > 0.05:
|
||||
parts.append("head turned to the right")
|
||||
else:
|
||||
parts.append("head facing forward")
|
||||
elif l_ear and not r_ear:
|
||||
parts.append("head turned strongly to the right")
|
||||
elif r_ear and not l_ear:
|
||||
parts.append("head turned strongly to the left")
|
||||
|
||||
# --- body rotation (shoulders vs hips) ---
|
||||
l_sh, r_sh = pt(5), pt(6)
|
||||
l_hip, r_hip = pt(11), pt(12)
|
||||
if l_sh and r_sh:
|
||||
sh_w = abs(r_sh[0] - l_sh[0])
|
||||
if l_hip and r_hip:
|
||||
hip_w = abs(r_hip[0] - l_hip[0])
|
||||
ratio = sh_w / max(hip_w, 0.01)
|
||||
if ratio < 0.5:
|
||||
parts.append("body rotated ~90° (side profile)")
|
||||
elif ratio < 0.75:
|
||||
parts.append("body rotated ~45° (three-quarter view)")
|
||||
else:
|
||||
parts.append("body facing forward")
|
||||
|
||||
# --- arm positions ---
|
||||
for side, sh_i, el_i, wr_i in [("left", 5, 7, 9), ("right", 6, 8, 10)]:
|
||||
sh, el, wr = pt(sh_i), pt(el_i), pt(wr_i)
|
||||
if not sh:
|
||||
continue
|
||||
if wr:
|
||||
dy = sh[1] - wr[1] # positive = wrist above shoulder
|
||||
dx = abs(wr[0] - sh[0])
|
||||
if dy > 0.15:
|
||||
if el and pt(el_i):
|
||||
el_dy = sh[1] - el[1]
|
||||
if el_dy > 0.05:
|
||||
parts.append(f"{side} arm raised straight up")
|
||||
else:
|
||||
parts.append(f"{side} arm raised with elbow bent")
|
||||
else:
|
||||
parts.append(f"{side} arm raised above shoulder")
|
||||
elif dx > 0.25:
|
||||
parts.append(f"{side} arm extended sideways")
|
||||
elif el:
|
||||
el_dx = abs(el[0] - sh[0])
|
||||
if el_dx > 0.15 and abs(wr[1] - sh[1]) < 0.1:
|
||||
parts.append(f"{side} arm extended forward")
|
||||
else:
|
||||
parts.append(f"{side} arm at side")
|
||||
else:
|
||||
parts.append(f"{side} arm at side")
|
||||
elif el:
|
||||
el_dy = sh[1] - el[1]
|
||||
if el_dy > 0.1:
|
||||
parts.append(f"{side} elbow raised")
|
||||
|
||||
# --- leg positions ---
|
||||
for side, hip_i, kn_i, an_i in [("left", 11, 13, 15), ("right", 12, 14, 16)]:
|
||||
hip, kn, an = pt(hip_i), pt(kn_i), pt(an_i)
|
||||
if not hip:
|
||||
continue
|
||||
if an:
|
||||
dx = abs(an[0] - hip[0])
|
||||
dy = an[1] - hip[1] # positive = ankle below hip (normal standing)
|
||||
if dy < 0.1:
|
||||
parts.append(f"{side} leg raised / knee up")
|
||||
elif dx > 0.2:
|
||||
parts.append(f"{side} leg stepped out to the side")
|
||||
elif kn:
|
||||
kn_dx = abs(kn[0] - hip[0])
|
||||
if kn_dx > 0.15:
|
||||
parts.append(f"{side} leg bent / lunging")
|
||||
else:
|
||||
parts.append(f"{side} leg straight, standing")
|
||||
else:
|
||||
parts.append(f"{side} leg straight")
|
||||
|
||||
if not parts:
|
||||
return "standing neutral pose"
|
||||
return ", ".join(parts)
|
||||
|
||||
|
||||
def _pad_for_pose(pil: Image.Image, keypoints: list, margin: float = 0.15) -> tuple[Image.Image, tuple[int,int,int,int]]:
|
||||
"""Expand the image so target keypoints fit within the canvas.
|
||||
|
||||
Only expands when keypoints are actually outside [0,W]×[0,H]. Expansion is
|
||||
symmetric on each axis (equal left/right, equal top/bottom) so the person
|
||||
stays centred on the new canvas. Fill is black to match PIL's RGBA→RGB fill.
|
||||
"""
|
||||
W, H = pil.size
|
||||
all_kps = [kp for kp in keypoints if len(kp) >= 2]
|
||||
if not all_kps:
|
||||
return pil, (0, 0, 0, 0)
|
||||
|
||||
xs = [kp[0] for kp in all_kps]
|
||||
ys = [kp[1] for kp in all_kps]
|
||||
|
||||
# How far each side actually exceeds the image bounds (0 if within bounds).
|
||||
over_l = max(0.0, -min(xs))
|
||||
over_r = max(0.0, max(xs) - W)
|
||||
over_t = max(0.0, -min(ys))
|
||||
over_b = max(0.0, max(ys) - H)
|
||||
|
||||
# If nothing is outside, no padding needed at all.
|
||||
if over_l == over_r == over_t == over_b == 0:
|
||||
return pil, (0, 0, 0, 0)
|
||||
|
||||
# Symmetric expansion per axis: take the larger overhang and add margin.
|
||||
px = int(max(over_l, over_r) + margin * W)
|
||||
py = int(max(over_t, over_b) + margin * H)
|
||||
|
||||
new_w, new_h = W + 2 * px, H + 2 * py
|
||||
canvas = Image.new("RGB", (new_w, new_h), color=(0, 0, 0))
|
||||
canvas.paste(pil, (px, py))
|
||||
print(f"[pose-gen] padded {W}×{H} → {new_w}×{new_h} (±{px}px x, ±{py}px y)")
|
||||
return canvas, (px, py, px, py)
|
||||
|
||||
|
||||
def _pose_gen_worker(job_id: str, model_filename: str, prompt: str, seed: int,
|
||||
keypoints: list | None = None,
|
||||
gesture_name: str | None = None,
|
||||
extra_filename: str | None = None):
|
||||
output_dir = _load_output_dir()
|
||||
try:
|
||||
model_path = os.path.join(output_dir, model_filename)
|
||||
model_pil = Image.open(model_path).convert("RGB")
|
||||
|
||||
# Auto-pad if the target pose would extend outside the current image bounds.
|
||||
if keypoints:
|
||||
model_pil, _padding = _pad_for_pose(model_pil, keypoints)
|
||||
|
||||
# No skeleton image — pose is encoded entirely in the text prompt.
|
||||
extra_images = None
|
||||
if extra_filename:
|
||||
ep = os.path.join(output_dir, extra_filename)
|
||||
extra_images = [Image.open(ep).convert("RGB")]
|
||||
|
||||
png_bytes = _run_pipeline(model_pil, prompt, seed, MAX_AREA, extra_images=extra_images)
|
||||
ts = time.strftime("%Y%m%d_%H%M%S")
|
||||
base_name = naming.get_base_name(model_filename)
|
||||
if not base_name.lower().endswith(".png"):
|
||||
base_name = os.path.splitext(base_name)[0] + ".png"
|
||||
out_name = f"{ts}_pose_{base_name}"
|
||||
out_path = os.path.join(output_dir, out_name)
|
||||
with open(out_path, "wb") as f:
|
||||
f.write(png_bytes)
|
||||
|
||||
person = database.get_person(model_filename)
|
||||
group_id = person[1] if person and person[1] else naming.get_base_name(model_filename)
|
||||
|
||||
embedding = embeddings.generate_embedding(out_path)
|
||||
next_order = database.get_next_sort_order(group_id)
|
||||
refs = [model_filename]
|
||||
if gesture_name:
|
||||
refs.append(f"gesture:{gesture_name}")
|
||||
database.upsert_person(
|
||||
out_name, filepath=out_path, embedding=embedding,
|
||||
group_id=group_id, prompt=prompt,
|
||||
pose=gesture_name or "custom",
|
||||
sort_order=next_order,
|
||||
source_refs=json.dumps(refs),
|
||||
)
|
||||
|
||||
jobs[job_id]["status"] = "done"
|
||||
jobs[job_id]["output"] = out_name
|
||||
_invalidate_static()
|
||||
except Exception as e:
|
||||
print(f"[pose-gen] error: {e}")
|
||||
jobs[job_id]["status"] = "error"
|
||||
jobs[job_id]["error"] = str(e)
|
||||
|
||||
|
||||
@app.post("/pose/render")
|
||||
def render_pose_image(req: PoseSimilarRequest):
|
||||
"""Render the supplied keypoints as an OpenPose-style skeleton PNG (base64) — for preview only."""
|
||||
import base64
|
||||
skeleton = _render_openpose_image(req.keypoints, req.width or 512, req.height or 768)
|
||||
buf = io.BytesIO()
|
||||
skeleton.save(buf, format="PNG")
|
||||
return {"image": base64.b64encode(buf.getvalue()).decode()}
|
||||
|
||||
|
||||
@app.post("/generate-with-pose")
|
||||
def generate_with_pose(req: PoseGenRequest):
|
||||
"""Generate the person in a specific body pose. Pose is encoded as text — no skeleton image
|
||||
is passed to Qwen to avoid wireframe bleed-through in the output."""
|
||||
output_dir = _load_output_dir()
|
||||
model_path = os.path.join(output_dir, req.model_filename)
|
||||
if not os.path.exists(model_path):
|
||||
raise HTTPException(404, f"Model image not found: {req.model_filename}")
|
||||
if req.extra_filename:
|
||||
ep = os.path.join(output_dir, req.extra_filename)
|
||||
if not os.path.exists(ep):
|
||||
raise HTTPException(404, f"Extra reference not found: {req.extra_filename}")
|
||||
|
||||
if req.prompt:
|
||||
prompt = req.prompt
|
||||
else:
|
||||
# Convert skeleton keypoints to natural-language description
|
||||
pose_text = _keypoints_to_pose_text(req.keypoints, req.width, req.height)
|
||||
if req.gesture_name:
|
||||
pose_text = f'{req.gesture_name} gesture: {pose_text}'
|
||||
prompt = (
|
||||
f"Change the body pose of the person in image 1 to: {pose_text}. "
|
||||
"Keep the person's face, hair, skin tone, clothing, and body proportions "
|
||||
"exactly as they are — only the limb positions and body orientation change. "
|
||||
"Transparent background."
|
||||
)
|
||||
|
||||
job_id = uuid.uuid4().hex[:8]
|
||||
jobs[job_id] = {"status": "running", "type": "pose_gen", "total": 1, "done": 0, "failed": 0}
|
||||
threading.Thread(
|
||||
target=_pose_gen_worker,
|
||||
args=(job_id, req.model_filename, prompt, req.seed),
|
||||
kwargs={
|
||||
"keypoints": req.keypoints,
|
||||
"gesture_name": req.gesture_name,
|
||||
"extra_filename": req.extra_filename,
|
||||
},
|
||||
daemon=True,
|
||||
).start()
|
||||
return {"job_id": job_id, "model": req.model_filename}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import uvicorn
|
||||
uvicorn.run(app, host=os.environ.get("HOST", "0.0.0.0"),
|
||||
|
||||
Reference in New Issue
Block a user