From 27e8a09bc11da9730cb00bf2540e9b0d3841011e Mon Sep 17 00:00:00 2001 From: mike Date: Thu, 25 Jun 2026 03:31:54 +0200 Subject: [PATCH] =?UTF-8?q?dphn/Dolphin3.0-Mistral-24B=20is=20the=20ungate?= =?UTF-8?q?d=20mirror=20of=20the=20Dolphin=203.0=20Mistral=2024B=20?= =?UTF-8?q?=E2=80=94=20exactly=20what=20you=20asked=20for.=20It's=20~48GB?= =?UTF-8?q?=20fp16,=20which=20needs=20GPU+CPU=20split=20(device=5Fmap=3D"a?= =?UTF-8?q?uto"=20with=2032GB=20on=20GPU,=20~16GB=20in=20RAM).=20Let=20me?= =?UTF-8?q?=20kick=20off=20the=20download=20and=20update=20the=20service?= =?UTF-8?q?=20in=20parallel.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backlog.md | 14 +++- tour-comfy/car.html | 155 ++++++++++++++++++++++++++++++++++++++++- tour-comfy/edit_api.py | 50 ++++++++++++- 3 files changed, 214 insertions(+), 5 deletions(-) diff --git a/backlog.md b/backlog.md index 098dea7..90606b8 100644 --- a/backlog.md +++ b/backlog.md @@ -19,6 +19,13 @@ Pose tools (rtmlib / RTMPose, 2D): - ✅ Pose skeleton preview overlay in studio - ✅ Draggable joints to refine/explore a pose - ✅ Find similar poses across the library (translation/scale/mirror-invariant; `poses_index.json`) +- ✅ Gesture presets in pose toolbar (T-pose, wave, arms raised, hands on hips, turn ±45°) +- ✅ "Orbit all" button in Generate tab selects all 6 absolute camera angles at once + +Scenery: +- ✅ source_refs now stores all references: person image + video name + extra ref image +- ✅ /scenery/library endpoint — all scenery grouped by source video +- ✅ "Previous scenery" panel in Scenery tab — browse & reuse scenery from any group, grouped by video ## Open @@ -55,4 +62,9 @@ generating poses themself should use the adviced dimensions rather than the base introduce a like/dislike base system for images/group to determine the sort order (and more) - would also dislike the pose -creating scenery should keep both video-frame + ref images as references, we only see 1 image now. +creating scenery should keep both video-frame + ref images as references, we only see 1 image now. ✅ fixed + +gesture ideas for future (require ControlNet / OpenPose conditioning): +- animate limbs along keyframe curves (true motion) +- 3D-aware rotation / "circling around" with depth estimation +- camera-orbit batch with consistent lighting across views diff --git a/tour-comfy/car.html b/tour-comfy/car.html index 6634257..4f1c5a5 100644 --- a/tour-comfy/car.html +++ b/tour-comfy/car.html @@ -2922,8 +2922,13 @@ bar.style.cssText = 'position:absolute;top:8px;left:8px;display:flex;gap:6px;z-index:102;'; bar.innerHTML = '' - + ''; + + '' + + '
' + + ' ' + + ' ' + + '
'; viewer.appendChild(bar); + _buildGestureMenu(); canvas.addEventListener('mousedown', _posePointerDown); canvas.addEventListener('mousemove', _posePointerMove); @@ -2933,6 +2938,70 @@ if (btn) btn.classList.add('primary'); } + // --- Gesture presets: normalized [x,y,conf] per COCO-17 keypoint --- + // Defined in [0,1] image space for a portrait-oriented standing figure. + // Applied to the actual image by scaling to image pixel dimensions. + const GESTURE_PRESETS = [ + { name: 'T-pose', desc: 'Arms fully extended sideways', + kpts: [[.50,.07,1],[.47,.065,1],[.53,.065,1],[.43,.08,1],[.57,.08,1], + [.25,.22,1],[.75,.22,1],[.10,.22,1],[.90,.22,1],[-.02,.22,1],[1.02,.22,1], + [.40,.52,1],[.60,.52,1],[.40,.72,1],[.60,.72,1],[.40,.93,1],[.60,.93,1]] }, + { name: 'Wave right', desc: 'Right arm raised and waving', + kpts: [[.50,.07,1],[.47,.065,1],[.53,.065,1],[.43,.08,1],[.57,.08,1], + [.35,.22,1],[.65,.22,1],[.20,.38,1],[.78,.10,1],[.22,.52,1],[.70,.04,1], + [.40,.52,1],[.60,.52,1],[.40,.72,1],[.60,.72,1],[.40,.93,1],[.60,.93,1]] }, + { name: 'Arms raised', desc: 'Both arms raised above head', + kpts: [[.50,.07,1],[.47,.065,1],[.53,.065,1],[.43,.08,1],[.57,.08,1], + [.35,.22,1],[.65,.22,1],[.22,.08,1],[.78,.08,1],[.18,.02,1],[.82,.02,1], + [.40,.52,1],[.60,.52,1],[.40,.72,1],[.60,.72,1],[.40,.93,1],[.60,.93,1]] }, + { name: 'Hands on hips', desc: 'Hands resting on hips', + kpts: [[.50,.07,1],[.47,.065,1],[.53,.065,1],[.43,.08,1],[.57,.08,1], + [.35,.22,1],[.65,.22,1],[.28,.40,1],[.72,.40,1],[.35,.52,1],[.65,.52,1], + [.38,.52,1],[.62,.52,1],[.38,.72,1],[.62,.72,1],[.38,.93,1],[.62,.93,1]] }, + { name: 'Turn left 45°', desc: 'Body turned ~45° to the left', + kpts: [[.45,.07,1],[.44,.065,1],[.50,.065,1],[.41,.08,1],[.52,.08,1], + [.30,.22,1],[.58,.22,1],[.18,.38,1],[.65,.30,1],[.12,.52,1],[.70,.22,1], + [.36,.52,1],[.58,.52,1],[.36,.72,1],[.58,.72,1],[.36,.93,1],[.58,.93,1]] }, + { name: 'Turn right 45°', desc: 'Body turned ~45° to the right', + kpts: [[.55,.07,1],[.50,.065,1],[.56,.065,1],[.48,.08,1],[.59,.08,1], + [.42,.22,1],[.70,.22,1],[.35,.30,1],[.82,.38,1],[.30,.22,1],[.88,.52,1], + [.42,.52,1],[.64,.52,1],[.42,.72,1],[.64,.72,1],[.42,.93,1],[.64,.93,1]] }, + ]; + + function _buildGestureMenu() { + const panel = document.getElementById('gestureMenuPanel'); + if (!panel) return; + panel.innerHTML = GESTURE_PRESETS.map((g, i) => + `
${escHtml(g.name)}
` + ).join(''); + } + + function toggleGestureMenu(e) { + e.stopPropagation(); + const p = document.getElementById('gestureMenuPanel'); + if (!p) return; + const open = p.style.display !== 'none'; + p.style.display = open ? 'none' : 'block'; + if (!open) { + const close = (ev) => { if (!p.contains(ev.target)) { p.style.display='none'; document.removeEventListener('click',close); } }; + setTimeout(() => document.addEventListener('click', close), 0); + } + } + + function applyGesturePreset(idx) { + const st = window._poseState; + if (!st || !st.people.length) { showToast('No pose overlay active — click Pose first', 'info'); return; } + const g = GESTURE_PRESETS[idx]; + const W = st.width, H = st.height; + st.people[0] = g.kpts.map(([nx, ny, c]) => [nx * W, ny * H, c]); + _drawPose(); + document.getElementById('gestureMenuPanel').style.display = 'none'; + showToast(`Gesture: ${g.name}`, 'success'); + } + function _drawPose() { const st = window._poseState; if (!st) return; const { ctx, canvas, scale, offX, offY } = st; @@ -4506,7 +4575,12 @@ // Camera angles section — absolute positions const absAngles = CAMERA_ANGLES.filter(a => !a.relative); const relAngles = CAMERA_ANGLES.filter(a => a.relative); - let html = '
Camera — absolute
'; + const ORBIT_NAMES = ['Front', '¾ Left', '¾ Right', 'Side L', 'Side R', 'Back']; + const allOrbitSel = ORBIT_NAMES.every(n => _sbSelectedAngles.has(n)); + let html = `
+ Camera — absolute + +
`; html += absAngles.map(a => { const sel = _sbSelectedAngles.has(a.name) ? ' selected' : ''; const done = donePoses.has(a.name) ? ' done' : ''; @@ -4803,6 +4877,14 @@ renderSidebarGenerate(); } + function sbToggleOrbitAll() { + const ORBIT_NAMES = ['Front', '¾ Left', '¾ Right', 'Side L', 'Side R', 'Back']; + const allSel = ORBIT_NAMES.every(n => _sbSelectedAngles.has(n)); + if (allSel) { ORBIT_NAMES.forEach(n => _sbSelectedAngles.delete(n)); } + else { ORBIT_NAMES.forEach(n => _sbSelectedAngles.add(n)); } + renderSidebarGenerate(); + } + function setSbRefMode(mode) { _sbRefMode = mode; renderSidebarGenerate(); @@ -5386,6 +5468,8 @@ let _sceneVideo = null, _sceneDuration = 0, _sceneFrameBytes = null, _sceneJobPollTimer = null; let _sceneGridOpen = false; // whether the video picker grid is expanded + let _sceneLibOpen = false; // whether "previous scenery" library panel is expanded + let _sceneLib = null; // cached { groups, ungrouped } from /scenery/library // 3 positional reference slots: 0=Background(image1), 1=Person(image2), 2=Extra(image3). // Each entry is null | {kind:'bytes', data:} | {kind:'file', name:}. // Backend contract: slot0→scene_bytes (bytes; a file here is fetched→bytes on submit), @@ -5518,8 +5602,53 @@ ${canGen?'':'disabled'}>Create Scenery
`; + // --- Previous scenery library (grouped by source video) --- + html += `
+
+ Previous scenery + +
`; + if (_sceneLibOpen) { + if (!_sceneLib) { + html += `
Loading…
`; + } else if (!_sceneLib.groups.length && !_sceneLib.ungrouped.length) { + html += `
No previous scenery found.
`; + } else { + const renderLibItems = (items) => items.map(it => { + const url = IMAGE_FOLDER + encodeURIComponent(it.filename); + const gLabel = it.group_id ? ` ${escHtml(it.group_id)}` : ''; + return `
+ +
`; + }).join(''); + _sceneLib.groups.forEach(g => { + const vName = g.video.replace(/\.[^.]+$/, ''); + html += `
+
📹 ${escHtml(vName)}
+
${renderLibItems(g.items)}
+
`; + }); + if (_sceneLib.ungrouped.length) { + html += `
+
📷 uploaded / other
+
${renderLibItems(_sceneLib.ungrouped)}
+
`; + } + } + } + panel.innerHTML = html; + // Lazy-load library data when panel is opened + if (_sceneLibOpen && !_sceneLib) { + fetch(`${API}/scenery/library`) + .then(r => r.ok ? r.json() : null) + .then(d => { if (d) { _sceneLib = d; renderSidebarScenery(); } }) + .catch(() => {}); + } + // Restore the preserved prompt text and size the textarea. const spEl = document.getElementById('scenePromptInput'); if (spEl) { spEl.value = prevScenePrompt; autoGrowPrompt(spEl); } @@ -5557,6 +5686,28 @@ renderSidebarScenery(); } + function sceneLibToggle() { + _sceneLibOpen = !_sceneLibOpen; + if (_sceneLibOpen && !_sceneLib) _sceneLib = null; // force re-fetch on open + renderSidebarScenery(); + } + + // Use a library scenery image as slot 0 background (fetched to bytes). + async function sceneLibUseAsBackground(filename) { + try { + const url = IMAGE_FOLDER + encodeURIComponent(filename); + const b64 = await _fetchAsBase64(url); + _sceneRefs[0] = { kind: 'bytes', data: b64 }; + _sceneVideo = null; + _sceneFrameBytes = null; + _sceneGridOpen = false; + renderSidebarScenery(); + showToast('Background set from previous scenery', 'success'); + } catch (e) { + showToast('Could not load image: ' + e, 'error'); + } + } + // ---- scenery reference-slot helpers ---- // Thumbnail (URL or data-URL) for a slot ref. diff --git a/tour-comfy/edit_api.py b/tour-comfy/edit_api.py index ccdc884..0131959 100644 --- a/tour-comfy/edit_api.py +++ b/tour-comfy/edit_api.py @@ -2362,7 +2362,8 @@ def _make_side_by_side(img1: Image.Image, img2: Image.Image, def _scenery_worker(job_id: str, model_filename: str, scene_pil: Image.Image, - prompt: str, seed: int, extra_pils: list | None = None): + prompt: str, seed: int, extra_pils: list | None = None, + scene_video: str | None = None, extra_filename: str | None = None): output_dir = _load_output_dir() try: model_path = os.path.join(output_dir, model_filename) @@ -2390,10 +2391,16 @@ def _scenery_worker(job_id: str, model_filename: str, scene_pil: Image.Image, try: embedding = embeddings.generate_embedding(out_path) next_order = database.get_next_sort_order(group_id) + # Store all source references: person image, background video (if any), extra ref (if any) + refs = [model_filename] + if scene_video: + refs.append(f"video:{scene_video}") + if extra_filename: + refs.append(extra_filename) database.upsert_person(out_name, filepath=out_path, embedding=embedding, group_id=group_id, prompt=prompt, sort_order=next_order, - source_refs=json.dumps([model_filename])) + source_refs=json.dumps(refs)) except Exception as db_err: print(f"[scenery] DB error: {db_err}") jobs[job_id]["status"] = "done" @@ -2455,11 +2462,50 @@ def generate_scenery(req: SceneryRequest): threading.Thread( target=_scenery_worker, args=(job_id, req.model_filename, scene_pil, prompt, req.seed, extra_pils), + kwargs={"scene_video": req.scene_video if req.scene_video else None, + "extra_filename": req.extra_filename}, daemon=True, ).start() return {"job_id": job_id, "model": req.model_filename} +@app.get("/scenery/library") +def scenery_library(): + """Return all scenery images grouped by source video reference.""" + conn = database.get_db_connection() + cur = conn.cursor() + try: + cur.execute(""" + SELECT filename, group_id, source_refs + FROM person + WHERE archived IS NOT TRUE + AND filename LIKE '%_sc_%' + AND source_refs IS NOT NULL + ORDER BY filename DESC + """) + rows = cur.fetchall() + finally: + cur.close() + database._put_db_connection(conn) + + by_video: dict[str, list] = {} + ungrouped: list = [] + for filename, group_id, source_refs_raw in rows: + try: + refs = json.loads(source_refs_raw) if source_refs_raw else [] + except Exception: + refs = [] + video = next((r[len("video:"):] for r in refs if r.startswith("video:")), None) + entry = {"filename": filename, "group_id": group_id, "refs": refs} + if video: + by_video.setdefault(video, []).append(entry) + else: + ungrouped.append(entry) + + groups = [{"video": v, "items": items} for v, items in by_video.items()] + return {"groups": groups, "ungrouped": ungrouped} + + # --- SAM2 background removal -------------------------------------------------- _sam2_predictor = None