aa
This commit is contained in:
mike
@@ -1,891 +0,0 @@
|
||||
# Copyright 2025 Qwen-Image Team and The HuggingFace Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import inspect
|
||||
import math
|
||||
from typing import Any, Callable, Dict, List, Optional, Union
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from transformers import Qwen2_5_VLForConditionalGeneration, Qwen2Tokenizer, Qwen2VLProcessor
|
||||
|
||||
from diffusers.image_processor import PipelineImageInput, VaeImageProcessor
|
||||
from diffusers.loaders import QwenImageLoraLoaderMixin
|
||||
from diffusers.models import AutoencoderKLQwenImage, QwenImageTransformer2DModel
|
||||
from diffusers.schedulers import FlowMatchEulerDiscreteScheduler
|
||||
from diffusers.utils import is_torch_xla_available, logging, replace_example_docstring
|
||||
from diffusers.utils.torch_utils import randn_tensor
|
||||
from diffusers.pipelines.pipeline_utils import DiffusionPipeline
|
||||
from diffusers.pipelines.qwenimage.pipeline_output import QwenImagePipelineOutput
|
||||
|
||||
|
||||
if is_torch_xla_available():
|
||||
import torch_xla.core.xla_model as xm
|
||||
|
||||
XLA_AVAILABLE = True
|
||||
else:
|
||||
XLA_AVAILABLE = False
|
||||
|
||||
|
||||
logger = logging.get_logger(__name__) # pylint: disable=invalid-name
|
||||
|
||||
EXAMPLE_DOC_STRING = """
|
||||
Examples:
|
||||
```py
|
||||
>>> import torch
|
||||
>>> from PIL import Image
|
||||
>>> from diffusers import QwenImageEditPlusPipeline
|
||||
>>> from diffusers.utils import load_image
|
||||
|
||||
>>> pipe = QwenImageEditPlusPipeline.from_pretrained("Qwen/Qwen-Image-Edit-2509", torch_dtype=torch.bfloat16)
|
||||
>>> pipe.to("cuda")
|
||||
>>> image = load_image(
|
||||
... "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/yarn-art-pikachu.png"
|
||||
... ).convert("RGB")
|
||||
>>> prompt = (
|
||||
... "Make Pikachu hold a sign that says 'Qwen Edit is awesome', yarn art style, detailed, vibrant colors"
|
||||
... )
|
||||
>>> # Depending on the variant being used, the pipeline call will slightly vary.
|
||||
>>> # Refer to the pipeline documentation for more details.
|
||||
>>> image = pipe(image, prompt, num_inference_steps=50).images[0]
|
||||
>>> image.save("qwenimage_edit_plus.png")
|
||||
```
|
||||
"""
|
||||
|
||||
CONDITION_IMAGE_SIZE = 384 * 384
|
||||
VAE_IMAGE_SIZE = 1024 * 1024
|
||||
|
||||
|
||||
# Copied from diffusers.pipelines.qwenimage.pipeline_qwenimage.calculate_shift
|
||||
def calculate_shift(
|
||||
image_seq_len,
|
||||
base_seq_len: int = 256,
|
||||
max_seq_len: int = 4096,
|
||||
base_shift: float = 0.5,
|
||||
max_shift: float = 1.15,
|
||||
):
|
||||
m = (max_shift - base_shift) / (max_seq_len - base_seq_len)
|
||||
b = base_shift - m * base_seq_len
|
||||
mu = image_seq_len * m + b
|
||||
return mu
|
||||
|
||||
|
||||
# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps
|
||||
def retrieve_timesteps(
|
||||
scheduler,
|
||||
num_inference_steps: Optional[int] = None,
|
||||
device: Optional[Union[str, torch.device]] = None,
|
||||
timesteps: Optional[List[int]] = None,
|
||||
sigmas: Optional[List[float]] = None,
|
||||
**kwargs,
|
||||
):
|
||||
r"""
|
||||
Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles
|
||||
custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`.
|
||||
|
||||
Args:
|
||||
scheduler (`SchedulerMixin`):
|
||||
The scheduler to get timesteps from.
|
||||
num_inference_steps (`int`):
|
||||
The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps`
|
||||
must be `None`.
|
||||
device (`str` or `torch.device`, *optional*):
|
||||
The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.
|
||||
timesteps (`List[int]`, *optional*):
|
||||
Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed,
|
||||
`num_inference_steps` and `sigmas` must be `None`.
|
||||
sigmas (`List[float]`, *optional*):
|
||||
Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed,
|
||||
`num_inference_steps` and `timesteps` must be `None`.
|
||||
|
||||
Returns:
|
||||
`Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the
|
||||
second element is the number of inference steps.
|
||||
"""
|
||||
if timesteps is not None and sigmas is not None:
|
||||
raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values")
|
||||
if timesteps is not None:
|
||||
accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
|
||||
if not accepts_timesteps:
|
||||
raise ValueError(
|
||||
f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
|
||||
f" timestep schedules. Please check whether you are using the correct scheduler."
|
||||
)
|
||||
scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs)
|
||||
timesteps = scheduler.timesteps
|
||||
num_inference_steps = len(timesteps)
|
||||
elif sigmas is not None:
|
||||
accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
|
||||
if not accept_sigmas:
|
||||
raise ValueError(
|
||||
f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
|
||||
f" sigmas schedules. Please check whether you are using the correct scheduler."
|
||||
)
|
||||
scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs)
|
||||
timesteps = scheduler.timesteps
|
||||
num_inference_steps = len(timesteps)
|
||||
else:
|
||||
scheduler.set_timesteps(num_inference_steps, device=device, **kwargs)
|
||||
timesteps = scheduler.timesteps
|
||||
return timesteps, num_inference_steps
|
||||
|
||||
|
||||
# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents
|
||||
def retrieve_latents(
|
||||
encoder_output: torch.Tensor, generator: Optional[torch.Generator] = None, sample_mode: str = "sample"
|
||||
):
|
||||
if hasattr(encoder_output, "latent_dist") and sample_mode == "sample":
|
||||
return encoder_output.latent_dist.sample(generator)
|
||||
elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax":
|
||||
return encoder_output.latent_dist.mode()
|
||||
elif hasattr(encoder_output, "latents"):
|
||||
return encoder_output.latents
|
||||
else:
|
||||
raise AttributeError("Could not access latents of provided encoder_output")
|
||||
|
||||
|
||||
def calculate_dimensions(target_area, ratio):
|
||||
width = math.sqrt(target_area * ratio)
|
||||
height = width / ratio
|
||||
|
||||
width = round(width / 32) * 32
|
||||
height = round(height / 32) * 32
|
||||
|
||||
return width, height
|
||||
|
||||
|
||||
class QwenImageEditPlusPipeline(DiffusionPipeline, QwenImageLoraLoaderMixin):
|
||||
r"""
|
||||
The Qwen-Image-Edit pipeline for image editing.
|
||||
|
||||
Args:
|
||||
transformer ([`QwenImageTransformer2DModel`]):
|
||||
Conditional Transformer (MMDiT) architecture to denoise the encoded image latents.
|
||||
scheduler ([`FlowMatchEulerDiscreteScheduler`]):
|
||||
A scheduler to be used in combination with `transformer` to denoise the encoded image latents.
|
||||
vae ([`AutoencoderKL`]):
|
||||
Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.
|
||||
text_encoder ([`Qwen2.5-VL-7B-Instruct`]):
|
||||
[Qwen2.5-VL-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-VL-7B-Instruct), specifically the
|
||||
[Qwen2.5-VL-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-VL-7B-Instruct) variant.
|
||||
tokenizer (`QwenTokenizer`):
|
||||
Tokenizer of class
|
||||
[CLIPTokenizer](https://huggingface.co/docs/transformers/en/model_doc/clip#transformers.CLIPTokenizer).
|
||||
"""
|
||||
|
||||
model_cpu_offload_seq = "text_encoder->transformer->vae"
|
||||
_callback_tensor_inputs = ["latents", "prompt_embeds"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
scheduler: FlowMatchEulerDiscreteScheduler,
|
||||
vae: AutoencoderKLQwenImage,
|
||||
text_encoder: Qwen2_5_VLForConditionalGeneration,
|
||||
tokenizer: Qwen2Tokenizer,
|
||||
processor: Qwen2VLProcessor,
|
||||
transformer: QwenImageTransformer2DModel,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
self.register_modules(
|
||||
vae=vae,
|
||||
text_encoder=text_encoder,
|
||||
tokenizer=tokenizer,
|
||||
processor=processor,
|
||||
transformer=transformer,
|
||||
scheduler=scheduler,
|
||||
)
|
||||
self.vae_scale_factor = 2 ** len(self.vae.temperal_downsample) if getattr(self, "vae", None) else 8
|
||||
self.latent_channels = self.vae.config.z_dim if getattr(self, "vae", None) else 16
|
||||
# QwenImage latents are turned into 2x2 patches and packed. This means the latent width and height has to be divisible
|
||||
# by the patch size. So the vae scale factor is multiplied by the patch size to account for this
|
||||
self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor * 2)
|
||||
self.tokenizer_max_length = 1024
|
||||
|
||||
self.prompt_template_encode = "<|im_start|>system\nDescribe the key features of the input image (color, shape, size, texture, objects, background), then explain how the user's text instruction should alter or modify the image. Generate a new image that meets the user's requirements while maintaining consistency with the original input where appropriate.<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n"
|
||||
self.prompt_template_encode_start_idx = 64
|
||||
self.default_sample_size = 128
|
||||
|
||||
# Copied from diffusers.pipelines.qwenimage.pipeline_qwenimage.QwenImagePipeline._extract_masked_hidden
|
||||
def _extract_masked_hidden(self, hidden_states: torch.Tensor, mask: torch.Tensor):
|
||||
bool_mask = mask.bool()
|
||||
valid_lengths = bool_mask.sum(dim=1)
|
||||
selected = hidden_states[bool_mask]
|
||||
split_result = torch.split(selected, valid_lengths.tolist(), dim=0)
|
||||
|
||||
return split_result
|
||||
|
||||
def _get_qwen_prompt_embeds(
|
||||
self,
|
||||
prompt: Union[str, List[str]] = None,
|
||||
image: Optional[torch.Tensor] = None,
|
||||
device: Optional[torch.device] = None,
|
||||
dtype: Optional[torch.dtype] = None,
|
||||
):
|
||||
device = device or self._execution_device
|
||||
dtype = dtype or self.text_encoder.dtype
|
||||
|
||||
prompt = [prompt] if isinstance(prompt, str) else prompt
|
||||
img_prompt_template = "Picture {}: <|vision_start|><|image_pad|><|vision_end|>"
|
||||
if isinstance(image, list):
|
||||
base_img_prompt = ""
|
||||
for i, img in enumerate(image):
|
||||
base_img_prompt += img_prompt_template.format(i + 1)
|
||||
elif image is not None:
|
||||
base_img_prompt = img_prompt_template.format(1)
|
||||
else:
|
||||
base_img_prompt = ""
|
||||
|
||||
template = self.prompt_template_encode
|
||||
|
||||
drop_idx = self.prompt_template_encode_start_idx
|
||||
txt = [template.format(base_img_prompt + e) for e in prompt]
|
||||
|
||||
model_inputs = self.processor(
|
||||
text=txt,
|
||||
images=image,
|
||||
padding=True,
|
||||
return_tensors="pt",
|
||||
).to(device)
|
||||
|
||||
outputs = self.text_encoder(
|
||||
input_ids=model_inputs.input_ids,
|
||||
attention_mask=model_inputs.attention_mask,
|
||||
pixel_values=model_inputs.pixel_values,
|
||||
image_grid_thw=model_inputs.image_grid_thw,
|
||||
output_hidden_states=True,
|
||||
)
|
||||
|
||||
hidden_states = outputs.hidden_states[-1]
|
||||
split_hidden_states = self._extract_masked_hidden(hidden_states, model_inputs.attention_mask)
|
||||
split_hidden_states = [e[drop_idx:] for e in split_hidden_states]
|
||||
attn_mask_list = [torch.ones(e.size(0), dtype=torch.long, device=e.device) for e in split_hidden_states]
|
||||
max_seq_len = max([e.size(0) for e in split_hidden_states])
|
||||
prompt_embeds = torch.stack(
|
||||
[torch.cat([u, u.new_zeros(max_seq_len - u.size(0), u.size(1))]) for u in split_hidden_states]
|
||||
)
|
||||
encoder_attention_mask = torch.stack(
|
||||
[torch.cat([u, u.new_zeros(max_seq_len - u.size(0))]) for u in attn_mask_list]
|
||||
)
|
||||
|
||||
prompt_embeds = prompt_embeds.to(dtype=dtype, device=device)
|
||||
|
||||
return prompt_embeds, encoder_attention_mask
|
||||
|
||||
# Copied from diffusers.pipelines.qwenimage.pipeline_qwenimage_edit.QwenImageEditPipeline.encode_prompt
|
||||
def encode_prompt(
|
||||
self,
|
||||
prompt: Union[str, List[str]],
|
||||
image: Optional[torch.Tensor] = None,
|
||||
device: Optional[torch.device] = None,
|
||||
num_images_per_prompt: int = 1,
|
||||
prompt_embeds: Optional[torch.Tensor] = None,
|
||||
prompt_embeds_mask: Optional[torch.Tensor] = None,
|
||||
max_sequence_length: int = 1024,
|
||||
):
|
||||
r"""
|
||||
|
||||
Args:
|
||||
prompt (`str` or `List[str]`, *optional*):
|
||||
prompt to be encoded
|
||||
image (`torch.Tensor`, *optional*):
|
||||
image to be encoded
|
||||
device: (`torch.device`):
|
||||
torch device
|
||||
num_images_per_prompt (`int`):
|
||||
number of images that should be generated per prompt
|
||||
prompt_embeds (`torch.Tensor`, *optional*):
|
||||
Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
|
||||
provided, text embeddings will be generated from `prompt` input argument.
|
||||
"""
|
||||
device = device or self._execution_device
|
||||
|
||||
prompt = [prompt] if isinstance(prompt, str) else prompt
|
||||
batch_size = len(prompt) if prompt_embeds is None else prompt_embeds.shape[0]
|
||||
|
||||
if prompt_embeds is None:
|
||||
prompt_embeds, prompt_embeds_mask = self._get_qwen_prompt_embeds(prompt, image, device)
|
||||
|
||||
_, seq_len, _ = prompt_embeds.shape
|
||||
prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)
|
||||
prompt_embeds = prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)
|
||||
prompt_embeds_mask = prompt_embeds_mask.repeat(1, num_images_per_prompt, 1)
|
||||
prompt_embeds_mask = prompt_embeds_mask.view(batch_size * num_images_per_prompt, seq_len)
|
||||
|
||||
return prompt_embeds, prompt_embeds_mask
|
||||
|
||||
# Copied from diffusers.pipelines.qwenimage.pipeline_qwenimage_edit.QwenImageEditPipeline.check_inputs
|
||||
def check_inputs(
|
||||
self,
|
||||
prompt,
|
||||
height,
|
||||
width,
|
||||
negative_prompt=None,
|
||||
prompt_embeds=None,
|
||||
negative_prompt_embeds=None,
|
||||
prompt_embeds_mask=None,
|
||||
negative_prompt_embeds_mask=None,
|
||||
callback_on_step_end_tensor_inputs=None,
|
||||
max_sequence_length=None,
|
||||
):
|
||||
if height % (self.vae_scale_factor * 2) != 0 or width % (self.vae_scale_factor * 2) != 0:
|
||||
logger.warning(
|
||||
f"`height` and `width` have to be divisible by {self.vae_scale_factor * 2} but are {height} and {width}. Dimensions will be resized accordingly"
|
||||
)
|
||||
|
||||
if callback_on_step_end_tensor_inputs is not None and not all(
|
||||
k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs
|
||||
):
|
||||
raise ValueError(
|
||||
f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}"
|
||||
)
|
||||
|
||||
if prompt is not None and prompt_embeds is not None:
|
||||
raise ValueError(
|
||||
f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
|
||||
" only forward one of the two."
|
||||
)
|
||||
elif prompt is None and prompt_embeds is None:
|
||||
raise ValueError(
|
||||
"Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."
|
||||
)
|
||||
elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):
|
||||
raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")
|
||||
|
||||
if negative_prompt is not None and negative_prompt_embeds is not None:
|
||||
raise ValueError(
|
||||
f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"
|
||||
f" {negative_prompt_embeds}. Please make sure to only forward one of the two."
|
||||
)
|
||||
|
||||
if prompt_embeds is not None and prompt_embeds_mask is None:
|
||||
raise ValueError(
|
||||
"If `prompt_embeds` are provided, `prompt_embeds_mask` also have to be passed. Make sure to generate `prompt_embeds_mask` from the same text encoder that was used to generate `prompt_embeds`."
|
||||
)
|
||||
if negative_prompt_embeds is not None and negative_prompt_embeds_mask is None:
|
||||
raise ValueError(
|
||||
"If `negative_prompt_embeds` are provided, `negative_prompt_embeds_mask` also have to be passed. Make sure to generate `negative_prompt_embeds_mask` from the same text encoder that was used to generate `negative_prompt_embeds`."
|
||||
)
|
||||
|
||||
if max_sequence_length is not None and max_sequence_length > 1024:
|
||||
raise ValueError(f"`max_sequence_length` cannot be greater than 1024 but is {max_sequence_length}")
|
||||
|
||||
@staticmethod
|
||||
# Copied from diffusers.pipelines.qwenimage.pipeline_qwenimage.QwenImagePipeline._pack_latents
|
||||
def _pack_latents(latents, batch_size, num_channels_latents, height, width):
|
||||
latents = latents.view(batch_size, num_channels_latents, height // 2, 2, width // 2, 2)
|
||||
latents = latents.permute(0, 2, 4, 1, 3, 5)
|
||||
latents = latents.reshape(batch_size, (height // 2) * (width // 2), num_channels_latents * 4)
|
||||
|
||||
return latents
|
||||
|
||||
@staticmethod
|
||||
# Copied from diffusers.pipelines.qwenimage.pipeline_qwenimage.QwenImagePipeline._unpack_latents
|
||||
def _unpack_latents(latents, height, width, vae_scale_factor):
|
||||
batch_size, num_patches, channels = latents.shape
|
||||
|
||||
# VAE applies 8x compression on images but we must also account for packing which requires
|
||||
# latent height and width to be divisible by 2.
|
||||
height = 2 * (int(height) // (vae_scale_factor * 2))
|
||||
width = 2 * (int(width) // (vae_scale_factor * 2))
|
||||
|
||||
latents = latents.view(batch_size, height // 2, width // 2, channels // 4, 2, 2)
|
||||
latents = latents.permute(0, 3, 1, 4, 2, 5)
|
||||
|
||||
latents = latents.reshape(batch_size, channels // (2 * 2), 1, height, width)
|
||||
|
||||
return latents
|
||||
|
||||
# Copied from diffusers.pipelines.qwenimage.pipeline_qwenimage_edit.QwenImageEditPipeline._encode_vae_image
|
||||
def _encode_vae_image(self, image: torch.Tensor, generator: torch.Generator):
|
||||
if isinstance(generator, list):
|
||||
image_latents = [
|
||||
retrieve_latents(self.vae.encode(image[i : i + 1]), generator=generator[i], sample_mode="argmax")
|
||||
for i in range(image.shape[0])
|
||||
]
|
||||
image_latents = torch.cat(image_latents, dim=0)
|
||||
else:
|
||||
image_latents = retrieve_latents(self.vae.encode(image), generator=generator, sample_mode="argmax")
|
||||
latents_mean = (
|
||||
torch.tensor(self.vae.config.latents_mean)
|
||||
.view(1, self.latent_channels, 1, 1, 1)
|
||||
.to(image_latents.device, image_latents.dtype)
|
||||
)
|
||||
latents_std = (
|
||||
torch.tensor(self.vae.config.latents_std)
|
||||
.view(1, self.latent_channels, 1, 1, 1)
|
||||
.to(image_latents.device, image_latents.dtype)
|
||||
)
|
||||
image_latents = (image_latents - latents_mean) / latents_std
|
||||
|
||||
return image_latents
|
||||
|
||||
def prepare_latents(
|
||||
self,
|
||||
images,
|
||||
batch_size,
|
||||
num_channels_latents,
|
||||
height,
|
||||
width,
|
||||
dtype,
|
||||
device,
|
||||
generator,
|
||||
latents=None,
|
||||
):
|
||||
# VAE applies 8x compression on images but we must also account for packing which requires
|
||||
# latent height and width to be divisible by 2.
|
||||
height = 2 * (int(height) // (self.vae_scale_factor * 2))
|
||||
width = 2 * (int(width) // (self.vae_scale_factor * 2))
|
||||
|
||||
shape = (batch_size, 1, num_channels_latents, height, width)
|
||||
|
||||
image_latents = None
|
||||
if images is not None:
|
||||
if not isinstance(images, list):
|
||||
images = [images]
|
||||
all_image_latents = []
|
||||
for image in images:
|
||||
image = image.to(device=device, dtype=dtype)
|
||||
if image.shape[1] != self.latent_channels:
|
||||
image_latents = self._encode_vae_image(image=image, generator=generator)
|
||||
else:
|
||||
image_latents = image
|
||||
if batch_size > image_latents.shape[0] and batch_size % image_latents.shape[0] == 0:
|
||||
# expand init_latents for batch_size
|
||||
additional_image_per_prompt = batch_size // image_latents.shape[0]
|
||||
image_latents = torch.cat([image_latents] * additional_image_per_prompt, dim=0)
|
||||
elif batch_size > image_latents.shape[0] and batch_size % image_latents.shape[0] != 0:
|
||||
raise ValueError(
|
||||
f"Cannot duplicate `image` of batch size {image_latents.shape[0]} to {batch_size} text prompts."
|
||||
)
|
||||
else:
|
||||
image_latents = torch.cat([image_latents], dim=0)
|
||||
|
||||
image_latent_height, image_latent_width = image_latents.shape[3:]
|
||||
image_latents = self._pack_latents(
|
||||
image_latents, batch_size, num_channels_latents, image_latent_height, image_latent_width
|
||||
)
|
||||
all_image_latents.append(image_latents)
|
||||
image_latents = torch.cat(all_image_latents, dim=1)
|
||||
|
||||
if isinstance(generator, list) and len(generator) != batch_size:
|
||||
raise ValueError(
|
||||
f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"
|
||||
f" size of {batch_size}. Make sure the batch size matches the length of the generators."
|
||||
)
|
||||
if latents is None:
|
||||
latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
|
||||
latents = self._pack_latents(latents, batch_size, num_channels_latents, height, width)
|
||||
else:
|
||||
latents = latents.to(device=device, dtype=dtype)
|
||||
|
||||
return latents, image_latents
|
||||
|
||||
@property
|
||||
def guidance_scale(self):
|
||||
return self._guidance_scale
|
||||
|
||||
@property
|
||||
def attention_kwargs(self):
|
||||
return self._attention_kwargs
|
||||
|
||||
@property
|
||||
def num_timesteps(self):
|
||||
return self._num_timesteps
|
||||
|
||||
@property
|
||||
def current_timestep(self):
|
||||
return self._current_timestep
|
||||
|
||||
@property
|
||||
def interrupt(self):
|
||||
return self._interrupt
|
||||
|
||||
@torch.no_grad()
|
||||
@replace_example_docstring(EXAMPLE_DOC_STRING)
|
||||
def __call__(
|
||||
self,
|
||||
image: Optional[PipelineImageInput] = None,
|
||||
prompt: Union[str, List[str]] = None,
|
||||
negative_prompt: Union[str, List[str]] = None,
|
||||
true_cfg_scale: float = 4.0,
|
||||
height: Optional[int] = None,
|
||||
width: Optional[int] = None,
|
||||
num_inference_steps: int = 50,
|
||||
sigmas: Optional[List[float]] = None,
|
||||
guidance_scale: Optional[float] = None,
|
||||
num_images_per_prompt: int = 1,
|
||||
generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,
|
||||
latents: Optional[torch.Tensor] = None,
|
||||
prompt_embeds: Optional[torch.Tensor] = None,
|
||||
prompt_embeds_mask: Optional[torch.Tensor] = None,
|
||||
negative_prompt_embeds: Optional[torch.Tensor] = None,
|
||||
negative_prompt_embeds_mask: Optional[torch.Tensor] = None,
|
||||
output_type: Optional[str] = "pil",
|
||||
return_dict: bool = True,
|
||||
attention_kwargs: Optional[Dict[str, Any]] = None,
|
||||
callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None,
|
||||
callback_on_step_end_tensor_inputs: List[str] = ["latents"],
|
||||
max_sequence_length: int = 512,
|
||||
):
|
||||
r"""
|
||||
Function invoked when calling the pipeline for generation.
|
||||
|
||||
Args:
|
||||
image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`):
|
||||
`Image`, numpy array or tensor representing an image batch to be used as the starting point. For both
|
||||
numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list
|
||||
or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a
|
||||
list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)` It can also accept image
|
||||
latents as `image`, but if passing latents directly it is not encoded again.
|
||||
prompt (`str` or `List[str]`, *optional*):
|
||||
The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.
|
||||
instead.
|
||||
negative_prompt (`str` or `List[str]`, *optional*):
|
||||
The prompt or prompts not to guide the image generation. If not defined, one has to pass
|
||||
`negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `true_cfg_scale` is
|
||||
not greater than `1`).
|
||||
true_cfg_scale (`float`, *optional*, defaults to 1.0):
|
||||
true_cfg_scale (`float`, *optional*, defaults to 1.0): Guidance scale as defined in [Classifier-Free
|
||||
Diffusion Guidance](https://huggingface.co/papers/2207.12598). `true_cfg_scale` is defined as `w` of
|
||||
equation 2. of [Imagen Paper](https://huggingface.co/papers/2205.11487). Classifier-free guidance is
|
||||
enabled by setting `true_cfg_scale > 1` and a provided `negative_prompt`. Higher guidance scale
|
||||
encourages to generate images that are closely linked to the text `prompt`, usually at the expense of
|
||||
lower image quality.
|
||||
height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
|
||||
The height in pixels of the generated image. This is set to 1024 by default for the best results.
|
||||
width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
|
||||
The width in pixels of the generated image. This is set to 1024 by default for the best results.
|
||||
num_inference_steps (`int`, *optional*, defaults to 50):
|
||||
The number of denoising steps. More denoising steps usually lead to a higher quality image at the
|
||||
expense of slower inference.
|
||||
sigmas (`List[float]`, *optional*):
|
||||
Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in
|
||||
their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed
|
||||
will be used.
|
||||
guidance_scale (`float`, *optional*, defaults to None):
|
||||
A guidance scale value for guidance distilled models. Unlike the traditional classifier-free guidance
|
||||
where the guidance scale is applied during inference through noise prediction rescaling, guidance
|
||||
distilled models take the guidance scale directly as an input parameter during forward pass. Guidance
|
||||
scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images
|
||||
that are closely linked to the text `prompt`, usually at the expense of lower image quality. This
|
||||
parameter in the pipeline is there to support future guidance-distilled models when they come up. It is
|
||||
ignored when not using guidance distilled models. To enable traditional classifier-free guidance,
|
||||
please pass `true_cfg_scale > 1.0` and `negative_prompt` (even an empty negative prompt like " " should
|
||||
enable classifier-free guidance computations).
|
||||
num_images_per_prompt (`int`, *optional*, defaults to 1):
|
||||
The number of images to generate per prompt.
|
||||
generator (`torch.Generator` or `List[torch.Generator]`, *optional*):
|
||||
One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)
|
||||
to make generation deterministic.
|
||||
latents (`torch.Tensor`, *optional*):
|
||||
Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
|
||||
generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
|
||||
tensor will be generated by sampling using the supplied random `generator`.
|
||||
prompt_embeds (`torch.Tensor`, *optional*):
|
||||
Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
|
||||
provided, text embeddings will be generated from `prompt` input argument.
|
||||
negative_prompt_embeds (`torch.Tensor`, *optional*):
|
||||
Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
|
||||
weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
|
||||
argument.
|
||||
output_type (`str`, *optional*, defaults to `"pil"`):
|
||||
The output format of the generate image. Choose between
|
||||
[PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
|
||||
return_dict (`bool`, *optional*, defaults to `True`):
|
||||
Whether or not to return a [`~pipelines.qwenimage.QwenImagePipelineOutput`] instead of a plain tuple.
|
||||
attention_kwargs (`dict`, *optional*):
|
||||
A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
|
||||
`self.processor` in
|
||||
[diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
|
||||
callback_on_step_end (`Callable`, *optional*):
|
||||
A function that calls at the end of each denoising steps during the inference. The function is called
|
||||
with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int,
|
||||
callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by
|
||||
`callback_on_step_end_tensor_inputs`.
|
||||
callback_on_step_end_tensor_inputs (`List`, *optional*):
|
||||
The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list
|
||||
will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the
|
||||
`._callback_tensor_inputs` attribute of your pipeline class.
|
||||
max_sequence_length (`int` defaults to 512): Maximum sequence length to use with the `prompt`.
|
||||
|
||||
Examples:
|
||||
|
||||
Returns:
|
||||
[`~pipelines.qwenimage.QwenImagePipelineOutput`] or `tuple`:
|
||||
[`~pipelines.qwenimage.QwenImagePipelineOutput`] if `return_dict` is True, otherwise a `tuple`. When
|
||||
returning a tuple, the first element is a list with the generated images.
|
||||
"""
|
||||
image_size = image[-1].size if isinstance(image, list) else image.size
|
||||
calculated_width, calculated_height = calculate_dimensions(1024 * 1024, image_size[0] / image_size[1])
|
||||
height = height or calculated_height
|
||||
width = width or calculated_width
|
||||
|
||||
multiple_of = self.vae_scale_factor * 2
|
||||
width = width // multiple_of * multiple_of
|
||||
height = height // multiple_of * multiple_of
|
||||
|
||||
# 1. Check inputs. Raise error if not correct
|
||||
self.check_inputs(
|
||||
prompt,
|
||||
height,
|
||||
width,
|
||||
negative_prompt=negative_prompt,
|
||||
prompt_embeds=prompt_embeds,
|
||||
negative_prompt_embeds=negative_prompt_embeds,
|
||||
prompt_embeds_mask=prompt_embeds_mask,
|
||||
negative_prompt_embeds_mask=negative_prompt_embeds_mask,
|
||||
callback_on_step_end_tensor_inputs=callback_on_step_end_tensor_inputs,
|
||||
max_sequence_length=max_sequence_length,
|
||||
)
|
||||
|
||||
self._guidance_scale = guidance_scale
|
||||
self._attention_kwargs = attention_kwargs
|
||||
self._current_timestep = None
|
||||
self._interrupt = False
|
||||
|
||||
# 2. Define call parameters
|
||||
if prompt is not None and isinstance(prompt, str):
|
||||
batch_size = 1
|
||||
elif prompt is not None and isinstance(prompt, list):
|
||||
batch_size = len(prompt)
|
||||
else:
|
||||
batch_size = prompt_embeds.shape[0]
|
||||
|
||||
device = self._execution_device
|
||||
# 3. Preprocess image
|
||||
if image is not None and not (isinstance(image, torch.Tensor) and image.size(1) == self.latent_channels):
|
||||
if not isinstance(image, list):
|
||||
image = [image]
|
||||
condition_image_sizes = []
|
||||
condition_images = []
|
||||
vae_image_sizes = []
|
||||
vae_images = []
|
||||
for img in image:
|
||||
image_width, image_height = img.size
|
||||
condition_width, condition_height = calculate_dimensions(
|
||||
CONDITION_IMAGE_SIZE, image_width / image_height
|
||||
)
|
||||
vae_width, vae_height = calculate_dimensions(VAE_IMAGE_SIZE, image_width / image_height)
|
||||
condition_image_sizes.append((condition_width, condition_height))
|
||||
vae_image_sizes.append((vae_width, vae_height))
|
||||
condition_images.append(self.image_processor.resize(img, condition_height, condition_width))
|
||||
vae_images.append(self.image_processor.preprocess(img, vae_height, vae_width).unsqueeze(2))
|
||||
|
||||
has_neg_prompt = negative_prompt is not None or (
|
||||
negative_prompt_embeds is not None and negative_prompt_embeds_mask is not None
|
||||
)
|
||||
|
||||
if true_cfg_scale > 1 and not has_neg_prompt:
|
||||
logger.warning(
|
||||
f"true_cfg_scale is passed as {true_cfg_scale}, but classifier-free guidance is not enabled since no negative_prompt is provided."
|
||||
)
|
||||
elif true_cfg_scale <= 1 and has_neg_prompt:
|
||||
logger.warning(
|
||||
" negative_prompt is passed but classifier-free guidance is not enabled since true_cfg_scale <= 1"
|
||||
)
|
||||
|
||||
do_true_cfg = true_cfg_scale > 1 and has_neg_prompt
|
||||
prompt_embeds, prompt_embeds_mask = self.encode_prompt(
|
||||
image=condition_images,
|
||||
prompt=prompt,
|
||||
prompt_embeds=prompt_embeds,
|
||||
prompt_embeds_mask=prompt_embeds_mask,
|
||||
device=device,
|
||||
num_images_per_prompt=num_images_per_prompt,
|
||||
max_sequence_length=max_sequence_length,
|
||||
)
|
||||
if do_true_cfg:
|
||||
negative_prompt_embeds, negative_prompt_embeds_mask = self.encode_prompt(
|
||||
image=condition_images,
|
||||
prompt=negative_prompt,
|
||||
prompt_embeds=negative_prompt_embeds,
|
||||
prompt_embeds_mask=negative_prompt_embeds_mask,
|
||||
device=device,
|
||||
num_images_per_prompt=num_images_per_prompt,
|
||||
max_sequence_length=max_sequence_length,
|
||||
)
|
||||
|
||||
# 4. Prepare latent variables
|
||||
num_channels_latents = self.transformer.config.in_channels // 4
|
||||
latents, image_latents = self.prepare_latents(
|
||||
vae_images,
|
||||
batch_size * num_images_per_prompt,
|
||||
num_channels_latents,
|
||||
height,
|
||||
width,
|
||||
prompt_embeds.dtype,
|
||||
device,
|
||||
generator,
|
||||
latents,
|
||||
)
|
||||
img_shapes = [
|
||||
[
|
||||
(1, height // self.vae_scale_factor // 2, width // self.vae_scale_factor // 2),
|
||||
*[
|
||||
(1, vae_height // self.vae_scale_factor // 2, vae_width // self.vae_scale_factor // 2)
|
||||
for vae_width, vae_height in vae_image_sizes
|
||||
],
|
||||
]
|
||||
] * batch_size
|
||||
|
||||
# 5. Prepare timesteps
|
||||
sigmas = np.linspace(1.0, 1 / num_inference_steps, num_inference_steps) if sigmas is None else sigmas
|
||||
image_seq_len = latents.shape[1]
|
||||
mu = calculate_shift(
|
||||
image_seq_len,
|
||||
self.scheduler.config.get("base_image_seq_len", 256),
|
||||
self.scheduler.config.get("max_image_seq_len", 4096),
|
||||
self.scheduler.config.get("base_shift", 0.5),
|
||||
self.scheduler.config.get("max_shift", 1.15),
|
||||
)
|
||||
timesteps, num_inference_steps = retrieve_timesteps(
|
||||
self.scheduler,
|
||||
num_inference_steps,
|
||||
device,
|
||||
sigmas=sigmas,
|
||||
mu=mu,
|
||||
)
|
||||
num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0)
|
||||
self._num_timesteps = len(timesteps)
|
||||
|
||||
# handle guidance
|
||||
if self.transformer.config.guidance_embeds and guidance_scale is None:
|
||||
raise ValueError("guidance_scale is required for guidance-distilled model.")
|
||||
elif self.transformer.config.guidance_embeds:
|
||||
guidance = torch.full([1], guidance_scale, device=device, dtype=torch.float32)
|
||||
guidance = guidance.expand(latents.shape[0])
|
||||
elif not self.transformer.config.guidance_embeds and guidance_scale is not None:
|
||||
logger.warning(
|
||||
f"guidance_scale is passed as {guidance_scale}, but ignored since the model is not guidance-distilled."
|
||||
)
|
||||
guidance = None
|
||||
elif not self.transformer.config.guidance_embeds and guidance_scale is None:
|
||||
guidance = None
|
||||
|
||||
if self.attention_kwargs is None:
|
||||
self._attention_kwargs = {}
|
||||
|
||||
txt_seq_lens = prompt_embeds_mask.sum(dim=1).tolist() if prompt_embeds_mask is not None else None
|
||||
|
||||
image_rotary_emb = self.transformer.pos_embed(img_shapes, txt_seq_lens, device=latents.device)
|
||||
if do_true_cfg:
|
||||
negative_txt_seq_lens = (
|
||||
negative_prompt_embeds_mask.sum(dim=1).tolist()
|
||||
if negative_prompt_embeds_mask is not None
|
||||
else None
|
||||
)
|
||||
uncond_image_rotary_emb = self.transformer.pos_embed(
|
||||
img_shapes, negative_txt_seq_lens, device=latents.device
|
||||
)
|
||||
else:
|
||||
uncond_image_rotary_emb = None
|
||||
|
||||
# 6. Denoising loop
|
||||
self.scheduler.set_begin_index(0)
|
||||
with self.progress_bar(total=num_inference_steps) as progress_bar:
|
||||
for i, t in enumerate(timesteps):
|
||||
if self.interrupt:
|
||||
continue
|
||||
|
||||
self._current_timestep = t
|
||||
|
||||
latent_model_input = latents
|
||||
if image_latents is not None:
|
||||
latent_model_input = torch.cat([latents, image_latents], dim=1)
|
||||
|
||||
# broadcast to batch dimension in a way that's compatible with ONNX/Core ML
|
||||
timestep = t.expand(latents.shape[0]).to(latents.dtype)
|
||||
with self.transformer.cache_context("cond"):
|
||||
noise_pred = self.transformer(
|
||||
hidden_states=latent_model_input,
|
||||
timestep=timestep / 1000,
|
||||
guidance=guidance,
|
||||
encoder_hidden_states_mask=prompt_embeds_mask,
|
||||
encoder_hidden_states=prompt_embeds,
|
||||
image_rotary_emb=image_rotary_emb,
|
||||
attention_kwargs=self.attention_kwargs,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
noise_pred = noise_pred[:, : latents.size(1)]
|
||||
|
||||
if do_true_cfg:
|
||||
with self.transformer.cache_context("uncond"):
|
||||
neg_noise_pred = self.transformer(
|
||||
hidden_states=latent_model_input,
|
||||
timestep=timestep / 1000,
|
||||
guidance=guidance,
|
||||
encoder_hidden_states_mask=negative_prompt_embeds_mask,
|
||||
encoder_hidden_states=negative_prompt_embeds,
|
||||
image_rotary_emb=uncond_image_rotary_emb,
|
||||
attention_kwargs=self.attention_kwargs,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
neg_noise_pred = neg_noise_pred[:, : latents.size(1)]
|
||||
comb_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred)
|
||||
|
||||
cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True)
|
||||
noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True)
|
||||
noise_pred = comb_pred * (cond_norm / noise_norm)
|
||||
|
||||
# compute the previous noisy sample x_t -> x_t-1
|
||||
latents_dtype = latents.dtype
|
||||
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
|
||||
|
||||
if latents.dtype != latents_dtype:
|
||||
if torch.backends.mps.is_available():
|
||||
# some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272
|
||||
latents = latents.to(latents_dtype)
|
||||
|
||||
if callback_on_step_end is not None:
|
||||
callback_kwargs = {}
|
||||
for k in callback_on_step_end_tensor_inputs:
|
||||
callback_kwargs[k] = locals()[k]
|
||||
callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
|
||||
|
||||
latents = callback_outputs.pop("latents", latents)
|
||||
prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
|
||||
|
||||
# call the callback, if provided
|
||||
if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
|
||||
progress_bar.update()
|
||||
|
||||
if XLA_AVAILABLE:
|
||||
xm.mark_step()
|
||||
|
||||
self._current_timestep = None
|
||||
if output_type == "latent":
|
||||
image = latents
|
||||
else:
|
||||
latents = self._unpack_latents(latents, height, width, self.vae_scale_factor)
|
||||
latents = latents.to(self.vae.dtype)
|
||||
latents_mean = (
|
||||
torch.tensor(self.vae.config.latents_mean)
|
||||
.view(1, self.vae.config.z_dim, 1, 1, 1)
|
||||
.to(latents.device, latents.dtype)
|
||||
)
|
||||
latents_std = 1.0 / torch.tensor(self.vae.config.latents_std).view(1, self.vae.config.z_dim, 1, 1, 1).to(
|
||||
latents.device, latents.dtype
|
||||
)
|
||||
latents = latents / latents_std + latents_mean
|
||||
image = self.vae.decode(latents, return_dict=False)[0][:, :, 0]
|
||||
image = self.image_processor.postprocess(image, output_type=output_type)
|
||||
|
||||
# Offload all models
|
||||
self.maybe_free_model_hooks()
|
||||
|
||||
if not return_dict:
|
||||
return (image,)
|
||||
|
||||
return QwenImagePipelineOutput(images=image)
|
||||
@@ -1,142 +0,0 @@
|
||||
"""
|
||||
Paired with a good language model. Thanks!
|
||||
"""
|
||||
|
||||
import torch
|
||||
from typing import Optional, Tuple
|
||||
from diffusers.models.transformers.transformer_qwenimage import apply_rotary_emb_qwen
|
||||
|
||||
try:
|
||||
from kernels import get_kernel
|
||||
_k = get_kernel("kernels-community/vllm-flash-attn3")
|
||||
_flash_attn_func = _k.flash_attn_func
|
||||
except Exception as e:
|
||||
_flash_attn_func = None
|
||||
_kernels_err = e
|
||||
|
||||
|
||||
def _ensure_fa3_available():
|
||||
if _flash_attn_func is None:
|
||||
raise ImportError(
|
||||
"FlashAttention-3 via Hugging Face `kernels` is required. "
|
||||
"Tried `get_kernel('kernels-community/vllm-flash-attn3')` and failed with:\n"
|
||||
f"{_kernels_err}"
|
||||
)
|
||||
|
||||
@torch.library.custom_op("flash::flash_attn_func", mutates_args=())
|
||||
def flash_attn_func(
|
||||
q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, causal: bool = False
|
||||
) -> torch.Tensor:
|
||||
outputs, lse = _flash_attn_func(q, k, v, causal=causal)
|
||||
return outputs
|
||||
|
||||
@flash_attn_func.register_fake
|
||||
def _(q, k, v, **kwargs):
|
||||
# two outputs:
|
||||
# 1. output: (batch, seq_len, num_heads, head_dim)
|
||||
# 2. softmax_lse: (batch, num_heads, seq_len) with dtype=torch.float32
|
||||
meta_q = torch.empty_like(q).contiguous()
|
||||
return meta_q #, q.new_empty((q.size(0), q.size(2), q.size(1)), dtype=torch.float32)
|
||||
|
||||
|
||||
class QwenDoubleStreamAttnProcessorFA3:
|
||||
"""
|
||||
FA3-based attention processor for Qwen double-stream architecture.
|
||||
Computes joint attention over concatenated [text, image] streams using vLLM FlashAttention-3
|
||||
accessed via Hugging Face `kernels`.
|
||||
|
||||
Notes / limitations:
|
||||
- General attention masks are not supported here (FA3 path). `is_causal=False` and no arbitrary mask.
|
||||
- Optional windowed attention / sink tokens / softcap can be plumbed through if you use those features.
|
||||
- Expects an available `apply_rotary_emb_qwen` in scope (same as your non-FA3 processor).
|
||||
"""
|
||||
|
||||
_attention_backend = "fa3" # for parity with your other processors, not used internally
|
||||
|
||||
def __init__(self):
|
||||
_ensure_fa3_available()
|
||||
|
||||
@torch.no_grad()
|
||||
def __call__(
|
||||
self,
|
||||
attn, # Attention module with to_q/to_k/to_v/add_*_proj, norms, to_out, to_add_out, and .heads
|
||||
hidden_states: torch.FloatTensor, # (B, S_img, D_model) image stream
|
||||
encoder_hidden_states: torch.FloatTensor = None, # (B, S_txt, D_model) text stream
|
||||
encoder_hidden_states_mask: torch.FloatTensor = None, # unused in FA3 path
|
||||
attention_mask: Optional[torch.FloatTensor] = None, # unused in FA3 path
|
||||
image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, # (img_freqs, txt_freqs)
|
||||
) -> Tuple[torch.FloatTensor, torch.FloatTensor]:
|
||||
if encoder_hidden_states is None:
|
||||
raise ValueError("QwenDoubleStreamAttnProcessorFA3 requires encoder_hidden_states (text stream).")
|
||||
if attention_mask is not None:
|
||||
# FA3 kernel path here does not consume arbitrary masks; fail fast to avoid silent correctness issues.
|
||||
raise NotImplementedError("attention_mask is not supported in this FA3 implementation.")
|
||||
|
||||
_ensure_fa3_available()
|
||||
|
||||
B, S_img, _ = hidden_states.shape
|
||||
S_txt = encoder_hidden_states.shape[1]
|
||||
|
||||
# ---- QKV projections (image/sample stream) ----
|
||||
img_q = attn.to_q(hidden_states) # (B, S_img, D)
|
||||
img_k = attn.to_k(hidden_states)
|
||||
img_v = attn.to_v(hidden_states)
|
||||
|
||||
# ---- QKV projections (text/context stream) ----
|
||||
txt_q = attn.add_q_proj(encoder_hidden_states) # (B, S_txt, D)
|
||||
txt_k = attn.add_k_proj(encoder_hidden_states)
|
||||
txt_v = attn.add_v_proj(encoder_hidden_states)
|
||||
|
||||
# ---- Reshape to (B, S, H, D_h) ----
|
||||
H = attn.heads
|
||||
img_q = img_q.unflatten(-1, (H, -1))
|
||||
img_k = img_k.unflatten(-1, (H, -1))
|
||||
img_v = img_v.unflatten(-1, (H, -1))
|
||||
|
||||
txt_q = txt_q.unflatten(-1, (H, -1))
|
||||
txt_k = txt_k.unflatten(-1, (H, -1))
|
||||
txt_v = txt_v.unflatten(-1, (H, -1))
|
||||
|
||||
# ---- Q/K normalization (per your module contract) ----
|
||||
if getattr(attn, "norm_q", None) is not None:
|
||||
img_q = attn.norm_q(img_q)
|
||||
if getattr(attn, "norm_k", None) is not None:
|
||||
img_k = attn.norm_k(img_k)
|
||||
if getattr(attn, "norm_added_q", None) is not None:
|
||||
txt_q = attn.norm_added_q(txt_q)
|
||||
if getattr(attn, "norm_added_k", None) is not None:
|
||||
txt_k = attn.norm_added_k(txt_k)
|
||||
|
||||
# ---- RoPE (Qwen variant) ----
|
||||
if image_rotary_emb is not None:
|
||||
img_freqs, txt_freqs = image_rotary_emb
|
||||
# expects tensors shaped (B, S, H, D_h)
|
||||
img_q = apply_rotary_emb_qwen(img_q, img_freqs, use_real=False)
|
||||
img_k = apply_rotary_emb_qwen(img_k, img_freqs, use_real=False)
|
||||
txt_q = apply_rotary_emb_qwen(txt_q, txt_freqs, use_real=False)
|
||||
txt_k = apply_rotary_emb_qwen(txt_k, txt_freqs, use_real=False)
|
||||
|
||||
# ---- Joint attention over [text, image] along sequence axis ----
|
||||
# Shapes: (B, S_total, H, D_h)
|
||||
q = torch.cat([txt_q, img_q], dim=1)
|
||||
k = torch.cat([txt_k, img_k], dim=1)
|
||||
v = torch.cat([txt_v, img_v], dim=1)
|
||||
|
||||
# FlashAttention-3 path expects (B, S, H, D_h) and returns (out, softmax_lse)
|
||||
out = flash_attn_func(q, k, v, causal=False) # out: (B, S_total, H, D_h)
|
||||
|
||||
# ---- Back to (B, S, D_model) ----
|
||||
out = out.flatten(2, 3).to(q.dtype)
|
||||
|
||||
# Split back to text / image segments
|
||||
txt_attn_out = out[:, :S_txt, :]
|
||||
img_attn_out = out[:, S_txt:, :]
|
||||
|
||||
# ---- Output projections ----
|
||||
img_attn_out = attn.to_out[0](img_attn_out)
|
||||
if len(attn.to_out) > 1:
|
||||
img_attn_out = attn.to_out[1](img_attn_out) # dropout if present
|
||||
|
||||
txt_attn_out = attn.to_add_out(txt_attn_out)
|
||||
|
||||
return img_attn_out, txt_attn_out
|
||||
@@ -1,642 +0,0 @@
|
||||
# Copyright 2025 Qwen-Image Team, The HuggingFace Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import functools
|
||||
import math
|
||||
from typing import Any, Dict, List, Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
from diffusers.configuration_utils import ConfigMixin, register_to_config
|
||||
from diffusers.loaders import FromOriginalModelMixin, PeftAdapterMixin
|
||||
from diffusers.utils import USE_PEFT_BACKEND, logging, scale_lora_layers, unscale_lora_layers
|
||||
from diffusers.utils.torch_utils import maybe_allow_in_graph
|
||||
from diffusers.models.attention import FeedForward, AttentionMixin
|
||||
from diffusers.models.attention_dispatch import dispatch_attention_fn
|
||||
from diffusers.models.attention_processor import Attention
|
||||
from diffusers.models.cache_utils import CacheMixin
|
||||
from diffusers.models.embeddings import TimestepEmbedding, Timesteps
|
||||
from diffusers.models.modeling_outputs import Transformer2DModelOutput
|
||||
from diffusers.models.modeling_utils import ModelMixin
|
||||
from diffusers.models.normalization import AdaLayerNormContinuous, RMSNorm
|
||||
|
||||
|
||||
logger = logging.get_logger(__name__) # pylint: disable=invalid-name
|
||||
|
||||
|
||||
def get_timestep_embedding(
|
||||
timesteps: torch.Tensor,
|
||||
embedding_dim: int,
|
||||
flip_sin_to_cos: bool = False,
|
||||
downscale_freq_shift: float = 1,
|
||||
scale: float = 1,
|
||||
max_period: int = 10000,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
This matches the implementation in Denoising Diffusion Probabilistic Models: Create sinusoidal timestep embeddings.
|
||||
|
||||
Args
|
||||
timesteps (torch.Tensor):
|
||||
a 1-D Tensor of N indices, one per batch element. These may be fractional.
|
||||
embedding_dim (int):
|
||||
the dimension of the output.
|
||||
flip_sin_to_cos (bool):
|
||||
Whether the embedding order should be `cos, sin` (if True) or `sin, cos` (if False)
|
||||
downscale_freq_shift (float):
|
||||
Controls the delta between frequencies between dimensions
|
||||
scale (float):
|
||||
Scaling factor applied to the embeddings.
|
||||
max_period (int):
|
||||
Controls the maximum frequency of the embeddings
|
||||
Returns
|
||||
torch.Tensor: an [N x dim] Tensor of positional embeddings.
|
||||
"""
|
||||
assert len(timesteps.shape) == 1, "Timesteps should be a 1d-array"
|
||||
|
||||
half_dim = embedding_dim // 2
|
||||
exponent = -math.log(max_period) * torch.arange(
|
||||
start=0, end=half_dim, dtype=torch.float32, device=timesteps.device
|
||||
)
|
||||
exponent = exponent / (half_dim - downscale_freq_shift)
|
||||
|
||||
emb = torch.exp(exponent).to(timesteps.dtype)
|
||||
emb = timesteps[:, None].float() * emb[None, :]
|
||||
|
||||
# scale embeddings
|
||||
emb = scale * emb
|
||||
|
||||
# concat sine and cosine embeddings
|
||||
emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=-1)
|
||||
|
||||
# flip sine and cosine embeddings
|
||||
if flip_sin_to_cos:
|
||||
emb = torch.cat([emb[:, half_dim:], emb[:, :half_dim]], dim=-1)
|
||||
|
||||
# zero pad
|
||||
if embedding_dim % 2 == 1:
|
||||
emb = torch.nn.functional.pad(emb, (0, 1, 0, 0))
|
||||
return emb
|
||||
|
||||
|
||||
def apply_rotary_emb_qwen(
|
||||
x: torch.Tensor,
|
||||
freqs_cis: Union[torch.Tensor, Tuple[torch.Tensor]],
|
||||
use_real: bool = True,
|
||||
use_real_unbind_dim: int = -1,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Apply rotary embeddings to input tensors using the given frequency tensor. This function applies rotary embeddings
|
||||
to the given query or key 'x' tensors using the provided frequency tensor 'freqs_cis'. The input tensors are
|
||||
reshaped as complex numbers, and the frequency tensor is reshaped for broadcasting compatibility. The resulting
|
||||
tensors contain rotary embeddings and are returned as real tensors.
|
||||
|
||||
Args:
|
||||
x (`torch.Tensor`):
|
||||
Query or key tensor to apply rotary embeddings. [B, S, H, D] xk (torch.Tensor): Key tensor to apply
|
||||
freqs_cis (`Tuple[torch.Tensor]`): Precomputed frequency tensor for complex exponentials. ([S, D], [S, D],)
|
||||
|
||||
Returns:
|
||||
Tuple[torch.Tensor, torch.Tensor]: Tuple of modified query tensor and key tensor with rotary embeddings.
|
||||
"""
|
||||
if use_real:
|
||||
cos, sin = freqs_cis # [S, D]
|
||||
cos = cos[None, None]
|
||||
sin = sin[None, None]
|
||||
cos, sin = cos.to(x.device), sin.to(x.device)
|
||||
|
||||
if use_real_unbind_dim == -1:
|
||||
# Used for flux, cogvideox, hunyuan-dit
|
||||
x_real, x_imag = x.reshape(*x.shape[:-1], -1, 2).unbind(-1) # [B, S, H, D//2]
|
||||
x_rotated = torch.stack([-x_imag, x_real], dim=-1).flatten(3)
|
||||
elif use_real_unbind_dim == -2:
|
||||
# Used for Stable Audio, OmniGen, CogView4 and Cosmos
|
||||
x_real, x_imag = x.reshape(*x.shape[:-1], 2, -1).unbind(-2) # [B, S, H, D//2]
|
||||
x_rotated = torch.cat([-x_imag, x_real], dim=-1)
|
||||
else:
|
||||
raise ValueError(f"`use_real_unbind_dim={use_real_unbind_dim}` but should be -1 or -2.")
|
||||
|
||||
out = (x.float() * cos + x_rotated.float() * sin).to(x.dtype)
|
||||
|
||||
return out
|
||||
else:
|
||||
x_rotated = torch.view_as_complex(x.float().reshape(*x.shape[:-1], -1, 2))
|
||||
freqs_cis = freqs_cis.unsqueeze(1)
|
||||
x_out = torch.view_as_real(x_rotated * freqs_cis).flatten(3)
|
||||
|
||||
return x_out.type_as(x)
|
||||
|
||||
|
||||
class QwenTimestepProjEmbeddings(nn.Module):
|
||||
def __init__(self, embedding_dim):
|
||||
super().__init__()
|
||||
|
||||
self.time_proj = Timesteps(num_channels=256, flip_sin_to_cos=True, downscale_freq_shift=0, scale=1000)
|
||||
self.timestep_embedder = TimestepEmbedding(in_channels=256, time_embed_dim=embedding_dim)
|
||||
|
||||
def forward(self, timestep, hidden_states):
|
||||
timesteps_proj = self.time_proj(timestep)
|
||||
timesteps_emb = self.timestep_embedder(timesteps_proj.to(dtype=hidden_states.dtype)) # (N, D)
|
||||
|
||||
conditioning = timesteps_emb
|
||||
|
||||
return conditioning
|
||||
|
||||
|
||||
class QwenEmbedRope(nn.Module):
|
||||
def __init__(self, theta: int, axes_dim: List[int], scale_rope=False):
|
||||
super().__init__()
|
||||
self.theta = theta
|
||||
self.axes_dim = axes_dim
|
||||
pos_index = torch.arange(4096)
|
||||
neg_index = torch.arange(4096).flip(0) * -1 - 1
|
||||
self.pos_freqs = torch.cat(
|
||||
[
|
||||
self.rope_params(pos_index, self.axes_dim[0], self.theta),
|
||||
self.rope_params(pos_index, self.axes_dim[1], self.theta),
|
||||
self.rope_params(pos_index, self.axes_dim[2], self.theta),
|
||||
],
|
||||
dim=1,
|
||||
)
|
||||
self.neg_freqs = torch.cat(
|
||||
[
|
||||
self.rope_params(neg_index, self.axes_dim[0], self.theta),
|
||||
self.rope_params(neg_index, self.axes_dim[1], self.theta),
|
||||
self.rope_params(neg_index, self.axes_dim[2], self.theta),
|
||||
],
|
||||
dim=1,
|
||||
)
|
||||
self.rope_cache = {}
|
||||
|
||||
# DO NOT USING REGISTER BUFFER HERE, IT WILL CAUSE COMPLEX NUMBERS LOSE ITS IMAGINARY PART
|
||||
self.scale_rope = scale_rope
|
||||
|
||||
def rope_params(self, index, dim, theta=10000):
|
||||
"""
|
||||
Args:
|
||||
index: [0, 1, 2, 3] 1D Tensor representing the position index of the token
|
||||
"""
|
||||
assert dim % 2 == 0
|
||||
freqs = torch.outer(index, 1.0 / torch.pow(theta, torch.arange(0, dim, 2).to(torch.float32).div(dim)))
|
||||
freqs = torch.polar(torch.ones_like(freqs), freqs)
|
||||
return freqs
|
||||
|
||||
def forward(self, video_fhw, txt_seq_lens, device):
|
||||
"""
|
||||
Args: video_fhw: [frame, height, width] a list of 3 integers representing the shape of the video Args:
|
||||
txt_length: [bs] a list of 1 integers representing the length of the text
|
||||
"""
|
||||
if self.pos_freqs.device != device:
|
||||
self.pos_freqs = self.pos_freqs.to(device)
|
||||
self.neg_freqs = self.neg_freqs.to(device)
|
||||
|
||||
if isinstance(video_fhw, list):
|
||||
video_fhw = video_fhw[0]
|
||||
if not isinstance(video_fhw, list):
|
||||
video_fhw = [video_fhw]
|
||||
|
||||
vid_freqs = []
|
||||
max_vid_index = 0
|
||||
for idx, fhw in enumerate(video_fhw):
|
||||
frame, height, width = fhw
|
||||
rope_key = f"{idx}_{height}_{width}"
|
||||
|
||||
if not torch.compiler.is_compiling():
|
||||
if rope_key not in self.rope_cache:
|
||||
self.rope_cache[rope_key] = self._compute_video_freqs(frame, height, width, idx)
|
||||
video_freq = self.rope_cache[rope_key]
|
||||
else:
|
||||
video_freq = self._compute_video_freqs(frame, height, width, idx)
|
||||
video_freq = video_freq.to(device)
|
||||
vid_freqs.append(video_freq)
|
||||
|
||||
if self.scale_rope:
|
||||
max_vid_index = max(height // 2, width // 2, max_vid_index)
|
||||
else:
|
||||
max_vid_index = max(height, width, max_vid_index)
|
||||
|
||||
max_len = max(txt_seq_lens)
|
||||
txt_freqs = self.pos_freqs[max_vid_index : max_vid_index + max_len, ...]
|
||||
vid_freqs = torch.cat(vid_freqs, dim=0)
|
||||
|
||||
return vid_freqs, txt_freqs
|
||||
|
||||
@functools.lru_cache(maxsize=None)
|
||||
def _compute_video_freqs(self, frame, height, width, idx=0):
|
||||
seq_lens = frame * height * width
|
||||
freqs_pos = self.pos_freqs.split([x // 2 for x in self.axes_dim], dim=1)
|
||||
freqs_neg = self.neg_freqs.split([x // 2 for x in self.axes_dim], dim=1)
|
||||
|
||||
freqs_frame = freqs_pos[0][idx : idx + frame].view(frame, 1, 1, -1).expand(frame, height, width, -1)
|
||||
if self.scale_rope:
|
||||
freqs_height = torch.cat([freqs_neg[1][-(height - height // 2) :], freqs_pos[1][: height // 2]], dim=0)
|
||||
freqs_height = freqs_height.view(1, height, 1, -1).expand(frame, height, width, -1)
|
||||
freqs_width = torch.cat([freqs_neg[2][-(width - width // 2) :], freqs_pos[2][: width // 2]], dim=0)
|
||||
freqs_width = freqs_width.view(1, 1, width, -1).expand(frame, height, width, -1)
|
||||
else:
|
||||
freqs_height = freqs_pos[1][:height].view(1, height, 1, -1).expand(frame, height, width, -1)
|
||||
freqs_width = freqs_pos[2][:width].view(1, 1, width, -1).expand(frame, height, width, -1)
|
||||
|
||||
freqs = torch.cat([freqs_frame, freqs_height, freqs_width], dim=-1).reshape(seq_lens, -1)
|
||||
return freqs.clone().contiguous()
|
||||
|
||||
|
||||
class QwenDoubleStreamAttnProcessor2_0:
|
||||
"""
|
||||
Attention processor for Qwen double-stream architecture, matching DoubleStreamLayerMegatron logic. This processor
|
||||
implements joint attention computation where text and image streams are processed together.
|
||||
"""
|
||||
|
||||
_attention_backend = None
|
||||
|
||||
def __init__(self):
|
||||
if not hasattr(F, "scaled_dot_product_attention"):
|
||||
raise ImportError(
|
||||
"QwenDoubleStreamAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0."
|
||||
)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
attn: Attention,
|
||||
hidden_states: torch.FloatTensor, # Image stream
|
||||
encoder_hidden_states: torch.FloatTensor = None, # Text stream
|
||||
encoder_hidden_states_mask: torch.FloatTensor = None,
|
||||
attention_mask: Optional[torch.FloatTensor] = None,
|
||||
image_rotary_emb: Optional[torch.Tensor] = None,
|
||||
) -> torch.FloatTensor:
|
||||
if encoder_hidden_states is None:
|
||||
raise ValueError("QwenDoubleStreamAttnProcessor2_0 requires encoder_hidden_states (text stream)")
|
||||
|
||||
seq_txt = encoder_hidden_states.shape[1]
|
||||
|
||||
# Compute QKV for image stream (sample projections)
|
||||
img_query = attn.to_q(hidden_states)
|
||||
img_key = attn.to_k(hidden_states)
|
||||
img_value = attn.to_v(hidden_states)
|
||||
|
||||
# Compute QKV for text stream (context projections)
|
||||
txt_query = attn.add_q_proj(encoder_hidden_states)
|
||||
txt_key = attn.add_k_proj(encoder_hidden_states)
|
||||
txt_value = attn.add_v_proj(encoder_hidden_states)
|
||||
|
||||
# Reshape for multi-head attention
|
||||
img_query = img_query.unflatten(-1, (attn.heads, -1))
|
||||
img_key = img_key.unflatten(-1, (attn.heads, -1))
|
||||
img_value = img_value.unflatten(-1, (attn.heads, -1))
|
||||
|
||||
txt_query = txt_query.unflatten(-1, (attn.heads, -1))
|
||||
txt_key = txt_key.unflatten(-1, (attn.heads, -1))
|
||||
txt_value = txt_value.unflatten(-1, (attn.heads, -1))
|
||||
|
||||
# Apply QK normalization
|
||||
if attn.norm_q is not None:
|
||||
img_query = attn.norm_q(img_query)
|
||||
if attn.norm_k is not None:
|
||||
img_key = attn.norm_k(img_key)
|
||||
if attn.norm_added_q is not None:
|
||||
txt_query = attn.norm_added_q(txt_query)
|
||||
if attn.norm_added_k is not None:
|
||||
txt_key = attn.norm_added_k(txt_key)
|
||||
|
||||
# Apply RoPE
|
||||
if image_rotary_emb is not None:
|
||||
img_freqs, txt_freqs = image_rotary_emb
|
||||
img_query = apply_rotary_emb_qwen(img_query, img_freqs, use_real=False)
|
||||
img_key = apply_rotary_emb_qwen(img_key, img_freqs, use_real=False)
|
||||
txt_query = apply_rotary_emb_qwen(txt_query, txt_freqs, use_real=False)
|
||||
txt_key = apply_rotary_emb_qwen(txt_key, txt_freqs, use_real=False)
|
||||
|
||||
# Concatenate for joint attention
|
||||
# Order: [text, image]
|
||||
joint_query = torch.cat([txt_query, img_query], dim=1)
|
||||
joint_key = torch.cat([txt_key, img_key], dim=1)
|
||||
joint_value = torch.cat([txt_value, img_value], dim=1)
|
||||
|
||||
# Compute joint attention
|
||||
joint_hidden_states = dispatch_attention_fn(
|
||||
joint_query,
|
||||
joint_key,
|
||||
joint_value,
|
||||
attn_mask=attention_mask,
|
||||
dropout_p=0.0,
|
||||
is_causal=False,
|
||||
backend=self._attention_backend,
|
||||
)
|
||||
|
||||
# Reshape back
|
||||
joint_hidden_states = joint_hidden_states.flatten(2, 3)
|
||||
joint_hidden_states = joint_hidden_states.to(joint_query.dtype)
|
||||
|
||||
# Split attention outputs back
|
||||
txt_attn_output = joint_hidden_states[:, :seq_txt, :] # Text part
|
||||
img_attn_output = joint_hidden_states[:, seq_txt:, :] # Image part
|
||||
|
||||
# Apply output projections
|
||||
img_attn_output = attn.to_out[0](img_attn_output)
|
||||
if len(attn.to_out) > 1:
|
||||
img_attn_output = attn.to_out[1](img_attn_output) # dropout
|
||||
|
||||
txt_attn_output = attn.to_add_out(txt_attn_output)
|
||||
|
||||
return img_attn_output, txt_attn_output
|
||||
|
||||
|
||||
@maybe_allow_in_graph
|
||||
class QwenImageTransformerBlock(nn.Module):
|
||||
def __init__(
|
||||
self, dim: int, num_attention_heads: int, attention_head_dim: int, qk_norm: str = "rms_norm", eps: float = 1e-6
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
self.dim = dim
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.attention_head_dim = attention_head_dim
|
||||
|
||||
# Image processing modules
|
||||
self.img_mod = nn.Sequential(
|
||||
nn.SiLU(),
|
||||
nn.Linear(dim, 6 * dim, bias=True), # For scale, shift, gate for norm1 and norm2
|
||||
)
|
||||
self.img_norm1 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
|
||||
self.attn = Attention(
|
||||
query_dim=dim,
|
||||
cross_attention_dim=None, # Enable cross attention for joint computation
|
||||
added_kv_proj_dim=dim, # Enable added KV projections for text stream
|
||||
dim_head=attention_head_dim,
|
||||
heads=num_attention_heads,
|
||||
out_dim=dim,
|
||||
context_pre_only=False,
|
||||
bias=True,
|
||||
processor=QwenDoubleStreamAttnProcessor2_0(),
|
||||
qk_norm=qk_norm,
|
||||
eps=eps,
|
||||
)
|
||||
self.img_norm2 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
|
||||
self.img_mlp = FeedForward(dim=dim, dim_out=dim, activation_fn="gelu-approximate")
|
||||
|
||||
# Text processing modules
|
||||
self.txt_mod = nn.Sequential(
|
||||
nn.SiLU(),
|
||||
nn.Linear(dim, 6 * dim, bias=True), # For scale, shift, gate for norm1 and norm2
|
||||
)
|
||||
self.txt_norm1 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
|
||||
# Text doesn't need separate attention - it's handled by img_attn joint computation
|
||||
self.txt_norm2 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
|
||||
self.txt_mlp = FeedForward(dim=dim, dim_out=dim, activation_fn="gelu-approximate")
|
||||
|
||||
def _modulate(self, x, mod_params):
|
||||
"""Apply modulation to input tensor"""
|
||||
shift, scale, gate = mod_params.chunk(3, dim=-1)
|
||||
return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1), gate.unsqueeze(1)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
encoder_hidden_states: torch.Tensor,
|
||||
encoder_hidden_states_mask: torch.Tensor,
|
||||
temb: torch.Tensor,
|
||||
image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
|
||||
joint_attention_kwargs: Optional[Dict[str, Any]] = None,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
# Get modulation parameters for both streams
|
||||
img_mod_params = self.img_mod(temb) # [B, 6*dim]
|
||||
txt_mod_params = self.txt_mod(temb) # [B, 6*dim]
|
||||
|
||||
# Split modulation parameters for norm1 and norm2
|
||||
img_mod1, img_mod2 = img_mod_params.chunk(2, dim=-1) # Each [B, 3*dim]
|
||||
txt_mod1, txt_mod2 = txt_mod_params.chunk(2, dim=-1) # Each [B, 3*dim]
|
||||
|
||||
# Process image stream - norm1 + modulation
|
||||
img_normed = self.img_norm1(hidden_states)
|
||||
img_modulated, img_gate1 = self._modulate(img_normed, img_mod1)
|
||||
|
||||
# Process text stream - norm1 + modulation
|
||||
txt_normed = self.txt_norm1(encoder_hidden_states)
|
||||
txt_modulated, txt_gate1 = self._modulate(txt_normed, txt_mod1)
|
||||
|
||||
# Use QwenAttnProcessor2_0 for joint attention computation
|
||||
# This directly implements the DoubleStreamLayerMegatron logic:
|
||||
# 1. Computes QKV for both streams
|
||||
# 2. Applies QK normalization and RoPE
|
||||
# 3. Concatenates and runs joint attention
|
||||
# 4. Splits results back to separate streams
|
||||
joint_attention_kwargs = joint_attention_kwargs or {}
|
||||
attn_output = self.attn(
|
||||
hidden_states=img_modulated, # Image stream (will be processed as "sample")
|
||||
encoder_hidden_states=txt_modulated, # Text stream (will be processed as "context")
|
||||
encoder_hidden_states_mask=encoder_hidden_states_mask,
|
||||
image_rotary_emb=image_rotary_emb,
|
||||
**joint_attention_kwargs,
|
||||
)
|
||||
|
||||
# QwenAttnProcessor2_0 returns (img_output, txt_output) when encoder_hidden_states is provided
|
||||
img_attn_output, txt_attn_output = attn_output
|
||||
|
||||
# Apply attention gates and add residual (like in Megatron)
|
||||
hidden_states = hidden_states + img_gate1 * img_attn_output
|
||||
encoder_hidden_states = encoder_hidden_states + txt_gate1 * txt_attn_output
|
||||
|
||||
# Process image stream - norm2 + MLP
|
||||
img_normed2 = self.img_norm2(hidden_states)
|
||||
img_modulated2, img_gate2 = self._modulate(img_normed2, img_mod2)
|
||||
img_mlp_output = self.img_mlp(img_modulated2)
|
||||
hidden_states = hidden_states + img_gate2 * img_mlp_output
|
||||
|
||||
# Process text stream - norm2 + MLP
|
||||
txt_normed2 = self.txt_norm2(encoder_hidden_states)
|
||||
txt_modulated2, txt_gate2 = self._modulate(txt_normed2, txt_mod2)
|
||||
txt_mlp_output = self.txt_mlp(txt_modulated2)
|
||||
encoder_hidden_states = encoder_hidden_states + txt_gate2 * txt_mlp_output
|
||||
|
||||
# Clip to prevent overflow for fp16
|
||||
if encoder_hidden_states.dtype == torch.float16:
|
||||
encoder_hidden_states = encoder_hidden_states.clip(-65504, 65504)
|
||||
if hidden_states.dtype == torch.float16:
|
||||
hidden_states = hidden_states.clip(-65504, 65504)
|
||||
|
||||
return encoder_hidden_states, hidden_states
|
||||
|
||||
|
||||
class QwenImageTransformer2DModel(ModelMixin, ConfigMixin, PeftAdapterMixin, FromOriginalModelMixin, CacheMixin, AttentionMixin):
|
||||
"""
|
||||
The Transformer model introduced in Qwen.
|
||||
|
||||
Args:
|
||||
patch_size (`int`, defaults to `2`):
|
||||
Patch size to turn the input data into small patches.
|
||||
in_channels (`int`, defaults to `64`):
|
||||
The number of channels in the input.
|
||||
out_channels (`int`, *optional*, defaults to `None`):
|
||||
The number of channels in the output. If not specified, it defaults to `in_channels`.
|
||||
num_layers (`int`, defaults to `60`):
|
||||
The number of layers of dual stream DiT blocks to use.
|
||||
attention_head_dim (`int`, defaults to `128`):
|
||||
The number of dimensions to use for each attention head.
|
||||
num_attention_heads (`int`, defaults to `24`):
|
||||
The number of attention heads to use.
|
||||
joint_attention_dim (`int`, defaults to `3584`):
|
||||
The number of dimensions to use for the joint attention (embedding/channel dimension of
|
||||
`encoder_hidden_states`).
|
||||
guidance_embeds (`bool`, defaults to `False`):
|
||||
Whether to use guidance embeddings for guidance-distilled variant of the model.
|
||||
axes_dims_rope (`Tuple[int]`, defaults to `(16, 56, 56)`):
|
||||
The dimensions to use for the rotary positional embeddings.
|
||||
"""
|
||||
|
||||
_supports_gradient_checkpointing = True
|
||||
_no_split_modules = ["QwenImageTransformerBlock"]
|
||||
_skip_layerwise_casting_patterns = ["pos_embed", "norm"]
|
||||
_repeated_blocks = ["QwenImageTransformerBlock"]
|
||||
|
||||
@register_to_config
|
||||
def __init__(
|
||||
self,
|
||||
patch_size: int = 2,
|
||||
in_channels: int = 64,
|
||||
out_channels: Optional[int] = 16,
|
||||
num_layers: int = 60,
|
||||
attention_head_dim: int = 128,
|
||||
num_attention_heads: int = 24,
|
||||
joint_attention_dim: int = 3584,
|
||||
guidance_embeds: bool = False, # TODO: this should probably be removed
|
||||
axes_dims_rope: Tuple[int, int, int] = (16, 56, 56),
|
||||
):
|
||||
super().__init__()
|
||||
self.out_channels = out_channels or in_channels
|
||||
self.inner_dim = num_attention_heads * attention_head_dim
|
||||
|
||||
self.pos_embed = QwenEmbedRope(theta=10000, axes_dim=list(axes_dims_rope), scale_rope=True)
|
||||
|
||||
self.time_text_embed = QwenTimestepProjEmbeddings(embedding_dim=self.inner_dim)
|
||||
|
||||
self.txt_norm = RMSNorm(joint_attention_dim, eps=1e-6)
|
||||
|
||||
self.img_in = nn.Linear(in_channels, self.inner_dim)
|
||||
self.txt_in = nn.Linear(joint_attention_dim, self.inner_dim)
|
||||
|
||||
self.transformer_blocks = nn.ModuleList(
|
||||
[
|
||||
QwenImageTransformerBlock(
|
||||
dim=self.inner_dim,
|
||||
num_attention_heads=num_attention_heads,
|
||||
attention_head_dim=attention_head_dim,
|
||||
)
|
||||
for _ in range(num_layers)
|
||||
]
|
||||
)
|
||||
|
||||
self.norm_out = AdaLayerNormContinuous(self.inner_dim, self.inner_dim, elementwise_affine=False, eps=1e-6)
|
||||
self.proj_out = nn.Linear(self.inner_dim, patch_size * patch_size * self.out_channels, bias=True)
|
||||
|
||||
self.gradient_checkpointing = False
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
encoder_hidden_states: torch.Tensor = None,
|
||||
encoder_hidden_states_mask: torch.Tensor = None,
|
||||
timestep: torch.LongTensor = None,
|
||||
image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
|
||||
guidance: torch.Tensor = None, # TODO: this should probably be removed
|
||||
attention_kwargs: Optional[Dict[str, Any]] = None,
|
||||
return_dict: bool = True,
|
||||
) -> Union[torch.Tensor, Transformer2DModelOutput]:
|
||||
"""
|
||||
The [`QwenTransformer2DModel`] forward method.
|
||||
|
||||
Args:
|
||||
hidden_states (`torch.Tensor` of shape `(batch_size, image_sequence_length, in_channels)`):
|
||||
Input `hidden_states`.
|
||||
encoder_hidden_states (`torch.Tensor` of shape `(batch_size, text_sequence_length, joint_attention_dim)`):
|
||||
Conditional embeddings (embeddings computed from the input conditions such as prompts) to use.
|
||||
encoder_hidden_states_mask (`torch.Tensor` of shape `(batch_size, text_sequence_length)`):
|
||||
Mask of the input conditions.
|
||||
timestep ( `torch.LongTensor`):
|
||||
Used to indicate denoising step.
|
||||
attention_kwargs (`dict`, *optional*):
|
||||
A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
|
||||
`self.processor` in
|
||||
[diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
|
||||
return_dict (`bool`, *optional*, defaults to `True`):
|
||||
Whether or not to return a [`~models.transformer_2d.Transformer2DModelOutput`] instead of a plain
|
||||
tuple.
|
||||
|
||||
Returns:
|
||||
If `return_dict` is True, an [`~models.transformer_2d.Transformer2DModelOutput`] is returned, otherwise a
|
||||
`tuple` where the first element is the sample tensor.
|
||||
"""
|
||||
if attention_kwargs is not None:
|
||||
attention_kwargs = attention_kwargs.copy()
|
||||
lora_scale = attention_kwargs.pop("scale", 1.0)
|
||||
else:
|
||||
lora_scale = 1.0
|
||||
|
||||
if USE_PEFT_BACKEND:
|
||||
# weight the lora layers by setting `lora_scale` for each PEFT layer
|
||||
scale_lora_layers(self, lora_scale)
|
||||
else:
|
||||
if attention_kwargs is not None and attention_kwargs.get("scale", None) is not None:
|
||||
logger.warning(
|
||||
"Passing `scale` via `joint_attention_kwargs` when not using the PEFT backend is ineffective."
|
||||
)
|
||||
|
||||
hidden_states = self.img_in(hidden_states)
|
||||
|
||||
timestep = timestep.to(hidden_states.dtype)
|
||||
encoder_hidden_states = self.txt_norm(encoder_hidden_states)
|
||||
encoder_hidden_states = self.txt_in(encoder_hidden_states)
|
||||
|
||||
if guidance is not None:
|
||||
guidance = guidance.to(hidden_states.dtype) * 1000
|
||||
|
||||
temb = (
|
||||
self.time_text_embed(timestep, hidden_states)
|
||||
if guidance is None
|
||||
else self.time_text_embed(timestep, guidance, hidden_states)
|
||||
)
|
||||
|
||||
for index_block, block in enumerate(self.transformer_blocks):
|
||||
if torch.is_grad_enabled() and self.gradient_checkpointing:
|
||||
encoder_hidden_states, hidden_states = self._gradient_checkpointing_func(
|
||||
block,
|
||||
hidden_states,
|
||||
encoder_hidden_states,
|
||||
encoder_hidden_states_mask,
|
||||
temb,
|
||||
image_rotary_emb,
|
||||
)
|
||||
|
||||
else:
|
||||
encoder_hidden_states, hidden_states = block(
|
||||
hidden_states=hidden_states,
|
||||
encoder_hidden_states=encoder_hidden_states,
|
||||
encoder_hidden_states_mask=encoder_hidden_states_mask,
|
||||
temb=temb,
|
||||
image_rotary_emb=image_rotary_emb,
|
||||
joint_attention_kwargs=attention_kwargs,
|
||||
)
|
||||
|
||||
# Use only the image part (hidden_states) from the dual-stream blocks
|
||||
hidden_states = self.norm_out(hidden_states, temb)
|
||||
output = self.proj_out(hidden_states)
|
||||
|
||||
if USE_PEFT_BACKEND:
|
||||
# remove `lora_scale` from each PEFT layer
|
||||
unscale_lora_layers(self, lora_scale)
|
||||
|
||||
if not return_dict:
|
||||
return (output,)
|
||||
|
||||
return Transformer2DModelOutput(sample=output)
|
||||
@@ -1,31 +1,37 @@
|
||||
# The Bride:
|
||||
Lie flat on your back.
|
||||
Bend one leg and bring it up to your chest, the other leg stretched out.
|
||||
Lying flat on her back, bending one leg and bringing it up to the chest, the other leg stretched out.
|
||||
Place one arm under your head, and the other arm stretched out to the side.
|
||||
Capture the curves and smooth lines of the body, focusing on the subtle curves of the back and the stretch of the bent leg.
|
||||
Realistic, no background
|
||||
Head on top.
|
||||
Realistic, anatomically precise
|
||||
|
||||
# The Dragon:
|
||||
# The draak (beta):
|
||||
Lie on your back with your arms stretched out to the sides.
|
||||
Bring your legs up to your chest, and cross one leg over the other.
|
||||
Looking into the camera.
|
||||
Bring your legs up to your chest, slightly lift for invitation, and cross one leg over the other.
|
||||
Lean up to balance on your elbows, showcasing the extended arms and strong legs.
|
||||
Focus on the elegant twist of the body and the intertwined legs.
|
||||
Only two feet top, and two arms with hands on head.
|
||||
Realistic, no background
|
||||
Realistic, anatomically precise
|
||||
|
||||
# The Butterfly:
|
||||
Lie on your back with your knees bent and your feet touching.
|
||||
Clasp your hands around your legs and pull your feet up towards your chest.
|
||||
Extend one leg over the other, showcasing the bent knees and the exposed stretch of the legs.
|
||||
Focus on the subtle, sexy stretch of the entire body.
|
||||
Realistic
|
||||
Lying flat on her back in a reclined butterfly pose.
|
||||
Soles of feet pressed firmly together, knees relaxed and falling open outward to the sides, hips fully open.
|
||||
Arms resting loosely overhead or at her sides.
|
||||
Looking directly into the camera.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# Butterfly (recline):
|
||||
lying flat on back in reclined butterfly pose.
|
||||
Soles of feet pressed together, knees relaxed and falling open outward to sides, hips fully open.
|
||||
Arms resting loosely overhead or at sides.
|
||||
Looking directly into camera. Serene, open.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Sleeper:
|
||||
Lie on your side with your upper arm behind your head and your lower arm stretched out.
|
||||
The upper leg bent, while the lower leg is straight.
|
||||
Lean slightly into the pose, showcasing the straight line of the lower leg and the bent upper leg.
|
||||
Highlight the soft, supple curves of the torso and hips.
|
||||
Realistic
|
||||
Realistic, anatomically precise
|
||||
|
||||
# The Backbend:
|
||||
Lie on your back with your knees bent and your feet touching.
|
||||
@@ -38,8 +44,7 @@ Realistic, anatomically precise
|
||||
Lie on your back with your knees bent and your feet touching.
|
||||
Bring your legs up to your chest and cross them.
|
||||
Clasp your hands around your feet and pull your legs towards your chest.
|
||||
Focus on the delicate, intricate pose and the symmetry of the legs.
|
||||
Realistic, anatomically precise
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# Grounded vs. Vulnerable S-Curve:
|
||||
A realistic nude photo.
|
||||
@@ -52,11 +57,17 @@ Hips are slightly lowered toward the ground, accentuating the lumbar curve.
|
||||
The pose conveys sultry energy, feminine power, and visual tension.
|
||||
Anatomically precise, high detail.
|
||||
|
||||
# S-Curve (beta):
|
||||
Kneel on all fours in an asymmetrical S-curve pose.
|
||||
One side grounded and stable — hand and knee firmly planted, muscles taut.
|
||||
Back arched upward, hips lowered, head lifted looking into camera.
|
||||
Perfect anatomy, realistic.
|
||||
|
||||
# The kneeler:
|
||||
A realistic artistic nude female.
|
||||
The model kneels on all fours with hands planted firmly under the shoulders and knees directly under the hips. One arm and the opposite leg are lifted completely off the ground, creating a powerful diagonal stretch across the body. Hips are lowered toward the ground. The composition emphasizes the muscular tension in the lifted limbs and the continuous curved lines of the arched back and torso. Strength and dynamic energy radiate from the pose. Studio lighting, anatomically precise, no background, plain neutral backdrop, high detail, professional figure reference photography.
|
||||
|
||||
# The kneele:
|
||||
# The kneele (beta):
|
||||
The model kneels on all fours with her hands planted firmly under her shoulders and her knees under her hips, creating a powerful, inviting stance. Her back is arched slightly, lifting her head towards the ceiling and drawing attention to her exposed back and shoulders.
|
||||
The muscles in her arm and leg are taut and defined, showing the effort required to hold the pose.
|
||||
Her right arm and leg are still and grounded, with her hands on the ground and her right leg tucked under her body. Her left side is vulnerable, inviting the viewer into her personal space and adding an element of risk and allure.
|
||||
@@ -77,4 +88,626 @@ Lie on your back with your arms stretched out to the sides.
|
||||
Bring your legs up to your chest and cross them.
|
||||
Bend your knees and clasp your hands around your feet.
|
||||
Lean back slightly into the pose, showcasing the bend in the knees and the clasp around the feet.
|
||||
Realistic, perfect anatomy
|
||||
Realistic, perfect anatomy
|
||||
|
||||
# The Butterfly2:
|
||||
Lays down with knees bent and feet touching, soles of feet pressed together in butterfly pose.
|
||||
Clasp your hands around your legs and pull your feet up towards your chest.
|
||||
Extend one leg over the other.
|
||||
Head on top.
|
||||
Realistic, perfect anatomy
|
||||
|
||||
# The Kneeling (Dynamic):
|
||||
kneeling on all fours, hands under shoulders, knees under hips.
|
||||
one arm and opposite leg slightly lifted off the ground, creating diagonal tension.
|
||||
hips lowered toward the ground, back forming a smooth curved arc.
|
||||
muscular definition in lifted limbs, continuous curved lines of torso.
|
||||
Looking directly into camera. Strong, dynamic.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Sphinx(beta):
|
||||
lying on stomach, propped up on forearms.
|
||||
elbows under shoulders, chest lifted, back arched deeply.
|
||||
legs extended straight behind, toes pointed.
|
||||
head lowered, face turned down toward the ground, chin tucked.
|
||||
Looking down, not at camera. Regal, poised.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Offering:
|
||||
kneeling upright, sitting back on heels.
|
||||
arms extended straight overhead, hands open and reaching upward.
|
||||
back arched slightly, chest open, shoulders down.
|
||||
hips anchored heavy on heels, spine long and stretched.
|
||||
Looking directly into camera. Devotional, open.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Reclining Twist:
|
||||
lying on side.
|
||||
top leg bent and raised, foot resting on ground in front of bottom leg.
|
||||
bottom leg extended straight, toes pointed.
|
||||
top arm resting on bent knee or reaching overhead.
|
||||
torso twisted slightly toward camera, hips stacked.
|
||||
Looking directly into camera. Relaxed, composed.
|
||||
Realistic, anatomically precise.
|
||||
|
||||
# The Ascension:
|
||||
lying on back.
|
||||
arms extended overhead, hands clasped or reaching past the head.
|
||||
legs lifted straight up together, fully extended toward ceiling.
|
||||
lower back pressed flat to the ground, shoulders relaxed.
|
||||
vertical line from shoulders through heels.
|
||||
Looking directly into camera. Weightless, transcendent.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Bridge:
|
||||
lying on back, feet flat on ground, knees bent.
|
||||
hips lifted high off the ground, forming an arch with the body.
|
||||
shoulders and head remain on the ground.
|
||||
arms extended along the ground overhead or hands clasped under the back.
|
||||
Looking directly into camera. Strong, open.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Lotus (recline):
|
||||
lying flat on back.
|
||||
legs crossed in lotus position, knees falling open to sides.
|
||||
arms resting at sides or one hand on belly, one on chest.
|
||||
spine long and flat against the ground.
|
||||
Looking directly into camera. Centered, tranquil.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Archer:
|
||||
lying on side.
|
||||
top leg bent and drawn up toward chest, foot planted in front of bottom leg.
|
||||
top arm reaching back to grasp the ankle or foot of the bent top leg.
|
||||
bottom leg extended straight, body forming a bow shape.
|
||||
Looking directly into camera. Tense, focused.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Crescent:
|
||||
standing or kneeling, one knee on ground, opposite foot planted forward.
|
||||
arms extended overhead, hands clasped or reaching.
|
||||
torso arched backward deeply, chest open to the sky.
|
||||
hips pushed forward, spine in deep backbend.
|
||||
Looking upward or directly into camera. Expansive, vulnerable.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Cocoon:
|
||||
lying on side, knees drawn up tightly to chest.
|
||||
arms wrapped around legs, hands clasping shins or ankles.
|
||||
body curled into a tight fetal ball.
|
||||
head tucked down, chin near knees.
|
||||
Looking directly into camera. Protected, intimate.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Swan Dive:
|
||||
lying on stomach, arms extended forward along the ground.
|
||||
chest and shoulders lifted, back arched deeply.
|
||||
legs lifted off the ground behind, knees straight or slightly bent, feet pointed.
|
||||
head lifted, neck elongated.
|
||||
Looking directly into camera. Graceful, soaring.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Suspended Split:
|
||||
lying on back.
|
||||
legs lifted and spread wide in a full split, perpendicular to the body.
|
||||
arms extended to sides or reaching up to support the legs.
|
||||
lower back grounded, shoulders relaxed.
|
||||
Looking directly into camera. Flexible, exposed.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Embrace:
|
||||
seated, legs extended straight forward, feet flexed.
|
||||
torso folded forward over the legs, chest resting on thighs.
|
||||
arms wrapped around the legs, hands reaching for feet or ankles.
|
||||
head turned to the side, cheek resting on the knees.
|
||||
Looking directly into camera. Intimate, yielding.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Cascade:
|
||||
seated, one leg extended straight forward, the other bent with foot tucked to inner thigh.
|
||||
torso twisted toward the extended leg, folding forward over it.
|
||||
arms reaching along the extended leg, hands grasping foot or ankle.
|
||||
head lowered near the shin.
|
||||
Looking directly into camera. Fluid, surrendered.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Flame:
|
||||
standing on one leg, opposite leg lifted and bent, foot pressed against inner thigh.
|
||||
arms raised overhead, hands pressed together.
|
||||
torso upright, hips squared, spine long.
|
||||
balanced, rooted through the standing foot.
|
||||
Looking directly into camera. Poised, burning.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Sphinx (Inverted):
|
||||
lying on stomach, propped up on forearms.
|
||||
elbows under shoulders, chest lifted, back arched deeply.
|
||||
legs extended straight behind, toes pointed.
|
||||
head lowered, face turned down toward the ground, chin tucked.
|
||||
Looking down, not at camera. Contemplative, submissive.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Garter:
|
||||
seated on the edge of a surface, legs slightly parted.
|
||||
hands at the thighs, fingers hooking into the top of thigh-high stockings or garter straps.
|
||||
torso leaning forward slightly, shoulders rolled back, chest presented.
|
||||
head tilted, lips parted.
|
||||
Looking directly into camera. Teasing, anticipatory.
|
||||
Perfect anatomy, realistic, trying on erotic lingerie
|
||||
|
||||
# The Unclasp:
|
||||
standing, back partially turned toward camera over one shoulder.
|
||||
arms reaching behind, hands at the back working to unhook a bra clasp.
|
||||
elbows drawn inward, shoulder blades prominent.
|
||||
hips cocked to one side, weight on one leg.
|
||||
Looking back over the shoulder into camera. Inviting, conspiratorial.
|
||||
Perfect anatomy, realistic, trying on erotic lingerie
|
||||
|
||||
# The Strap Slide:
|
||||
standing, one strap of a slip or camisole slid off the shoulder.
|
||||
arm relaxed at the side, letting the fabric hang loose on one side.
|
||||
opposite hand touching the collarbone or trailing down the sternum.
|
||||
hip popped, body in a subtle S-curve.
|
||||
Looking directly into camera. Seductive, slow.
|
||||
Perfect anatomy, realistic, trying on erotic lingerie
|
||||
|
||||
# The Lace Pull:
|
||||
lying on back, knees bent, feet flat on the ground.
|
||||
hands at the hips, fingers hooked into the waistband of lace panties.
|
||||
hips lifted slightly off the ground, fabric being pulled downward.
|
||||
thighs parted slightly, tension in the abdomen.
|
||||
Looking directly into camera. Vulnerable, exposed.
|
||||
Perfect anatomy, realistic, trying on erotic lingerie
|
||||
|
||||
# The Corset Lace:
|
||||
standing, back to camera, head turned to look over shoulder.
|
||||
arms reaching behind to pull the laces of a corset tighter.
|
||||
elbows wide, shoulder blades drawn together, waist cinched narrow.
|
||||
legs straight, feet apart for balance.
|
||||
Looking back into camera. Restricted, breathless.
|
||||
Perfect anatomy, realistic, trying on erotic lingerie
|
||||
|
||||
# The Garter Roll:
|
||||
lying on stomach, propped on elbows.
|
||||
legs bent at the knees, feet kicked up and crossed in the air.
|
||||
hands reaching back to roll a stocking up the calf or thigh.
|
||||
back arched, hips slightly lifted, fabric of lingerie riding up.
|
||||
Looking back over the shoulder into camera. Playful, coquettish.
|
||||
Perfect anatomy, realistic, trying on erotic lingerie
|
||||
|
||||
# The Hook:
|
||||
standing facing camera, one leg crossed over the other at the ankle.
|
||||
both hands at the center of the chest, fingers working the hook-and-eye closure of a bodysuit or teddy.
|
||||
head tilted down, eyes looking up through lashes.
|
||||
shoulders rounded forward, creating a valley of shadow at the cleavage.
|
||||
Looking up into camera. Coy, demure.
|
||||
Perfect anatomy, realistic, trying on erotic lingerie
|
||||
|
||||
# The Thigh Band:
|
||||
seated with legs extended forward, torso reclined back on hands.
|
||||
one knee bent, foot flat on the ground, thigh open.
|
||||
opposite hand resting on the inner thigh, fingers tracing the edge of a garter band or stocking top.
|
||||
head thrown back slightly, neck exposed.
|
||||
Looking directly into camera. Languid, indulgent.
|
||||
Perfect anatomy, realistic, trying on erotic lingerie
|
||||
|
||||
# The Mirror Check:
|
||||
standing in three-quarter profile, one hand on the hip.
|
||||
other hand smoothing the fabric of a sheer chemise or babydoll down the side of the torso.
|
||||
weight on the back leg, front leg slightly bent, toe pointed.
|
||||
chin lifted, checking the fit in an implied mirror.
|
||||
Looking to the side, eyes catching the camera in reflection. Self-aware, assessing.
|
||||
Perfect anatomy, realistic, trying on erotic lingerie
|
||||
|
||||
# The Snap:
|
||||
standing, legs apart, hands at the crotch of a bodysuit or teddy.
|
||||
fingers working the snap closures at the gusset.
|
||||
torso twisted slightly, one shoulder dropped lower than the other.
|
||||
lips pursed in concentration, then softening into a smile.
|
||||
Looking directly into camera. Direct, unapologetic.
|
||||
Perfect anatomy, realistic, trying on erotic lingerie
|
||||
|
||||
# The Shoulder Drop:
|
||||
standing, back to camera.
|
||||
shoulders rolling forward to let the straps of a slip or bra fall down the arms.
|
||||
arms hanging loose, fabric pooling at the elbows.
|
||||
head turned to look back over one shoulder.
|
||||
hips swayed to one side, creating a diagonal line from shoulder to hip.
|
||||
Looking back into camera. Revealing, inevitable.
|
||||
Perfect anatomy, realistic, trying on erotic lingerie
|
||||
|
||||
# The Step-In:
|
||||
standing on one leg, opposite leg lifted, foot stepping into the leg opening of panties or a bodysuit.
|
||||
hands holding the garment open at thigh level.
|
||||
torso leaning slightly to balance, one arm extended for stability.
|
||||
head down, watching the fabric glide up the leg.
|
||||
Looking down, then up into camera with a half-smile. Intimate, domestic.
|
||||
Perfect anatomy, realistic, trying on erotic lingerie
|
||||
|
||||
# The Pearl Strand:
|
||||
lying on back, knees bent and parted, feet flat on the ground.
|
||||
hands at the neck, fingers working to fasten a pearl necklace or choker.
|
||||
elbows wide, chest lifted, collarbones prominent.
|
||||
head tilted back, throat elongated.
|
||||
Looking directly into camera. Elegant, adorned.
|
||||
Perfect anatomy, realistic, trying on erotic lingerie
|
||||
|
||||
# The Glove Tug:
|
||||
seated on the floor, legs tucked to one side.
|
||||
one arm extended, fingers of the opposite hand slowly pulling a long satin or lace glove up the forearm.
|
||||
head turned to watch the motion, lips slightly parted.
|
||||
shoulders relaxed, back curved in a gentle C-shape.
|
||||
Looking at the glove, then into camera. Methodical, sensual.
|
||||
Perfect anatomy, realistic, trying on erotic lingerie
|
||||
|
||||
# The Ribbon Tie:
|
||||
standing, legs apart, hands behind the back at the waist.
|
||||
fingers working to tie a satin ribbon or bow at the small of the back.
|
||||
torso twisted to expose the side, ribs visible with the stretch.
|
||||
head turned over the shoulder, chin lifted.
|
||||
Looking back into camera. Decorative, bound.
|
||||
Perfect anatomy, realistic, trying on erotic lingerie
|
||||
|
||||
# The Hem Lift:
|
||||
standing, legs crossed at the ankles.
|
||||
both hands at the sides of a short slip or nightgown, fingers pinching the hem.
|
||||
fabric lifted slowly, revealing the upper thighs.
|
||||
weight shifted to one hip, creating a diagonal line.
|
||||
Looking directly into camera, chin down, eyes up. Provocative, measured.
|
||||
Perfect anatomy, realistic, trying on erotic lingerie
|
||||
|
||||
# The Clasp Front:
|
||||
lying on side, propped on one elbow.
|
||||
free hand at the center of the chest, working the front clasp of a bra.
|
||||
fingers pinching the metal closure, about to release it.
|
||||
top leg bent and drawn forward, knee resting on the ground.
|
||||
Looking directly into camera. Anticipatory, breath held.
|
||||
Perfect anatomy, realistic, trying on erotic lingerie
|
||||
|
||||
# The Sash Untie:
|
||||
standing, one hand on the hip.
|
||||
other hand pulling the end of a silk sash or robe tie.
|
||||
hips cocked, weight on the back leg.
|
||||
fabric beginning to part at the waist, hinting at what lies beneath.
|
||||
Looking directly into camera. Unwrapping, revelatory.
|
||||
Perfect anatomy, realistic, trying on erotic lingerie
|
||||
|
||||
# The Zipper Pull:
|
||||
standing in profile, one shoulder dropped toward camera.
|
||||
arm reaching behind the back, fingers finding the pull of a side or back zipper.
|
||||
head turned to look back, neck stretched.
|
||||
zipper descending slowly, fabric parting along the spine or side.
|
||||
Looking back into camera. Mechanical, inevitable.
|
||||
Perfect anatomy, realistic, trying on erotic lingerie
|
||||
|
||||
# The Stocking Peel:
|
||||
seated, one leg extended, the other bent with foot flat on the ground.
|
||||
hands at the ankle of the extended leg, slowly peeling a stocking downward.
|
||||
toe pointed, arch of the foot visible as the fabric reveals skin.
|
||||
torso leaning forward, hair falling over one shoulder.
|
||||
Looking at the leg, then into camera. Unveiling, deliberate.
|
||||
Perfect anatomy, realistic, trying on erotic lingerie
|
||||
|
||||
# The Halter Knot:
|
||||
standing, back to camera, arms raised overhead.
|
||||
hands at the nape of the neck, tying or untying the knot of a halter top.
|
||||
elbows wide, shoulder blades drawn together, back muscles defined.
|
||||
spine straight, hips narrow.
|
||||
Looking back over the shoulder into camera. Taut, suspended.
|
||||
Perfect anatomy, realistic, trying on erotic lingerie
|
||||
|
||||
# The Garter Snap:
|
||||
lying on back, one knee bent, foot flat, the other leg extended.
|
||||
hand at the inner thigh, fingers snapping the clasp of a garter to a stocking.
|
||||
hips slightly rotated to expose the attachment point.
|
||||
head turned to the side, watching the action.
|
||||
Looking at the hand, then into camera. Precise, functional.
|
||||
Perfect anatomy, realistic, trying on erotic lingerie
|
||||
|
||||
# The Drape:
|
||||
standing, one hand holding a sheer robe or kimono open at the chest.
|
||||
other arm hanging loose at the side.
|
||||
shoulders back, one hip popped, fabric cascading down the body.
|
||||
head tilted, hair falling over one eye.
|
||||
Looking directly into camera. Framed, theatrical.
|
||||
Perfect anatomy, realistic, trying on erotic lingerie
|
||||
|
||||
# The Fold:
|
||||
seated on the floor, legs spread wide in a straddle.
|
||||
torso folded forward between the legs, chest approaching the ground.
|
||||
arms extended forward along the floor, hands reaching past the feet.
|
||||
head down, then lifting to look back through the legs at the camera.
|
||||
Looking back through the legs into camera. Folded, inverted.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Plow:
|
||||
lying on back, legs lifted overhead and lowered behind the head.
|
||||
toes touching or approaching the ground behind the head.
|
||||
arms extended along the ground, palms down for support.
|
||||
hips lifted high, spine deeply folded.
|
||||
Looking up at the legs, then into camera. Compressed, intense.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Mermaid:
|
||||
lying on stomach, legs together, feet pointed.
|
||||
arms extended forward, hands clasped.
|
||||
torso lifted, back arched, chest open.
|
||||
head thrown back, hair cascading down the spine.
|
||||
Looking upward, not at camera. Aquatic, flowing.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Scorpion:
|
||||
lying on stomach, propped on forearms.
|
||||
legs lifted and bent, knees drawing toward the shoulders.
|
||||
feet arched over the head, toes pointing toward the crown.
|
||||
back deeply arched, hips compressed.
|
||||
Looking forward, through the gap between the feet. Contorted, fierce.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Wreath:
|
||||
seated, legs crossed tightly, feet tucked under opposite thighs.
|
||||
arms wrapped around the body, hands clasping opposite shoulders.
|
||||
head resting on one shoulder, chin tucked.
|
||||
body forming a compact, self-contained circle.
|
||||
Looking directly into camera. Enclosed, self-held.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Needle:
|
||||
standing on one leg, torso folded forward at the hip.
|
||||
free leg extended straight back, parallel to the ground.
|
||||
arms extended back alongside the body, hands reaching toward the extended foot.
|
||||
head aligned with the spine, gaze directed at the ground.
|
||||
Looking down, not at camera. Linear, piercing.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Cobra:
|
||||
lying on stomach, hands under shoulders.
|
||||
arms straight, pushing the torso up and back.
|
||||
hips and legs remain on the ground.
|
||||
chest lifted high, shoulders rolled back.
|
||||
head tilted back, neck elongated.
|
||||
Looking upward, not at camera. Rising, alert.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Tuck:
|
||||
seated, knees drawn to chest, feet lifted off the ground.
|
||||
arms wrapped around the shins, hands clasping.
|
||||
back rounded, chin resting on the knees.
|
||||
body compacted into a tight ball, balanced on the sit bones.
|
||||
Looking directly into camera. Compact, balanced.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Dancer:
|
||||
standing on one leg, opposite leg lifted high behind, knee bent, foot grasped by the hand of the same side.
|
||||
free arm extended forward for balance.
|
||||
torso leaning forward, parallel to the ground.
|
||||
head lifted, looking along the extended arm.
|
||||
Looking forward, not directly at camera. Graceful, airborne.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Sphynx (Upright):
|
||||
seated with legs extended straight forward.
|
||||
torso leaning back, supported on the hands behind.
|
||||
chest lifted, head tilted back.
|
||||
legs together, toes pointed.
|
||||
body forming a long, reclined line from head to heels.
|
||||
Looking upward, not at camera. Regal, exposed.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Thread:
|
||||
lying on back, one leg extended straight up.
|
||||
arm of the same side reaching up to grasp the foot or ankle.
|
||||
other arm extended to the side, palm down.
|
||||
other leg bent, foot flat on the ground.
|
||||
head turned toward the raised leg.
|
||||
Looking at the leg, then into camera. Strung, pulled taut.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Anchor:
|
||||
seated, legs spread wide in a straddle.
|
||||
torso upright, spine long.
|
||||
arms extended to the sides at shoulder height, palms facing down.
|
||||
head centered, chin level.
|
||||
body forming a stable, grounded T-shape.
|
||||
Looking directly into camera. Rooted, immovable.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Cradle:
|
||||
lying on back, knees bent and drawn toward the chest.
|
||||
arms reaching through the gap between the legs, hands clasping the outer edges of the feet.
|
||||
legs spread wide, knees drawing toward the armpits.
|
||||
lower back flat on the ground, shoulders relaxed.
|
||||
Looking directly into camera. Held, cradled.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Pike:
|
||||
seated, legs extended straight forward together, feet flexed.
|
||||
torso folded forward over the legs, chest resting on the thighs.
|
||||
arms extended alongside the legs, hands reaching for the toes or beyond.
|
||||
head down, forehead near the shins.
|
||||
Looking down, not at camera. Sharp, folded.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Wheel:
|
||||
lying on back, feet flat on ground near the hips, hands flat on ground near the shoulders.
|
||||
hips lifted high, arms and legs straight, body forming an arch.
|
||||
chest open, head hanging back, hair cascading toward the ground.
|
||||
weight distributed evenly between hands and feet.
|
||||
Looking backward, upside down, into camera. Inverted, expansive.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Spiral:
|
||||
standing, feet apart.
|
||||
torso twisted deeply, one shoulder forward, the other back.
|
||||
arms extended to the sides, following the twist of the torso.
|
||||
head turned to look over the forward shoulder.
|
||||
hips squared, lower body stable, upper body rotated.
|
||||
Looking back over the shoulder into camera. Wound, coiled.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Ballerina:
|
||||
standing on the toes of one foot, opposite leg extended straight back, lifted high.
|
||||
torso leaning forward, arms extended to the sides in a soft curve.
|
||||
head lifted, neck long, gaze following the line of the back leg.
|
||||
body forming a long, diagonal line from fingertips to toes.
|
||||
Looking forward, not directly at camera. Light, elevated.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Nest:
|
||||
lying on side, knees drawn up, bottom arm extended under the head.
|
||||
top arm draped over the body, hand resting on the hip or waist.
|
||||
legs stacked, slightly bent, feet tucked together.
|
||||
body curled in a loose, relaxed C-shape.
|
||||
Looking directly into camera. Resting, nested.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Perch:
|
||||
seated with one leg bent, foot flat on the ground, knee raised.
|
||||
other leg extended to the side, foot pointed.
|
||||
arm of the bent leg side resting on the raised knee.
|
||||
other arm extended to the side or overhead.
|
||||
torso leaning slightly toward the extended leg.
|
||||
Looking directly into camera. Poised, watchful.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Bow:
|
||||
lying on stomach, knees bent, feet lifted toward the head.
|
||||
arms reaching back to grasp the ankles or feet.
|
||||
chest and thighs lifted off the ground, body forming a taut bow.
|
||||
head tilted back, neck stretched.
|
||||
Looking upward, not at camera. Tense, arced.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Fan:
|
||||
seated, legs spread wide in a straddle.
|
||||
torso upright, spine long.
|
||||
arms extended to the sides, palms open, fingers spread.
|
||||
head tilted slightly back, chin lifted.
|
||||
body forming a wide, open V-shape.
|
||||
Looking directly into camera. Open, radiant.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Sickle:
|
||||
lying on back, legs lifted and bent.
|
||||
knees drawing toward the chest, then extending and lowering toward the head.
|
||||
feet arched, toes pointing toward the ground behind the head.
|
||||
arms wrapped around the thighs or extended to the sides.
|
||||
body folded tightly, spine compressed.
|
||||
Looking at the knees, then into camera. Curved, hooked.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Lantern:
|
||||
kneeling, knees apart, sitting back on the heels.
|
||||
torso upright, spine long.
|
||||
arms extended overhead, hands clasped or holding an imaginary object.
|
||||
chest open, shoulders down.
|
||||
body forming a tall, contained column.
|
||||
Looking directly into camera. Illuminated, contained.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Compass:
|
||||
seated, one leg extended straight forward.
|
||||
other leg bent, foot placed on the inner thigh of the extended leg.
|
||||
torso folded forward over the extended leg.
|
||||
arm of the extended leg side reaching along the leg toward the foot.
|
||||
other arm wrapping around the back or reaching overhead.
|
||||
Looking down, then into camera. Directed, focused.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Vessel:
|
||||
lying on back, knees bent, feet flat on the ground.
|
||||
arms extended overhead, hands clasped or holding the knees.
|
||||
hips lifted slightly, lower back arched.
|
||||
chest open, throat exposed.
|
||||
body forming a shallow bowl or vessel shape.
|
||||
Looking directly into camera. Receptive, open.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Quiver:
|
||||
lying on stomach, legs together, arms at the sides.
|
||||
torso lifted in a backbend, chest and shoulders arching upward.
|
||||
head thrown back, neck stretched.
|
||||
legs remain on the ground, feet pointed.
|
||||
body forming a gentle, upward curve from hips.
|
||||
Looking upward, not at camera. Trembling, stretched.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Loom:
|
||||
seated, legs extended straight forward, feet flexed.
|
||||
torso upright, spine long.
|
||||
arms extended forward, hands reaching toward the feet.
|
||||
fingers spread, as if weaving or grasping threads.
|
||||
head centered, gaze directed at the hands.
|
||||
Looking at the hands, then into camera. Weaving, constructing.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Hourglass:
|
||||
standing, feet apart, hands on the hips.
|
||||
torso twisted, one shoulder forward, one back.
|
||||
weight shifted to one leg, opposite hip popped.
|
||||
waist compressed, creating a pronounced hourglass silhouette.
|
||||
head turned to look over the shoulder.
|
||||
Looking back into camera. Pinched, exaggerated.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Tidal:
|
||||
lying on side, bottom leg extended straight.
|
||||
top leg lifted high, extended, foot pointed.
|
||||
bottom arm extended overhead along the ground.
|
||||
top arm reaching up to grasp the lifted foot or ankle.
|
||||
body forming a long, stretched line.
|
||||
Looking at the lifted leg, then into camera. Flowing, stretched.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Hinge:
|
||||
standing, feet together.
|
||||
torso folded forward at the hips, back flat and parallel to the ground.
|
||||
arms extended back alongside the body, hands reaching toward the heels.
|
||||
legs straight, knees locked.
|
||||
head aligned with the spine, gaze directed at the ground.
|
||||
Looking down, not at camera. Mechanical, precise.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Chalice:
|
||||
lying on back, knees bent and drawn toward the chest.
|
||||
arms wrapped around the legs, hands clasping the shins.
|
||||
feet lifted, soles facing upward.
|
||||
lower back flat, shoulders relaxed.
|
||||
body forming a compact, cupped shape.
|
||||
Looking directly into camera. Held, cupped.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Kite:
|
||||
lying on stomach, arms extended forward.
|
||||
legs lifted off the ground, spread wide in a V-shape.
|
||||
chest and arms remain on the ground, hips and legs elevated.
|
||||
head lifted, looking forward.
|
||||
body forming a kite or diamond shape.
|
||||
Looking forward, not at camera. Aerial, lifted.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Rung:
|
||||
seated, legs extended straight forward.
|
||||
torso leaning back, supported on the hands behind.
|
||||
legs lifted off the ground, held together and straight.
|
||||
body forming a horizontal line from shoulders to heels, supported by the arms.
|
||||
head lifted, looking at the toes.
|
||||
Looking at the toes, then into camera. Suspended, horizontal.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Sling:
|
||||
lying on back, one knee drawn to the chest.
|
||||
arm of the same side reaching through the gap between the legs to grasp the shin or ankle.
|
||||
other leg extended straight along the ground.
|
||||
other arm extended to the side.
|
||||
head turned toward the bent knee.
|
||||
Looking at the knee, then into camera. Held, supported.
|
||||
Perfect anatomy, realistic
|
||||
|
||||
# The Vault:
|
||||
standing, legs apart, arms extended overhead.
|
||||
torso arched backward deeply, hips pushed forward.
|
||||
chest open to the sky, head hanging back.
|
||||
body forming a deep, continuous arch from fingertips to heels.
|
||||
Looking upward, not at camera. Arched, soaring.
|
||||
Perfect anatomy, realistic
|
||||
Reference in New Issue
Block a user