diff --git a/docs/source/en/api/pipelines/ltx2.md b/docs/source/en/api/pipelines/ltx2.md index e69b158bdf8a..2fa85c9e4af9 100644 --- a/docs/source/en/api/pipelines/ltx2.md +++ b/docs/source/en/api/pipelines/ltx2.md @@ -915,6 +915,73 @@ print(f"predicted {seconds:.2f}s -> {num_frames} frames") Converting a 2.5 checkpoint picks the head up automatically with `--full_pipeline`, or on its own with `--duration_head`. Checkpoints predating 2.5 have no such weights, and conversion skips the component rather than failing. +### Motion speed for LTX-2.5 + +LTX-2 places every video token on a time axis in seconds, derived by default from `frame_rate`, the playback rate. `motion_speed` (default `1.0`) separates the two: the time axis is conditioned on `frame_rate / motion_speed`, while the audio length, the [duration head](#automatic-duration-for-ltx-25) and the returned video still follow `frame_rate`. With `frame_rate=24.0` and `motion_speed=0.2`, the model is told the action was captured at 120 fps and the result plays back at 24 fps, which gives 5x slow motion. + +The base model is trained around 24/25/30/60 fps, so speeds far from `1.0` only make sense with an adapter trained for them, such as [`Lightricks/LTX-2.5-22b-LoRA-Slow-Motion-Control`](https://huggingface.co/Lightricks/LTX-2.5-22b-LoRA-Slow-Motion-Control) (gated: accept its license on the Hub first). Without `motion_speed`, that LoRA loads as a plain LoRA and gives none of the motion control. The model card suggests speeds of `1.0`, `0.5`, `0.2`, `0.1`, `0.05` and `0.025`, at LoRA strength `1.0`. Leave speed words ("slow motion", "high fps") out of the prompt, and reuse the same caption across speeds, since `motion_speed` alone drives the effect. + +```py +import torch +from diffusers import LTX2ImageToVideoPipeline +from diffusers.pipelines.ltx2.utils import DISTILLED_SIGMA_VALUES +from diffusers.utils import encode_video, load_image + +device = "cuda" # or "mps", "xpu", "cpu" +width = 1920 # the LoRA's recommended resolution; the effect is much weaker at 768x512 +height = 1088 +frame_rate = 24.0 # playback rate, always 24 for this LoRA +model_path = "Lightricks/LTX-2.5-Diffusers" + +pipe = LTX2ImageToVideoPipeline.from_pretrained(model_path, dtype=torch.bfloat16) +pipe.load_lora_weights( + "Lightricks/LTX-2.5-22b-LoRA-Slow-Motion-Control", + weight_name="ltx-2.5-22b-lora-slow-motion-control-1.0.safetensors", + adapter_name="slow_motion", +) +pipe.set_adapters("slow_motion", 1.0) +pipe.enable_sequential_cpu_offload(device=device) +pipe.vae.enable_tiling() + +image = load_image( + "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/astronaut.jpg" +) +# No speed wording: the same caption is reused at every speed. +prompt = ( + "An astronaut hatches from a fragile egg on the surface of the Moon, the shell cracking apart and lunar dust " + "lifting into the air, while the camera pushes in." +) + +for motion_speed in (1.0, 0.2): + video, audio = pipe( + image=image, + prompt=prompt, + width=width, + height=height, + num_frames=121, + frame_rate=frame_rate, + motion_speed=motion_speed, # 1.0 = real time, 0.2 = 5x slow motion + sigmas=DISTILLED_SIGMA_VALUES, + guidance_scale=1.0, + audio_guidance_scale=1.0, + generator=torch.Generator(device).manual_seed(42), + output_type="np", + return_dict=False, + ) + + encode_video( + video[0], + fps=frame_rate, # the file plays at the playback rate, not frame_rate / motion_speed + audio=audio[0].float().cpu(), + audio_sample_rate=pipe.vocoder.config.output_sampling_rate, + output_path=f"ltx2_5_i2v_motion_speed_{motion_speed}.mp4", + ) +``` + +Both clips last 121 frames (about 5 seconds at 24 fps) and only the pace of the motion changes. In [two-stage generation](#two-stage-generation-for-ltx-25), pass the same `motion_speed` to **both** stages. Stage 2 re-denoises the video, and it would undo the slow motion if it ran on the real-time axis. The [reference driver](https://huggingface.co/Lightricks/LTX-2.5-22b-LoRA-Slow-Motion-Control/blob/main/distilled_speed_demo.py) does the same by wrapping the stage object both passes share. + +`motion_speed` is also an input of the LTX-2 / LTX-2.5 modular blocks, and it is passed to the modular pipeline the same way. + ### LTX-2.5 Modular LTX-2.5 is also available as a modular pipeline. The default blockset uses the diffusion decoder and predicts the video duration when `num_frames` is omitted. It applies guidance separately to video and audio through the `guider` and `audio_guider` components. See [`LTX2Guidance`] for the available guidance parameters. By default, the modular pipeline will download the prompt enhancer and processor from the [google/gemma-4-E2B-it](https://huggingface.co/google/gemma-4-E2B-it) repo. Below is a T2V modular example: diff --git a/src/diffusers/modular_pipelines/ltx2/before_denoise.py b/src/diffusers/modular_pipelines/ltx2/before_denoise.py index 1878c01305fb..f3d8aaa023f8 100644 --- a/src/diffusers/modular_pipelines/ltx2/before_denoise.py +++ b/src/diffusers/modular_pipelines/ltx2/before_denoise.py @@ -25,6 +25,7 @@ # the other shared LTX-2 data/utilities enumerated in `encoders.py`. Imported from the pipelines path here only so # the draft is runnable. from ...pipelines.ltx2.pipeline_ltx2_ic_lora import LTX2ReferenceCondition +from ...pipelines.ltx2.utils import conditioning_frame_rate from ...schedulers import FlowMatchEulerDiscreteScheduler from ...utils import logging from ...utils.torch_utils import randn_tensor @@ -752,6 +753,16 @@ def inputs(self) -> list[InputParam]: InputParam( "frame_rate", type_hint=float, default=24.0, description="Frames per second of the generated video." ), + InputParam( + "motion_speed", + type_hint=float, + default=1.0, + description=( + "Speed of the motion relative to real time. The positional embeddings are conditioned on " + "`frame_rate / motion_speed`, so values below 1 slow the motion down, e.g. `0.2` for the " + "Slow-Motion-Control LoRA's 5x slow motion. `frame_rate` stays the playback rate." + ), + ), InputParam("audio_num_frames", type_hint=int, required=True), InputParam.template("num_images_per_prompt", name="num_videos_per_prompt"), InputParam( @@ -790,7 +801,12 @@ def __call__(self, components, state: PipelineState) -> tuple[LTX2ModularPipelin latent_num_frames = (block_state.num_frames - 1) // components.vae_temporal_compression_ratio + 1 block_state.video_coords = components.transformer.rope.prepare_video_coords( - batch_size, latent_num_frames, latent_height, latent_width, device, fps=block_state.frame_rate + batch_size, + latent_num_frames, + latent_height, + latent_width, + device, + fps=conditioning_frame_rate(block_state.frame_rate, block_state.motion_speed), ) block_state.audio_coords = components.transformer.audio_rope.prepare_audio_coords( batch_size, block_state.audio_num_frames, device @@ -863,6 +879,16 @@ def inputs(self) -> list[InputParam]: InputParam( "frame_rate", type_hint=float, default=24.0, description="Frames per second of the generated video." ), + InputParam( + "motion_speed", + type_hint=float, + default=1.0, + description=( + "Speed of the motion relative to real time. The positional embeddings are conditioned on " + "`frame_rate / motion_speed`, so values below 1 slow the motion down, e.g. `0.2` for the " + "Slow-Motion-Control LoRA's 5x slow motion. `frame_rate` stays the playback rate." + ), + ), InputParam( "noise_scale", type_hint=float, @@ -1022,7 +1048,7 @@ def __call__(self, components, state: PipelineState) -> tuple[LTX2ModularPipelin keyframe_latent_width=kf_latent_width, pixel_frame_idx=(latent_idx - 1) * frame_scale_factor + 1, num_pixel_frames=num_pixel_frames, - fps=block_state.frame_rate, + fps=conditioning_frame_rate(block_state.frame_rate, block_state.motion_speed), patch_size=spatial_patch, patch_size_t=temporal_patch, scale_factors=scale_factors, @@ -1159,6 +1185,16 @@ def inputs(self) -> list[InputParam]: InputParam( "frame_rate", type_hint=float, default=24.0, description="Frames per second of the generated video." ), + InputParam( + "motion_speed", + type_hint=float, + default=1.0, + description=( + "Speed of the motion relative to real time. The positional embeddings are conditioned on " + "`frame_rate / motion_speed`, so values below 1 slow the motion down, e.g. `0.2` for the " + "Slow-Motion-Control LoRA's 5x slow motion. `frame_rate` stays the playback rate." + ), + ), InputParam( "noise_scale", type_hint=float, @@ -1317,7 +1353,7 @@ def __call__(self, components, state: PipelineState) -> tuple[LTX2ModularPipelin keyframe_latent_width=kf_latent_width, pixel_frame_idx=(latent_idx - 1) * frame_scale_factor + 1, num_pixel_frames=num_pixel_frames, - fps=block_state.frame_rate, + fps=conditioning_frame_rate(block_state.frame_rate, block_state.motion_speed), patch_size=spatial_patch, patch_size_t=temporal_patch, scale_factors=scale_factors, @@ -1706,6 +1742,16 @@ def inputs(self) -> list[InputParam]: InputParam( "frame_rate", type_hint=float, default=24.0, description="Frames per second of the generated video." ), + InputParam( + "motion_speed", + type_hint=float, + default=1.0, + description=( + "Speed of the motion relative to real time. The positional embeddings are conditioned on " + "`frame_rate / motion_speed`, so values below 1 slow the motion down, e.g. `0.2` for the " + "Slow-Motion-Control LoRA's 5x slow motion. `frame_rate` stays the playback rate." + ), + ), InputParam("audio_num_frames", type_hint=int, required=True), InputParam( "appended_coords", @@ -1750,7 +1796,12 @@ def __call__(self, components, state: PipelineState) -> tuple[LTX2ModularPipelin latent_num_frames = (block_state.num_frames - 1) // components.vae_temporal_compression_ratio + 1 video_coords = components.transformer.rope.prepare_video_coords( - batch_size, latent_num_frames, latent_height, latent_width, device, fps=block_state.frame_rate + batch_size, + latent_num_frames, + latent_height, + latent_width, + device, + fps=conditioning_frame_rate(block_state.frame_rate, block_state.motion_speed), ) block_state.video_coords = torch.cat([video_coords, block_state.appended_coords.to(video_coords.dtype)], dim=2) block_state.audio_coords = components.transformer.audio_rope.prepare_audio_coords( diff --git a/src/diffusers/modular_pipelines/ltx2/denoise.py b/src/diffusers/modular_pipelines/ltx2/denoise.py index 2fb4951377a1..f83b86a656f6 100644 --- a/src/diffusers/modular_pipelines/ltx2/denoise.py +++ b/src/diffusers/modular_pipelines/ltx2/denoise.py @@ -21,6 +21,7 @@ from ...configuration_utils import FrozenDict from ...guiders import LTX2Guidance from ...models import LTX2VideoTransformer3DModel +from ...pipelines.ltx2.utils import conditioning_frame_rate from ...schedulers import FlowMatchEulerDiscreteScheduler from ..modular_pipeline import ( BlockState, @@ -319,6 +320,16 @@ def inputs(self) -> list[InputParam]: InputParam( "frame_rate", type_hint=float, default=24.0, description="Frames per second of the generated video." ), + InputParam( + "motion_speed", + type_hint=float, + default=1.0, + description=( + "Speed of the motion relative to real time. The positional embeddings are conditioned on " + "`frame_rate / motion_speed`, so values below 1 slow the motion down, e.g. `0.2` for the " + "Slow-Motion-Control LoRA's 5x slow motion. `frame_rate` stays the playback rate." + ), + ), InputParam( "use_cross_timestep", type_hint=bool, @@ -361,7 +372,7 @@ def __call__( num_frames=latent_num_frames, height=latent_height, width=latent_width, - fps=block_state.frame_rate, + fps=conditioning_frame_rate(block_state.frame_rate, block_state.motion_speed), use_cross_timestep=block_state.use_cross_timestep, attention_kwargs=block_state.attention_kwargs, perturbation_mask=None, diff --git a/src/diffusers/modular_pipelines/ltx2/encoders.py b/src/diffusers/modular_pipelines/ltx2/encoders.py index ae3d29b5cfe3..433879d46cf4 100644 --- a/src/diffusers/modular_pipelines/ltx2/encoders.py +++ b/src/diffusers/modular_pipelines/ltx2/encoders.py @@ -44,6 +44,7 @@ LTX2_5_I2V_DEFAULT_SYSTEM_PROMPT, LTX2_5_T2V_DEFAULT_SYSTEM_PROMPT, apply_image_conditioning_crf, + conditioning_frame_rate, resolve_default_image_crf, ) from ...utils import logging @@ -1197,6 +1198,16 @@ def inputs(self) -> list[InputParam]: InputParam( "frame_rate", type_hint=float, default=24.0, description="Frames per second of the generated video." ), + InputParam( + "motion_speed", + type_hint=float, + default=1.0, + description=( + "Speed of the motion relative to real time. The positional embeddings are conditioned on " + "`frame_rate / motion_speed`, so values below 1 slow the motion down, e.g. `0.2` for the " + "Slow-Motion-Control LoRA's 5x slow motion. `frame_rate` stays the playback rate." + ), + ), InputParam.template("generator"), ] @@ -1285,7 +1296,7 @@ def __call__(self, components, state: PipelineState) -> tuple[LTX2ModularPipelin height=ref_latent_height, width=ref_latent_width, device=device, - fps=block_state.frame_rate, + fps=conditioning_frame_rate(block_state.frame_rate, block_state.motion_speed), ) if downscale_factor != 1: ref_coords[:, 1, :, :] = ref_coords[:, 1, :, :] * downscale_factor diff --git a/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx2.py b/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx2.py index 86428328a5a6..aa0990f3cc80 100644 --- a/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx2.py +++ b/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx2.py @@ -322,6 +322,10 @@ class LTX2CoreDenoiseStep(SequentialPipelineBlocks): Frames per second of the generated video. audio_latents (`Tensor`, *optional*): Optional pre-encoded audio latents; random noise is used when not provided. + motion_speed (`float`, *optional*, defaults to 1.0): + Speed of the motion relative to real time. The positional embeddings are conditioned on `frame_rate / + motion_speed`, so values below 1 slow the motion down, e.g. `0.2` for the Slow-Motion-Control LoRA's 5x + slow motion. `frame_rate` stays the playback rate. dtype (`dtype`): The dtype the model inputs are cast to. **denoiser_input_fields (`None`, *optional*): @@ -424,6 +428,10 @@ class LTX2Image2VideoCoreDenoiseStep(SequentialPipelineBlocks): Frames per second of the generated video. audio_latents (`Tensor`, *optional*): Optional pre-encoded audio latents; random noise is used when not provided. + motion_speed (`float`, *optional*, defaults to 1.0): + Speed of the motion relative to real time. The positional embeddings are conditioned on `frame_rate / + motion_speed`, so values below 1 slow the motion down, e.g. `0.2` for the Slow-Motion-Control LoRA's 5x + slow motion. `frame_rate` stays the playback rate. dtype (`dtype`): The dtype the model inputs are cast to. **denoiser_input_fields (`None`, *optional*): @@ -520,6 +528,10 @@ class LTX2ConditionCoreDenoiseStep(SequentialPipelineBlocks): `LTX2AutoDurationStep`). frame_rate (`float`, *optional*, defaults to 24.0): Frames per second of the generated video. + motion_speed (`float`, *optional*, defaults to 1.0): + Speed of the motion relative to real time. The positional embeddings are conditioned on `frame_rate / + motion_speed`, so values below 1 slow the motion down, e.g. `0.2` for the Slow-Motion-Control LoRA's 5x + slow motion. `frame_rate` stays the playback rate. noise_scale (`float`, *optional*): Initial noise level for the un-conditioned tokens. `None` (default) resolves to `sigmas[0]` when custom `sigmas` are supplied, else 1.0. @@ -676,6 +688,10 @@ class LTX2AutoReferenceEncoderStep(ConditionalPipelineBlocks): `LTX2AutoDurationStep`). frame_rate (`float`, *optional*, defaults to 24.0): Frames per second of the generated video. + motion_speed (`float`, *optional*, defaults to 1.0): + Speed of the motion relative to real time. The positional embeddings are conditioned on `frame_rate / + motion_speed`, so values below 1 slow the motion down, e.g. `0.2` for the Slow-Motion-Control LoRA's 5x + slow motion. `frame_rate` stays the playback rate. generator (`Generator`, *optional*): Torch generator for deterministic generation. @@ -831,6 +847,10 @@ class LTX2InContextCoreDenoiseStep(SequentialPipelineBlocks): `LTX2AutoDurationStep`). frame_rate (`float`, *optional*, defaults to 24.0): Frames per second of the generated video. + motion_speed (`float`, *optional*, defaults to 1.0): + Speed of the motion relative to real time. The positional embeddings are conditioned on `frame_rate / + motion_speed`, so values below 1 slow the motion down, e.g. `0.2` for the Slow-Motion-Control LoRA's 5x + slow motion. `frame_rate` stays the playback rate. noise_scale (`float`, *optional*): Initial noise level for the un-conditioned tokens. `None` (default) resolves to `sigmas[0]` when custom `sigmas` are supplied, else 1.0. @@ -962,6 +982,10 @@ class LTX2AutoCoreDenoiseStep(ConditionalPipelineBlocks): `LTX2AutoDurationStep`). frame_rate (`float`, *optional*, defaults to 24.0): Frames per second of the generated video. + motion_speed (`float`, *optional*, defaults to 1.0): + Speed of the motion relative to real time. The positional embeddings are conditioned on `frame_rate / + motion_speed`, so values below 1 slow the motion down, e.g. `0.2` for the Slow-Motion-Control LoRA's 5x + slow motion. `frame_rate` stays the playback rate. noise_scale (`float`, *optional*): Initial noise level for the un-conditioned tokens. `None` (default) resolves to `sigmas[0]` when custom `sigmas` are supplied, else 1.0. @@ -1297,6 +1321,10 @@ class LTX2Blocks(SequentialPipelineBlocks): which keeps the provided latents. audio_latents (`Tensor`, *optional*): Optional pre-encoded audio latents; random noise is used when not provided. + motion_speed (`float`, *optional*, defaults to 1.0): + Speed of the motion relative to real time. The positional embeddings are conditioned on `frame_rate / + motion_speed`, so values below 1 slow the motion down, e.g. `0.2` for the Slow-Motion-Control LoRA's 5x + slow motion. `frame_rate` stays the playback rate. **denoiser_input_fields (`None`, *optional*): conditional model inputs for the denoiser: e.g. prompt_embeds, negative_prompt_embeds, etc. use_cross_timestep (`bool`, *optional*, defaults to True): @@ -1412,6 +1440,10 @@ class LTX2ImageToVideoBlocks(SequentialPipelineBlocks): VAE-encoded reference-image latents used for image-to-video conditioning. audio_latents (`Tensor`, *optional*): Optional pre-encoded audio latents; random noise is used when not provided. + motion_speed (`float`, *optional*, defaults to 1.0): + Speed of the motion relative to real time. The positional embeddings are conditioned on `frame_rate / + motion_speed`, so values below 1 slow the motion down, e.g. `0.2` for the Slow-Motion-Control LoRA's 5x + slow motion. `frame_rate` stays the playback rate. **denoiser_input_fields (`None`, *optional*): conditional model inputs for the denoiser: e.g. prompt_embeds, negative_prompt_embeds, etc. use_cross_timestep (`bool`, *optional*, defaults to True): @@ -1511,6 +1543,10 @@ class LTX2ConditionBlocks(SequentialPipelineBlocks): The number of images to generate per prompt. latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. + motion_speed (`float`, *optional*, defaults to 1.0): + Speed of the motion relative to real time. The positional embeddings are conditioned on `frame_rate / + motion_speed`, so values below 1 slow the motion down, e.g. `0.2` for the Slow-Motion-Control LoRA's 5x + slow motion. `frame_rate` stays the playback rate. noise_scale (`float`, *optional*): Initial noise level for the un-conditioned tokens. `None` (default) resolves to `sigmas[0]` when custom `sigmas` are supplied, else 1.0. @@ -1633,6 +1669,10 @@ class LTX2InContextBlocks(SequentialPipelineBlocks): `conditioning_attention_strength`. frame_rate (`float`, *optional*, defaults to 24.0): Frames per second of the generated video. + motion_speed (`float`, *optional*, defaults to 1.0): + Speed of the motion relative to real time. The positional embeddings are conditioned on `frame_rate / + motion_speed`, so values below 1 slow the motion down, e.g. `0.2` for the Slow-Motion-Control LoRA's 5x + slow motion. `frame_rate` stays the playback rate. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. reference_latents (`Tensor`, *optional*): @@ -1794,6 +1834,10 @@ class LTX2AutoBlocks(SequentialPipelineBlocks): Optional pixel-space mask of shape (1, 1, F, H, W) with values in [0, 1] giving spatially varying attention strength. Downsampled to the reference's latent grid and multiplied by `conditioning_attention_strength`. + motion_speed (`float`, *optional*, defaults to 1.0): + Speed of the motion relative to real time. The positional embeddings are conditioned on `frame_rate / + motion_speed`, so values below 1 slow the motion down, e.g. `0.2` for the Slow-Motion-Control LoRA's 5x + slow motion. `frame_rate` stays the playback rate. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. condition_latents (`list`, *optional*): diff --git a/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx25.py b/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx25.py index 7c77aba94a74..3280820eab0c 100644 --- a/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx25.py +++ b/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx25.py @@ -290,6 +290,10 @@ class LTX25AutoBlocks(SequentialPipelineBlocks): Optional pixel-space mask of shape (1, 1, F, H, W) with values in [0, 1] giving spatially varying attention strength. Downsampled to the reference's latent grid and multiplied by `conditioning_attention_strength`. + motion_speed (`float`, *optional*, defaults to 1.0): + Speed of the motion relative to real time. The positional embeddings are conditioned on `frame_rate / + motion_speed`, so values below 1 slow the motion down, e.g. `0.2` for the Slow-Motion-Control LoRA's 5x + slow motion. `frame_rate` stays the playback rate. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. condition_latents (`list`, *optional*): diff --git a/src/diffusers/pipelines/ltx2/pipeline_ltx2.py b/src/diffusers/pipelines/ltx2/pipeline_ltx2.py index 4e5ced0b4ec8..baedfcd4b305 100644 --- a/src/diffusers/pipelines/ltx2/pipeline_ltx2.py +++ b/src/diffusers/pipelines/ltx2/pipeline_ltx2.py @@ -52,6 +52,9 @@ LTX2_5_T2V_DEFAULT_SYSTEM_PROMPT, resolve_default_image_crf, ) +from .utils import ( + conditioning_frame_rate as get_conditioning_frame_rate, +) from .vocoder import LTX2Vocoder, LTX2VocoderWithBWE @@ -935,6 +938,7 @@ def __call__( min_seconds: float = 1.0, max_seconds: float = 20.0, frame_rate: float = 24.0, + motion_speed: float = 1.0, num_inference_steps: int = 30, sigmas: list[float] | None = None, timesteps: list[int] = None, @@ -998,6 +1002,11 @@ def __call__( present. Ignored when `num_frames` is set explicitly. Must be strictly greater than `min_seconds`. frame_rate (`float`, *optional*, defaults to `24.0`): The frames per second (FPS) of the generated video. + motion_speed (`float`, *optional*, defaults to `1.0`): + Speed of the motion relative to real time. The positional embeddings are conditioned on `frame_rate / + motion_speed`, so values below 1 slow the motion down, e.g. `0.2` for the Slow-Motion-Control LoRA's 5x + slow motion. `frame_rate` stays the playback rate: audio duration and the predicted number of frames + follow it. num_inference_steps (`int`, *optional*, defaults to 30): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. @@ -1150,6 +1159,8 @@ def __call__( audio_modality_scale = audio_modality_scale or modality_scale audio_guidance_rescale = audio_guidance_rescale or guidance_rescale + conditioning_frame_rate = get_conditioning_frame_rate(frame_rate, motion_speed) + # 1. Check inputs. Raise error if not correct self.check_inputs( prompt=prompt, @@ -1367,7 +1378,12 @@ def __call__( # 6. Prepare micro-conditions # Pre-compute video and audio positional ids as they will be the same at each step of the denoising loop video_coords = self.transformer.rope.prepare_video_coords( - latents.shape[0], latent_num_frames, latent_height, latent_width, latents.device, fps=frame_rate + latents.shape[0], + latent_num_frames, + latent_height, + latent_width, + latents.device, + fps=conditioning_frame_rate, ) audio_coords = self.transformer.audio_rope.prepare_audio_coords( audio_latents.shape[0], audio_num_frames, audio_latents.device @@ -1408,7 +1424,7 @@ def __call__( num_frames=latent_num_frames, height=latent_height, width=latent_width, - fps=frame_rate, + fps=conditioning_frame_rate, audio_num_frames=audio_num_frames, video_coords=video_coords, audio_coords=audio_coords, @@ -1480,7 +1496,7 @@ def __call__( num_frames=latent_num_frames, height=latent_height, width=latent_width, - fps=frame_rate, + fps=conditioning_frame_rate, audio_num_frames=audio_num_frames, video_coords=video_pos_ids, audio_coords=audio_pos_ids, @@ -1520,7 +1536,7 @@ def __call__( num_frames=latent_num_frames, height=latent_height, width=latent_width, - fps=frame_rate, + fps=conditioning_frame_rate, audio_num_frames=audio_num_frames, video_coords=video_pos_ids, audio_coords=audio_pos_ids, diff --git a/src/diffusers/pipelines/ltx2/pipeline_ltx2_condition.py b/src/diffusers/pipelines/ltx2/pipeline_ltx2_condition.py index e1bff845302c..ef137ae0dc79 100644 --- a/src/diffusers/pipelines/ltx2/pipeline_ltx2_condition.py +++ b/src/diffusers/pipelines/ltx2/pipeline_ltx2_condition.py @@ -56,6 +56,9 @@ apply_image_conditioning_crf, resolve_default_image_crf, ) +from .utils import ( + conditioning_frame_rate as get_conditioning_frame_rate, +) from .vocoder import LTX2Vocoder, LTX2VocoderWithBWE @@ -1355,6 +1358,7 @@ def __call__( min_seconds: float = 1.0, max_seconds: float = 20.0, frame_rate: float = 24.0, + motion_speed: float = 1.0, num_inference_steps: int = 30, sigmas: list[float] | None = None, timesteps: list[float] | None = None, @@ -1420,6 +1424,11 @@ def __call__( present. Ignored when `num_frames` is set explicitly. Must be strictly greater than `min_seconds`. frame_rate (`float`, *optional*, defaults to `24.0`): The frames per second (FPS) of the generated video. + motion_speed (`float`, *optional*, defaults to `1.0`): + Speed of the motion relative to real time. The positional embeddings are conditioned on `frame_rate / + motion_speed`, so values below 1 slow the motion down, e.g. `0.2` for the Slow-Motion-Control LoRA's 5x + slow motion. `frame_rate` stays the playback rate: audio duration and the predicted number of frames + follow it. num_inference_steps (`int`, *optional*, defaults to 30): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. @@ -1574,6 +1583,8 @@ def __call__( audio_modality_scale = audio_modality_scale or modality_scale audio_guidance_rescale = audio_guidance_rescale or guidance_rescale + conditioning_frame_rate = get_conditioning_frame_rate(frame_rate, motion_speed) + # 1. Check inputs. Raise error if not correct self.check_inputs( prompt=prompt, @@ -1721,7 +1732,7 @@ def __call__( height=height, width=width, num_frames=num_frames, - frame_rate=frame_rate, + frame_rate=conditioning_frame_rate, noise_scale=noise_scale, dtype=torch.float32, device=device, @@ -1792,7 +1803,12 @@ def __call__( # 6. Prepare micro-conditions # Pre-compute video and audio positional ids as they will be the same at each step of the denoising loop video_coords = self.transformer.rope.prepare_video_coords( - latents.shape[0], latent_num_frames, latent_height, latent_width, latents.device, fps=frame_rate + latents.shape[0], + latent_num_frames, + latent_height, + latent_width, + latents.device, + fps=conditioning_frame_rate, ) audio_coords = self.transformer.audio_rope.prepare_audio_coords( audio_latents.shape[0], audio_num_frames, audio_latents.device @@ -1840,7 +1856,7 @@ def __call__( num_frames=latent_num_frames, height=latent_height, width=latent_width, - fps=frame_rate, + fps=conditioning_frame_rate, audio_num_frames=audio_num_frames, video_coords=video_coords, audio_coords=audio_coords, @@ -1916,7 +1932,7 @@ def __call__( num_frames=latent_num_frames, height=latent_height, width=latent_width, - fps=frame_rate, + fps=conditioning_frame_rate, audio_num_frames=audio_num_frames, video_coords=video_pos_ids, audio_coords=audio_pos_ids, @@ -1958,7 +1974,7 @@ def __call__( num_frames=latent_num_frames, height=latent_height, width=latent_width, - fps=frame_rate, + fps=conditioning_frame_rate, audio_num_frames=audio_num_frames, video_coords=video_pos_ids, audio_coords=audio_pos_ids, diff --git a/src/diffusers/pipelines/ltx2/pipeline_ltx2_ic_lora.py b/src/diffusers/pipelines/ltx2/pipeline_ltx2_ic_lora.py index 2924a086c721..3d13ea4effa2 100644 --- a/src/diffusers/pipelines/ltx2/pipeline_ltx2_ic_lora.py +++ b/src/diffusers/pipelines/ltx2/pipeline_ltx2_ic_lora.py @@ -56,6 +56,9 @@ apply_image_conditioning_crf, resolve_default_image_crf, ) +from .utils import ( + conditioning_frame_rate as get_conditioning_frame_rate, +) from .vocoder import LTX2Vocoder, LTX2VocoderWithBWE @@ -1787,6 +1790,7 @@ def __call__( width: int = 768, num_frames: int = 121, frame_rate: float = 24.0, + motion_speed: float = 1.0, num_inference_steps: int = 30, sigmas: list[float] | None = None, timesteps: list[float] | None = None, @@ -1864,6 +1868,11 @@ def __call__( The number of video frames to generate. Must satisfy `(n - 1) % 8 == 0`. frame_rate (`float`, *optional*, defaults to `24.0`): The frames per second (FPS) of the generated video. + motion_speed (`float`, *optional*, defaults to `1.0`): + Speed of the motion relative to real time. The positional embeddings are conditioned on `frame_rate / + motion_speed`, so values below 1 slow the motion down, e.g. `0.2` for the Slow-Motion-Control LoRA's 5x + slow motion. `frame_rate` stays the playback rate: audio duration and the predicted number of frames + follow it. num_inference_steps (`int`, *optional*, defaults to 30): The number of denoising steps. sigmas (`List[float]`, *optional*): @@ -1964,6 +1973,8 @@ def __call__( audio_modality_scale = audio_modality_scale or modality_scale audio_guidance_rescale = audio_guidance_rescale or guidance_rescale + conditioning_frame_rate = get_conditioning_frame_rate(frame_rate, motion_speed) + # 1. Check inputs self.check_inputs( prompt=prompt, @@ -2101,7 +2112,7 @@ def __call__( height=height, width=width, num_frames=num_frames, - frame_rate=frame_rate, + frame_rate=conditioning_frame_rate, noise_scale=noise_scale, dtype=torch.float32, device=device, @@ -2195,7 +2206,12 @@ def __call__( # 7. Prepare positional coordinates video_coords = self.transformer.rope.prepare_video_coords( - latents.shape[0], latent_num_frames, latent_height, latent_width, latents.device, fps=frame_rate + latents.shape[0], + latent_num_frames, + latent_height, + latent_width, + latents.device, + fps=conditioning_frame_rate, ) if appended_coords is not None: video_coords = torch.cat([video_coords, appended_coords], dim=2) @@ -2258,7 +2274,7 @@ def __call__( num_frames=latent_num_frames, height=latent_height, width=latent_width, - fps=frame_rate, + fps=conditioning_frame_rate, audio_num_frames=audio_num_frames, video_coords=video_coords, audio_coords=audio_coords, @@ -2347,7 +2363,7 @@ def __call__( num_frames=latent_num_frames, height=latent_height, width=latent_width, - fps=frame_rate, + fps=conditioning_frame_rate, audio_num_frames=audio_num_frames, video_coords=video_pos_ids, audio_coords=audio_pos_ids, @@ -2396,7 +2412,7 @@ def __call__( num_frames=latent_num_frames, height=latent_height, width=latent_width, - fps=frame_rate, + fps=conditioning_frame_rate, audio_num_frames=audio_num_frames, video_coords=video_pos_ids, audio_coords=audio_pos_ids, diff --git a/src/diffusers/pipelines/ltx2/pipeline_ltx2_image2video.py b/src/diffusers/pipelines/ltx2/pipeline_ltx2_image2video.py index 36b7effdfd6c..7f88a56f16cf 100644 --- a/src/diffusers/pipelines/ltx2/pipeline_ltx2_image2video.py +++ b/src/diffusers/pipelines/ltx2/pipeline_ltx2_image2video.py @@ -53,6 +53,9 @@ apply_image_conditioning_crf, resolve_default_image_crf, ) +from .utils import ( + conditioning_frame_rate as get_conditioning_frame_rate, +) from .vocoder import LTX2Vocoder, LTX2VocoderWithBWE @@ -990,6 +993,7 @@ def __call__( min_seconds: float = 1.0, max_seconds: float = 20.0, frame_rate: float = 24.0, + motion_speed: float = 1.0, num_inference_steps: int = 30, sigmas: list[float] | None = None, timesteps: list[int] | None = None, @@ -1056,6 +1060,11 @@ def __call__( present. Ignored when `num_frames` is set explicitly. Must be strictly greater than `min_seconds`. frame_rate (`float`, *optional*, defaults to `24.0`): The frames per second (FPS) of the generated video. + motion_speed (`float`, *optional*, defaults to `1.0`): + Speed of the motion relative to real time. The positional embeddings are conditioned on `frame_rate / + motion_speed`, so values below 1 slow the motion down, e.g. `0.2` for the Slow-Motion-Control LoRA's 5x + slow motion. `frame_rate` stays the playback rate: audio duration and the predicted number of frames + follow it. num_inference_steps (`int`, *optional*, defaults to 30): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. @@ -1212,6 +1221,8 @@ def __call__( audio_modality_scale = audio_modality_scale or modality_scale audio_guidance_rescale = audio_guidance_rescale or guidance_rescale + conditioning_frame_rate = get_conditioning_frame_rate(frame_rate, motion_speed) + # 1. Check inputs. Raise error if not correct self.check_inputs( prompt=prompt, @@ -1442,7 +1453,12 @@ def __call__( # 6. Prepare micro-conditions # Pre-compute video and audio positional ids as they will be the same at each step of the denoising loop video_coords = self.transformer.rope.prepare_video_coords( - latents.shape[0], latent_num_frames, latent_height, latent_width, latents.device, fps=frame_rate + latents.shape[0], + latent_num_frames, + latent_height, + latent_width, + latents.device, + fps=conditioning_frame_rate, ) audio_coords = self.transformer.audio_rope.prepare_audio_coords( audio_latents.shape[0], audio_num_frames, audio_latents.device @@ -1484,7 +1500,7 @@ def __call__( num_frames=latent_num_frames, height=latent_height, width=latent_width, - fps=frame_rate, + fps=conditioning_frame_rate, audio_num_frames=audio_num_frames, video_coords=video_coords, audio_coords=audio_coords, @@ -1558,7 +1574,7 @@ def __call__( num_frames=latent_num_frames, height=latent_height, width=latent_width, - fps=frame_rate, + fps=conditioning_frame_rate, audio_num_frames=audio_num_frames, video_coords=video_pos_ids, audio_coords=audio_pos_ids, @@ -1599,7 +1615,7 @@ def __call__( num_frames=latent_num_frames, height=latent_height, width=latent_width, - fps=frame_rate, + fps=conditioning_frame_rate, audio_num_frames=audio_num_frames, video_coords=video_pos_ids, audio_coords=audio_pos_ids, diff --git a/src/diffusers/pipelines/ltx2/utils.py b/src/diffusers/pipelines/ltx2/utils.py index e9b90b41fb5d..986c40f47654 100644 --- a/src/diffusers/pipelines/ltx2/utils.py +++ b/src/diffusers/pipelines/ltx2/utils.py @@ -303,6 +303,26 @@ class PromptEnhancementConfig: # ruff: enable[E501] +def conditioning_frame_rate(frame_rate: float, motion_speed: float) -> float: + """ + Frame rate the transformer's positional embeddings are conditioned on: the playback `frame_rate` divided by + `motion_speed`. A `motion_speed` below 1 conditions on a higher frame rate, so motion plays back slower (e.g. `0.2` + for the Slow-Motion-Control LoRA's 5x slow motion). + + Args: + frame_rate (`float`): + Playback frame rate of the generated video. + motion_speed (`float`): + Speed of the motion relative to real time. Must be positive. + + Returns: + `float`: The frame rate the positional embeddings use. + """ + if motion_speed <= 0: + raise ValueError(f"`motion_speed` must be positive, but got {motion_speed}.") + return frame_rate / motion_speed + + def resolve_default_image_crf(text_encoder: Any) -> int: """Return the image-conditioning H.264 CRF that matches the loaded text-encoder generation. diff --git a/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx25.py b/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx25.py index 69080095c627..e12bdebd6f90 100644 --- a/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx25.py +++ b/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx25.py @@ -22,6 +22,7 @@ from diffusers.pipelines.ltx2.pipeline_ltx2_condition import LTX2VideoCondition from diffusers.pipelines.ltx2.pipeline_ltx2_ic_lora import LTX2ReferenceCondition +from ...testing_utils import assert_tensors_close from ..testing_utils import ( BaseModularPipelineTesterConfig, ModularLoadingTesterMixin, @@ -182,6 +183,28 @@ def test_auto_duration_predicts_a_grid_valid_frame_count(self): assert (num_frames - 1) % pipe.vae_temporal_compression_ratio == 0 assert 0 < num_frames <= round(2.0 * inputs["frame_rate"]) + def test_motion_speed_rescales_only_the_video_time_axis(self): + pipe = self.get_pipeline().to("cpu") + outputs = ["videos", "audio", "video_coords", "audio_coords"] + + default = pipe(**self.get_dummy_inputs(), output=outputs) + same = pipe(**self.get_dummy_inputs(), motion_speed=1.0, output=outputs) + slow = pipe(**self.get_dummy_inputs(), motion_speed=0.5, output=outputs) + + assert_tensors_close(same["videos"], default["videos"], atol=1e-6, rtol=0) + # Half speed doubles the conditioning rate: the time axis of the video positions halves, nothing else moves. + assert_tensors_close(slow["video_coords"][:, 0], default["video_coords"][:, 0] / 2, atol=1e-6, rtol=1e-6) + assert torch.equal(slow["video_coords"][:, 1:], default["video_coords"][:, 1:]) + assert torch.equal(slow["audio_coords"], default["audio_coords"]) + assert slow["videos"].shape == default["videos"].shape + assert slow["audio"].shape == default["audio"].shape + assert not torch.allclose(slow["videos"], default["videos"]) + + def test_motion_speed_must_be_positive(self): + pipe = self.get_pipeline().to("cpu") + with pytest.raises(ValueError, match="motion_speed"): + pipe(**self.get_dummy_inputs(), motion_speed=0.0, output="videos") + class TestLTX25Text2VideoModularPipelineLoading(LTX25Text2VideoModularPipelineTesterConfig, ModularLoadingTesterMixin): pass diff --git a/tests/pipelines/ltx2/test_ltx2.py b/tests/pipelines/ltx2/test_ltx2.py index 0ad9da2b46a3..d727ee1dcba8 100644 --- a/tests/pipelines/ltx2/test_ltx2.py +++ b/tests/pipelines/ltx2/test_ltx2.py @@ -244,6 +244,36 @@ def test_invalid_duration_bounds_raise(self): with pytest.raises(ValueError, match="min_seconds"): pipe(**inputs) + def test_motion_speed_changes_the_video_only(self): + pipe = self.get_pipeline() + + default = pipe(**self.get_dummy_inputs()) + same = pipe(**{**self.get_dummy_inputs(), "motion_speed": 1.0}) + slow = pipe(**{**self.get_dummy_inputs(), "motion_speed": 0.5}) + + # The default speed of 1 leaves the pipeline unchanged. + assert_tensors_close(same.frames, default.frames, atol=1e-6, rtol=0) + assert_tensors_close(same.audio, default.audio, atol=1e-6, rtol=0) + # A different speed changes the video, not the output shapes or the audio length. + assert slow.frames.shape == default.frames.shape + assert slow.audio.shape == default.audio.shape + assert not torch.allclose(slow.frames, default.frames) + + def test_duration_head_follows_frame_rate_not_motion_speed(self): + pipe = self.get_pipeline_with_duration_head() + inputs = self.get_dummy_inputs() + inputs.pop("num_frames") + inputs.update(min_seconds=1.0, max_seconds=2.0) + + default = pipe(**inputs).frames + slow = pipe(**{**inputs, "motion_speed": 0.2}).frames + assert slow.shape[1] == default.shape[1] + + def test_motion_speed_must_be_positive(self): + pipe = self.get_pipeline() + with pytest.raises(ValueError, match="motion_speed"): + pipe(**{**self.get_dummy_inputs(), "motion_speed": 0.0}) + class TestLTX2PipelineMemory(LTX2PipelineTesterConfig, LTX2MemoryTesterMixin): """Memory optimization tests (CPU offload, group offload, layerwise casting) for the LTX2 pipeline."""