Add LTX-2.5 DFR pipeline (keyframe slots, spatial detailing, tiled temporal rounds)

docs/source/en/api/pipelines/ltx2.md

@@ -1030,6 +1030,237 @@ encode_video(
 
 You can see the supported workflows in the docs for each blockset (e.g. [`LTX2AutoBlocks`], [`LTX25AutoBlocks`]).
 
+### Diffusion Fidelity Rendering (DFR) for LTX-2.5
+
+`LTX2DFRPipeline` trades wall-clock time for detail fidelity. Each `__call__` is **one denoise pass** at `height` × `width`: it generates video plus extra single-pixel-frame **keyframe slots**, or re-denoises supplied latents seeded from those slots. Callers compose stages the same way as other LTX two-stage pipelines — this pipeline, [`LTX2LatentUpsamplePipeline`], this pipeline again, then [`LTX2DFRTemporalRefinePipeline`] for each temporal round.
+
+A slot costs a full latent frame of tokens to buy one pixel frame, which relaxes the effective temporal compression at that position — so the surrounding video is conditioned on genuinely new frames instead of interpolated ones. Slot positions come from a segment grid aligned to the VAE's temporal border (24 or 32 pixel frames, whichever pads the request less). The canvas is padded to a whole number of segments internally; `output_type="latent"` returns that padded grid so a slot on the pad is not dropped. Trim with `trim_canvas` before VAE decode.
+
+This needs a transformer whose config sets `use_keyframes_abs_pos_embedding`, which marks single-pixel-frame latents with a learned embedding. LTX-2.5 checkpoints ship it; the pipeline raises on anything older rather than spending the token budget on tokens it cannot interpret.
+
+Budget for the extra tokens: each slot adds one latent frame's worth, so stage 2 runs a longer sequence than the equivalent two-stage distilled pass — +31% at 1024x1536 / 121 frames (24576 -> 32256 tokens, 5 slots on a 24-frame segment grid). Peak activation memory scales with that, so a resolution that just fits the plain distilled recipe may need `enable_sequential_cpu_offload`, `vae.enable_tiling()`, or a smaller canvas under DFR.
+
+`return_dict=False` stays `(frames, audio)` so the diffusion-decoder path does not break. Composition uses `return_dict=True` for `keyframes` and `keyframe_positions`.
+
+The full recipe below is the one worth starting from: 1088×1920 image-to-video, one x2 temporal refine round, and the x2 spatial detailing IC-LoRA on stage 2.
+
+```py
+import torch
+from diffusers import (
+    LTX2DFRPipeline,
+    LTX2DFRTemporalRefinePipeline,
+    LTX2LatentUpsamplePipeline,
+    LTXEulerAncestralRFScheduler,
+)
+from diffusers.pipelines.ltx2 import LTX2LatentUpsamplerModel
+from diffusers.pipelines.ltx2.dfr_core import trim_canvas
+from diffusers.pipelines.ltx2.pipeline_ltx2_condition import LTX2VideoCondition
+from diffusers.pipelines.ltx2.utils import STAGE_2_DISTILLED_SIGMA_VALUES
+from diffusers.utils import encode_video, load_image
+
+pipe = LTX2DFRPipeline.from_pretrained("Lightricks/LTX-2.5-Diffusers", torch_dtype=torch.bfloat16)
+latent_upsampler = LTX2LatentUpsamplerModel.from_pretrained(
+    "Lightricks/LTX-2.5-Diffusers", subfolder="latent_upsampler", torch_dtype=torch.bfloat16
+)
+# The x2 temporal upsampler is not in the published `model_index.json` — convert it with
+# `--temporal_latent_upsampler`.
+temporal_latent_upsampler = LTX2LatentUpsamplerModel.from_pretrained(
+    "path/to/converted/temporal_latent_upsampler", torch_dtype=torch.bfloat16
+)
+upsample_pipe = LTX2LatentUpsamplePipeline(vae=pipe.vae, latent_upsampler=latent_upsampler)
+temporal_pipe = LTX2DFRTemporalRefinePipeline(
+    scheduler=LTXEulerAncestralRFScheduler(eta=0.5),
+    vae=pipe.vae,
+    audio_vae=pipe.audio_vae,
+    text_encoder=pipe.text_encoder,
+    tokenizer=pipe.tokenizer,
+    connectors=pipe.connectors,
+    transformer=pipe.transformer,
+    vocoder=pipe.vocoder,
+    temporal_latent_upsampler=temporal_latent_upsampler,
+)
+
+# All three pipelines share the same components, so place them together. Do not call
+# `enable_model_cpu_offload()` on one of them: its hooks would own modules the other two also call,
+# and `temporal_latent_upsampler` — held only by `temporal_pipe` — would never reach the device.
+pipe.to("cuda")
+upsample_pipe.to("cuda")
+temporal_pipe.to("cuda")
+
+image = load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cat.png")
+prompt = "A tabby cat stretching in a sunlit window, dust motes drifting in the light"
+conditions = [LTX2VideoCondition(frames=image, index=0, strength=1.0)]
+height, width = 1088, 1920
+frame_rate = 24.0
+generator = torch.Generator(device="cuda").manual_seed(0)
+
+num_frames = 121
+out = pipe(
+    prompt=prompt,
+    conditions=conditions,
+    height=height // 2,
+    width=width // 2,
+    num_frames=num_frames,
+    frame_rate=frame_rate,
+    generator=generator,
+    output_type="latent",
+)
+up_video = upsample_pipe(latents=out.frames, output_type="latent", return_dict=False)[0]
+up_keyframes = upsample_pipe(latents=out.keyframes, output_type="latent", return_dict=False)[0]
+
+# Load after stage 1 so the adapter is never disabled. `set_adapters` does not re-enable a
+# transformer that already had `disable_adapters()` called on it.
+pipe.load_lora_weights("Lightricks/LTX-2.5-22b-IC-LoRA-Pixel-Spatial-Upscaler", adapter_name="detailing")
+pipe.set_adapters(["detailing"], adapter_weights=[0.5])
+out2 = pipe(
+    prompt=prompt,
+    conditions=conditions,
+    latents=up_video,
+    audio_latents=out.audio,
+    keyframes_latents=up_keyframes,
+    keyframe_positions=out.keyframe_positions,
+    reference_latents=out.frames,
+    height=height,
+    width=width,
+    num_frames=num_frames,
+    frame_rate=frame_rate,
+    noise_scale=STAGE_2_DISTILLED_SIGMA_VALUES[0],
+    sigmas=STAGE_2_DISTILLED_SIGMA_VALUES,
+    generator=generator,
+    output_type="latent",
+)
+pipe.transformer.disable_adapters()
+
+# `num_frames` here is the *padded* canvas the latents actually cover, which `resolve_canvas` may have
+# grown past the 121 that were asked for. `condition_num_frames` stays the original request so a
+# negative `condition.index` does not wrap onto the pad.
+ratio = pipe.vae.temporal_compression_ratio
+canvas_frames = (out2.frames.shape[2] - 1) * ratio + 1
+out3 = temporal_pipe(
+    latents=out2.frames,
+    keyframes_latents=out2.keyframes,
+    keyframe_positions=out2.keyframe_positions,
+    audio_latents=out.audio,
+    prompt=prompt,
+    conditions=conditions,
+    height=height,
+    width=width,
+    num_frames=canvas_frames,
+    frame_rate=frame_rate,
+    source_seconds=canvas_frames / frame_rate,
+    condition_num_frames=num_frames,
+    generator=generator,
+    output_type="latent",
+)
+
+# Keep the padded canvas until decode so a slot on the pad is not dropped. `trim_canvas` counts *pixel*
+# frames, and the round mapped `N -> 2 (N - 1) + 1`.
+playback_fps = frame_rate * 2
+requested_frames = (num_frames - 1) * 2 + 1
+video_latents = trim_canvas(out3.frames, requested_frames, ratio)
+timestep = None
+if pipe.vae.config.timestep_conditioning:
+    timestep = torch.zeros(video_latents.shape[0], device=video_latents.device, dtype=pipe.vae.dtype)
+video = pipe.vae.decode(video_latents.to(pipe.vae.dtype), timestep, return_dict=False)[0]
+video = pipe.video_processor.postprocess_video(video, output_type="np")
+
+# Audio is stage 1's. Cut it to the video's duration so a muxed container does not outlast the picture.
+audio = pipe.vocoder(pipe.audio_vae.decode(out.audio.to(pipe.audio_vae.dtype), return_dict=False)[0])
+audio_samples = round(requested_frames / playback_fps * pipe.vocoder.config.output_sampling_rate)
+audio = audio[..., : min(audio.shape[-1], audio_samples)]
+
+encode_video(
+    video[0],
+    fps=playback_fps,
+    audio=audio[0].float().cpu(),
+    audio_sample_rate=pipe.vocoder.config.output_sampling_rate,
+    output_path="ltx2_5_dfr.mp4",
+)
+```
+
+`height` and `width` are **this pass**, not the final output. Stage 1 runs at half the 1080p canvas (544×960); stage 2 at 1088×1920. Each must be divisible by the VAE's spatial compression ratio (32 on LTX-2.5 for a single pass; 64 when stage 1 is half of 1080p). This is why 1080p is **1920×1088** and 4K is **3840×2176**. Each pass runs a fixed distilled schedule (`sigmas`), so there is no `num_inference_steps`; the distilled schedules are trained without guidance, so there is no `negative_prompt` or `guidance_scale` either. The shipped audio is stage 1's — later passes still run an audio stream so the video branch has cross-modal attention; the waveform itself is not refined after stage 1.
+
+**Spatial detailing.** Load the 2x spatial detailing IC-LoRA under a named adapter **after stage 1** and activate it for stage 2 only (`set_adapters(["detailing"], adapter_weights=[0.5])`). Stage 2 then attends to the stage-1 half-resolution latent as `reference_latents`. Stage 1 and the temporal rounds run with the adapter off. If you load the LoRA before stage 1, `transformer.disable_adapters()` turns it off, and `set_adapters` does **not** turn it back on — call `transformer.enable_adapters()` before stage 2, or load the weights after stage 1 as in the recipe. `reference_downscale_factor` (default `2`) scales the reference tokens' spatial coordinates into the target's coordinate space.
+
+**Temporal refinement.** [`LTX2DFRTemporalRefinePipeline`] is one round: temporally upsample, tile on keyframe seams, ancestral-denoise with [`LTXEulerAncestralRFScheduler`] (`eta=0.5`), stitch by dropping the later tile's lead-in, and merge the carry-keyframe bag. Construct that scheduler yourself — the round is refused with anything else, since a deterministic step would run to completion and only return a softer canvas. Call the pipeline once per round; loop for 2x / 4x, passing `round_index`. After a round, `keyframe_positions` cannot be re-derived from the original `num_frames` and must be passed through. Each tile is handed the slice of the frozen stage-1 audio covering its own playback window. `source_seconds` is the *stage-1* duration and stays fixed across rounds, so later rounds must pass it explicitly rather than take the default.
+
+Conditioning fps is 60 whenever playback is above 30, independently of muxing: RoPE time is `pixel_frame / fps`, so a 120 fps time base would halve every token's temporal span versus the trained distribution, and 48 fps would stretch it. Both lie that they are 60 and treat the decoded frames at the playback rate.
+
+**A third spatial stage.** Compose it; there is no fourth pipeline. Spatially upsample the **video only**, rebuild carry keyframes in RGB (`decode` → Lanczos ×2 → `encode` via [`~LTX2DFRPipeline.rebuild_epilogue_keyframes`]; never latent-upsample epilogue keyframes), then:
+
+```py
+from diffusers.pipelines.ltx2.dfr_layout import epilogue_tiles, pixel_to_latent_index
+
+# One more doubling on top of the recipe above, so every stage below the output halves again:
+# `epilogue_height` must be divisible by 128 (`4 * 32`), which is why 4K is 3840x2176.
+epilogue_height, epilogue_width = height * 2, width * 2
+refined_frames = (out3.frames.shape[2] - 1) * ratio + 1
+
+up_video = upsample_pipe(latents=out3.frames, output_type="latent", return_dict=False)[0]
+epilogue_keyframes = pipe.rebuild_epilogue_keyframes(
+    out3.keyframes,
+    decode_timestep=0.0,
+    decode_noise_scale=0.0,
+    seed=0,
+    device=up_video.device,
+    dtype=torch.float32,
+)
+
+# Temporal cuts land on the seams the *last* round stitched on -- the positions handed into it, doubled --
+# not on every carry keyframe, since the slots that round invented sit mid-window.
+tiles = epilogue_tiles(
+    latent_shape=(
+        (refined_frames - 1) // ratio + 1,
+        epilogue_height // pipe.vae.spatial_compression_ratio,
+        epilogue_width // pipe.vae.spatial_compression_ratio,
+    ),
+    frame_tiles=2,  # 2 ** number of temporal rounds
+    frame_seams=[pixel_to_latent_index(2 * p, ratio) for p in out2.keyframe_positions],
+)
+
+pipe.transformer.enable_adapters()  # the epilogue is a detailing pass too
+out4 = pipe(
+    prompt=prompt,
+    conditions=conditions,
+    latents=up_video,
+    audio_latents=out.audio,
+    generate_slots=False,
+    guidance_keyframe_latents=epilogue_keyframes,
+    guidance_keyframe_positions=out3.keyframe_positions,
+    reference_latents=out3.frames,
+    height=epilogue_height,
+    width=epilogue_width,
+    num_frames=refined_frames,
+    frame_rate=playback_fps,
+    noise_scale=STAGE_2_DISTILLED_SIGMA_VALUES[0],
+    sigmas=STAGE_2_DISTILLED_SIGMA_VALUES,
+    freeze_audio=True,
+    video_tiles=tiles,
+    generator=generator,
+    output_type="latent",
+)
+pipe.transformer.disable_adapters()
+```
+
+The two axes are seamed differently. Neither side of a spatial border holds a known answer, so those overlaps are blended with trapezoidal weights. Temporal tiles are cut on the last refine round's keyframe seams.
+
+**Decoding with the diffusion decoder.** For maximum detail fidelity, stay on `output_type="latent"` and hand the (already denormalized, possibly `trim_canvas`'d) latents to [`LTX2VideoDiffusionDecodePipeline`].
+
+```py
+from diffusers import LTX2VideoDiffusionDecodePipeline
+from diffusers.models.autoencoders.ltx2_diffusion_decoder import LTX2VideoDiffusionDecoderModel
+
+decoder = LTX2VideoDiffusionDecoderModel.from_pretrained(
+    "Lightricks/LTX-2.5-Diffusers", subfolder="diffusion_decoder", dtype=torch.bfloat16
+)
+decode_pipe = LTX2VideoDiffusionDecodePipeline(
+    diffusion_decoder=decoder, scheduler=pipe.scheduler, vae=pipe.vae
+)
+decode_pipe.enable_model_cpu_offload()
+# `denormalize=False`: the `output_type="latent"` path already applied the latent statistics.
+video = decode_pipe(latents=out3.frames, denormalize=False, output_type="np", return_dict=False)[0]
+```
+
 ## LTX2Pipeline
 
 [[autodoc]] LTX2Pipeline
@@ -1048,6 +1279,22 @@ You can see the supported workflows in the docs for each blockset (e.g. [`LTX2Au
   - all
   - __call__
 
+## LTX2DFRPipeline
+
+[[autodoc]] LTX2DFRPipeline
+  - all
+  - __call__
+
+## LTX2DFRTemporalRefinePipeline
+
+[[autodoc]] LTX2DFRTemporalRefinePipeline
+  - all
+  - __call__
+
+## LTX2DFRPipelineOutput
+
+[[autodoc]] pipelines.ltx2.pipeline_output.LTX2DFRPipelineOutput
+
 ## LTX2LatentUpsamplePipeline
 
 [[autodoc]] LTX2LatentUpsamplePipeline

scripts/convert_ltx2_to_diffusers.py

@@ -443,7 +443,7 @@ def get_ltx2_transformer_config(version: str) -> tuple[dict[str, Any], dict[str,
                 "use_prompt_embeddings": False,
                 "perturbed_attn": True,
                 # The only transformer-level deltas from 2.3: the video FFN drops its bias (audio_ff_bias and
-                # use_prompt_adaln_single keep their True defaults for this checkpoint), and 2.5.1+ carries a
+                # use_prompt_adaln_single keep their True defaults for this checkpoint), and 2.5 carries a
                 # learned keyframe absolute-position embedding.
                 "ff_bias": False,
                 "use_keyframes_abs_pos_embedding": True,
@@ -1222,7 +1222,7 @@ def get_ltx2_spatial_latent_upsampler_config(version: str):
             "rational_spatial_scale": 2.0,
             "use_rational_resampler": True,
         }
-    elif version == "2.3":
+    elif version in ("2.3", "2.5"):
         config = {
             "in_channels": 128,
             "mid_channels": 1024,
@@ -1238,9 +1238,21 @@ def get_ltx2_spatial_latent_upsampler_config(version: str):
     return config
 
 
-def convert_ltx2_spatial_latent_upsampler(
-    original_state_dict: dict[str, Any], config: dict[str, Any], dtype: torch.dtype
-):
+def get_ltx2_temporal_latent_upsampler_config(version: str):
+    if version != "2.5":
+        raise ValueError(f"Unsupported version: {version}")
+    # The temporal x2 upsampler is narrower than its spatial sibling and pixel-shuffles along time only.
+    return {
+        "in_channels": 128,
+        "mid_channels": 512,
+        "num_blocks_per_stage": 4,
+        "dims": 3,
+        "spatial_upsample": False,
+        "temporal_upsample": True,
+    }
+
+
+def convert_ltx2_latent_upsampler(original_state_dict: dict[str, Any], config: dict[str, Any], dtype: torch.dtype):
     with init_empty_weights():
         latent_upsampler = LTX2LatentUpsamplerModel(**config)
 
@@ -1379,6 +1391,12 @@ def none_or_str(value: str):
             "google/gemma-4-E2B-it or google/gemma-4-E4B-it."
         ),
     )
+    parser.add_argument(
+        "--temporal_latent_upsampler_filename",
+        default="ltx-2.5-latent-temporal-upscaler-x2-bf16-1.0.safetensors",
+        type=none_or_str,
+        help="Temporal x2 latent upsampler filename (LTX-2.5, used by the DFR pipeline's temporal refine rounds)",
+    )
     parser.add_argument(
         "--latent_upsampler_filename",
         default="ltx-2-spatial-upscaler-x2-1.0.safetensors",
@@ -1410,6 +1428,11 @@ def none_or_str(value: str):
     parser.add_argument("--vocoder", action="store_true", help="Whether to convert the vocoder model")
     parser.add_argument("--text_encoder", action="store_true", help="Whether to conver the text encoder")
     parser.add_argument("--latent_upsampler", action="store_true", help="Whether to convert the latent upsampler")
+    parser.add_argument(
+        "--temporal_latent_upsampler",
+        action="store_true",
+        help="Whether to convert the temporal x2 latent upsampler (LTX-2.5)",
+    )
     parser.add_argument(
         "--full_pipeline",
         action="store_true",
@@ -1608,14 +1631,25 @@ def main(args):
             repo_id=args.original_state_dict_repo_id, filename=args.latent_upsampler_filename
         )
         latent_upsampler_config = get_ltx2_spatial_latent_upsampler_config(args.version)
-        latent_upsampler = convert_ltx2_spatial_latent_upsampler(
+        latent_upsampler = convert_ltx2_latent_upsampler(
             original_latent_upsampler_ckpt,
             latent_upsampler_config,
             dtype=vae_dtype,
         )
         if not args.full_pipeline and not args.upsample_pipeline:
             latent_upsampler.save_pretrained(os.path.join(args.output_path, "latent_upsampler"))
 
+    if args.temporal_latent_upsampler:
+        original_temporal_upsampler_ckpt = load_hub_or_local_checkpoint(
+            repo_id=args.original_state_dict_repo_id, filename=args.temporal_latent_upsampler_filename
+        )
+        temporal_latent_upsampler = convert_ltx2_latent_upsampler(
+            original_temporal_upsampler_ckpt,
+            get_ltx2_temporal_latent_upsampler_config(args.version),
+            dtype=vae_dtype,
+        )
+        temporal_latent_upsampler.save_pretrained(os.path.join(args.output_path, "temporal_latent_upsampler"))
+
     if args.full_pipeline:
         is_distilled_ckpt = "distilled" in args.combined_filename
         if is_distilled_ckpt:

src/diffusers/__init__.py

@@ -725,6 +725,9 @@
             "LongCatImageEditPipeline",
             "LongCatImagePipeline",
             "LTX2ConditionPipeline",
+            "LTX2DFRPipeline",
+            "LTX2DFRPipelineOutput",
+            "LTX2DFRTemporalRefinePipeline",
             "LTX2HDRPipeline",
             "LTX2ImageToVideoPipeline",
             "LTX2InContextPipeline",
@@ -1572,6 +1575,9 @@
             LongCatImageEditPipeline,
             LongCatImagePipeline,
             LTX2ConditionPipeline,
+            LTX2DFRPipeline,
+            LTX2DFRPipelineOutput,
+            LTX2DFRTemporalRefinePipeline,
             LTX2HDRPipeline,
             LTX2ImageToVideoPipeline,
             LTX2InContextPipeline,