From 89edc3f632020d4306706213e34787a7625a0b4c Mon Sep 17 00:00:00 2001 From: Daniel Gu Date: Thu, 20 Aug 2026 23:13:48 -0700 Subject: [PATCH 1/4] Move LTX2Guidance from /modular_pipelines/ltx2/ to guiders/ to allow LTX-2 guiders to be correctly loaded from checkpoint --- docs/source/en/api/modular_diffusers/guiders.md | 4 ++++ src/diffusers/__init__.py | 2 ++ src/diffusers/guiders/__init__.py | 1 + .../ltx2/guider.py => guiders/ltx2_guidance.py} | 14 +++++++++----- src/diffusers/modular_pipelines/ltx2/denoise.py | 2 +- src/diffusers/utils/dummy_pt_objects.py | 15 +++++++++++++++ 6 files changed, 32 insertions(+), 6 deletions(-) rename src/diffusers/{modular_pipelines/ltx2/guider.py => guiders/ltx2_guidance.py} (95%) diff --git a/docs/source/en/api/modular_diffusers/guiders.md b/docs/source/en/api/modular_diffusers/guiders.md index a24eb7220749..3f8732029b3f 100644 --- a/docs/source/en/api/modular_diffusers/guiders.md +++ b/docs/source/en/api/modular_diffusers/guiders.md @@ -37,3 +37,7 @@ Guiders are components in Modular Diffusers that control how the diffusion proce ## TangentialClassifierFreeGuidance [[autodoc]] diffusers.guiders.tangential_classifier_free_guidance.TangentialClassifierFreeGuidance + +## LTX2Guidance + +[[autodoc]] diffusers.guiders.ltx2_guidance.LTX2Guidance diff --git a/src/diffusers/__init__.py b/src/diffusers/__init__.py index b7d79b8ee97d..747bec2bdf84 100644 --- a/src/diffusers/__init__.py +++ b/src/diffusers/__init__.py @@ -189,6 +189,7 @@ "ClassifierFreeGuidance", "ClassifierFreeZeroStarGuidance", "FrequencyDecoupledGuidance", + "LTX2Guidance", "PerturbedAttentionGuidance", "SkipLayerGuidance", "SmoothedEnergyGuidance", @@ -1064,6 +1065,7 @@ ClassifierFreeGuidance, ClassifierFreeZeroStarGuidance, FrequencyDecoupledGuidance, + LTX2Guidance, PerturbedAttentionGuidance, SkipLayerGuidance, SmoothedEnergyGuidance, diff --git a/src/diffusers/guiders/__init__.py b/src/diffusers/guiders/__init__.py index 88fae37f5d00..fe2a07858e71 100644 --- a/src/diffusers/guiders/__init__.py +++ b/src/diffusers/guiders/__init__.py @@ -24,6 +24,7 @@ from .classifier_free_zero_star_guidance import ClassifierFreeZeroStarGuidance from .frequency_decoupled_guidance import FrequencyDecoupledGuidance from .guider_utils import BaseGuidance + from .ltx2_guidance import LTX2Guidance from .magnitude_aware_guidance import MagnitudeAwareGuidance from .perturbed_attention_guidance import PerturbedAttentionGuidance from .skip_layer_guidance import SkipLayerGuidance diff --git a/src/diffusers/modular_pipelines/ltx2/guider.py b/src/diffusers/guiders/ltx2_guidance.py similarity index 95% rename from src/diffusers/modular_pipelines/ltx2/guider.py rename to src/diffusers/guiders/ltx2_guidance.py index 69c6d582b247..32de02a8000c 100644 --- a/src/diffusers/modular_pipelines/ltx2/guider.py +++ b/src/diffusers/guiders/ltx2_guidance.py @@ -33,12 +33,16 @@ # close-but-not-bitwise check. import math +from typing import TYPE_CHECKING import torch -from ...configuration_utils import register_to_config -from ...guiders.guider_utils import BaseGuidance, GuiderOutput, rescale_noise_cfg -from ..modular_pipeline import BlockState +from ..configuration_utils import register_to_config +from .guider_utils import BaseGuidance, GuiderOutput, rescale_noise_cfg + + +if TYPE_CHECKING: + from ..modular_pipelines.modular_pipeline import BlockState class LTX2Guidance(BaseGuidance): @@ -121,7 +125,7 @@ def is_conditional(self) -> bool: # conditioning as `pred_cond` while the denoiser overrides their per-pass model flags after preparation. _PREDICTION_INDEX = {"pred_cond": 0, "pred_uncond": 1, "pred_cond_stg": 2, "pred_cond_modality": 3} - def prepare_inputs(self, guider_inputs: dict) -> list[BlockState]: + def prepare_inputs(self, guider_inputs: dict) -> list["BlockState"]: # One identifier-tagged batch per active pass. Every value in `guider_inputs` is a 4-tuple indexed by # `_PREDICTION_INDEX` ([cond, uncond, stg, modality]); each pass reads its own slot, so the per-pass model # flags (`spatio_temporal_guidance_blocks`, `isolate_modalities`) are carried exactly like the encoder @@ -131,7 +135,7 @@ def prepare_inputs(self, guider_inputs: dict) -> list[BlockState]: for pred in self.active_predictions() ] - def prepare_inputs_from_block_state(self, data: BlockState, input_fields: dict) -> list[BlockState]: + def prepare_inputs_from_block_state(self, data: "BlockState", input_fields: dict) -> list["BlockState"]: # One identifier-tagged batch per active pass. Each value in `input_fields` maps a transformer argument to a # 4-tuple of block-state attribute names indexed by `_PREDICTION_INDEX` ([cond, uncond, stg, modality]); the # base helper reads the pass's slot off `data`. The denoiser then sets the per-pass model flags and fills diff --git a/src/diffusers/modular_pipelines/ltx2/denoise.py b/src/diffusers/modular_pipelines/ltx2/denoise.py index 5cc5a4e57abc..9742a6c7f3bc 100644 --- a/src/diffusers/modular_pipelines/ltx2/denoise.py +++ b/src/diffusers/modular_pipelines/ltx2/denoise.py @@ -19,6 +19,7 @@ import torch from ...configuration_utils import FrozenDict +from ...guiders import LTX2Guidance from ...models import LTX2VideoTransformer3DModel from ...schedulers import FlowMatchEulerDiscreteScheduler from ..modular_pipeline import ( @@ -28,7 +29,6 @@ PipelineState, ) from ..modular_pipeline_utils import ComponentSpec, InputParam -from .guider import LTX2Guidance # Velocity-space helpers, mirrored from `diffusers.pipelines.ltx2.pipeline_ltx2.LTX2Pipeline` and redefined here diff --git a/src/diffusers/utils/dummy_pt_objects.py b/src/diffusers/utils/dummy_pt_objects.py index f34008252ab6..1598814f835a 100644 --- a/src/diffusers/utils/dummy_pt_objects.py +++ b/src/diffusers/utils/dummy_pt_objects.py @@ -107,6 +107,21 @@ def from_pretrained(cls, *args, **kwargs): requires_backends(cls, ["torch"]) +class LTX2Guidance(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + class PerturbedAttentionGuidance(metaclass=DummyObject): _backends = ["torch"] From 731a6b133ee972e40793b228d6135bc31fd6ec4a Mon Sep 17 00:00:00 2001 From: Daniel Gu Date: Fri, 21 Aug 2026 01:22:17 -0700 Subject: [PATCH 2/4] Add regression test for LTX-2 guiders declared as from_pretrained components Covers the round trip this move fixes: a checkpoint that declares `guider` and `audio_guider` in `modular_model_index.json` must record a resolvable (library, class_name) pair and reload its non-default guidance scales. The existing `test_modular_index_consistency` skips components without a `pretrained_model_name_or_path`, so `from_config` guiders were uncovered. Co-Authored-By: Claude Opus 5 (1M context) --- .../ltx2/test_modular_pipeline_ltx2.py | 38 ++++++++++++++++++- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx2.py b/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx2.py index e73944f4534a..2ee330ece20a 100644 --- a/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx2.py +++ b/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx2.py @@ -13,12 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json + import numpy as np import PIL.Image import pytest import torch -from diffusers.modular_pipelines import LTX2AutoBlocks, LTX2ModularPipeline +from diffusers import LTX2Guidance, ModularPipeline +from diffusers.modular_pipelines import ComponentSpec, LTX2AutoBlocks, LTX2ModularPipeline from diffusers.pipelines.ltx2.pipeline_ltx2_condition import LTX2VideoCondition from diffusers.pipelines.ltx2.pipeline_ltx2_ic_lora import LTX2ReferenceCondition @@ -166,7 +169,38 @@ def test_auto_duration_predicts_a_grid_valid_frame_count(self): class TestLTX2Text2VideoModularPipelineLoading(LTX2Text2VideoModularPipelineTesterConfig, ModularLoadingTesterMixin): - pass + def test_guiders_round_trip_as_pretrained_components(self, tmp_path): + # `guider` and `audio_guider` default to `from_config`, so a checkpoint that ships non-default guidance + # scales has to declare them as `from_pretrained` components instead. That path records the class as a + # (library, class_name) pair in `modular_model_index.json`, which only resolves back to `LTX2Guidance` + # while the class lives in `diffusers.guiders` and is exported from the top-level namespace. + pipe = self.get_pipeline() + + staging = tmp_path / "guiders" + pipe.guider.new(guidance_scale=4.0).save_pretrained(str(staging / "guider")) + pipe.audio_guider.new(guidance_scale=8.0).save_pretrained(str(staging / "audio_guider")) + pipe.update_components( + guider=ComponentSpec( + "guider", LTX2Guidance, pretrained_model_name_or_path=str(staging), subfolder="guider" + ).load(), + audio_guider=ComponentSpec( + "audio_guider", LTX2Guidance, pretrained_model_name_or_path=str(staging), subfolder="audio_guider" + ).load(), + ) + + repo = tmp_path / "repo" + pipe.save_pretrained(str(repo), overwrite_modular_index=True) + + index = json.loads((repo / "modular_model_index.json").read_text()) + for name in ("guider", "audio_guider"): + assert index[name][2]["type_hint"] == ["diffusers", "LTX2Guidance"] + assert index[name][2]["pretrained_model_name_or_path"] == str(repo) + assert index[name][2]["subfolder"] == name + + loaded_pipe = ModularPipeline.from_pretrained(str(repo)) + loaded_pipe.load_components(names=["guider", "audio_guider"]) + assert loaded_pipe.guider.config.guidance_scale == 4.0 + assert loaded_pipe.audio_guider.config.guidance_scale == 8.0 class TestLTX2Text2VideoModularPipelineMemory(LTX2Text2VideoModularPipelineTesterConfig, ModularMemoryTesterMixin): From 6a26008813bb11bdea1ceca5fefd8b2d9fb3fc43 Mon Sep 17 00:00:00 2001 From: Daniel Gu Date: Fri, 21 Aug 2026 17:00:39 -0700 Subject: [PATCH 3/4] Migrate LTX2Guidance docs to guider docs after move --- docs/source/en/api/pipelines/ltx2.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/source/en/api/pipelines/ltx2.md b/docs/source/en/api/pipelines/ltx2.md index e73689b4af52..c323d288f3ef 100644 --- a/docs/source/en/api/pipelines/ltx2.md +++ b/docs/source/en/api/pipelines/ltx2.md @@ -1085,7 +1085,3 @@ You can see the supported workflows in the docs for each blockset (e.g. [`LTX2Au ## LTX25AutoBlocks [[autodoc]] LTX25AutoBlocks - -## LTX2Guidance - -[[autodoc]] modular_pipelines.ltx2.guider.LTX2Guidance From 2303e12083144f8d039e16a61455bd0b5b41d4bd Mon Sep 17 00:00:00 2001 From: Daniel Gu Date: Fri, 21 Aug 2026 18:25:18 -0700 Subject: [PATCH 4/4] Improve LTX2Guidance docs and add arg docstrings --- src/diffusers/guiders/ltx2_guidance.py | 59 ++++++++++--------- .../modular_pipelines/ltx2/denoise.py | 8 +++ 2 files changed, 40 insertions(+), 27 deletions(-) diff --git a/src/diffusers/guiders/ltx2_guidance.py b/src/diffusers/guiders/ltx2_guidance.py index 32de02a8000c..d856b4ea65c0 100644 --- a/src/diffusers/guiders/ltx2_guidance.py +++ b/src/diffusers/guiders/ltx2_guidance.py @@ -12,26 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Two single-modality guiders (`LTX2Guidance`, one as the video `guider` and one as the `audio_guider`), driven -# through the standard guider API (`prepare_inputs_from_block_state`). Wired into `LTX2LoopDenoiser`. -# -# The denoiser owns a `guider_input_fields` map (transformer arg -> per-pass block-state attribute names, indexed -# [cond, uncond, stg, modality]); `guider.prepare_inputs_from_block_state(block_state, guider_input_fields)` yields -# one identifier-tagged batch per active pass, carrying that pass's encoder tensors. The per-pass model flags -# (`spatio_temporal_guidance_blocks`, `isolate_modalities`) are pass-identity constants rather than block-state -# conditioning, so the denoiser sets them on each batch by identifier after preparation. The denoiser runs each -# pass as its own single-batch forward, then each guider's `forward`/`__call__` combines its modality (CFG + STG + -# modality-isolation, delta formulation in x0 space). -# -# Parity note. Because every pass (cond/uncond included) is a separate single-batch forward -- not the batched -# `torch.cat([latents] * 2)` the standard `LTX2Pipeline` uses -- this does NOT match the reference bitwise. GPU -# matmul is not batch-invariant, so `cond` computed alone differs from `cond` inside a batch-of-2: ~1e-6/op in fp32 -# (still within the harness's fp32 tolerance) but ~1e-2/op in bf16, which the CFG delta and sampler amplify to -# ~10% mean-relative latent divergence. A batched cond+uncond forward would be fp32-bitwise, but it can't drive -# the guider API per-pass, so this design trades bitwiseness for using the guider API end-to-end. Since -# fp32-within-tolerance (not bitwise) is the modular-ecosystem norm, gate parity on fp32 and treat bf16 as a -# close-but-not-bitwise check. - import math from typing import TYPE_CHECKING @@ -52,10 +32,38 @@ class LTX2Guidance(BaseGuidance): - spatio-temporal guidance (STG), an extra pass with a set of transformer blocks perturbed, - modality-isolation guidance, an extra pass with A2V/V2A cross-attention disabled. - Instantiated once per modality (`guider` for video, `audio_guider` for audio) with that modality's scales; - that keeps the per-modality scales in independent, fully-defined component configs. The combine is done in - whatever space the caller feeds it — the LTX-2 denoiser converts velocity->x0 before calling this and back - afterwards, so `forward` operates on x0 predictions. + One instance drives one modality, so a multimodal checkpoint carries several: LTX-2.X pairs a video `guider` + with an `audio_guider`, each holding its own scales. The final guidance estimate is computed in whatever space + the caller feeds it — the LTX-2 denoiser converts velocity->x0 before calling this and back afterwards, so + `forward` operates on x0 predictions. + + Unlike `SkipLayerGuidance`, which applies its perturbations with hooks, `LTX2Guidance` expects its non-CFG + perturbations to be configurable through model forward arguments (`spatio_temporal_guidance_blocks` and + `isolate_modalities` in `LTX2VideoTransformer3DModel`). The calling denoiser block is responsible for setting + those per pass identifier; see `LTX2LoopDenoiser` for a usage example. + + Args: + guidance_scale (`float`, defaults to `1.0`): + CFG scale for this modality. The CFG pass is skipped entirely at `1.0`. + stg_scale (`float`, defaults to `0.0`): + Spatio-temporal guidance scale for this modality. The STG pass is skipped entirely at `0.0`. + modality_scale (`float`, defaults to `1.0`): + Modality-isolation guidance scale for this modality. That pass is skipped entirely at `1.0`. + guidance_rescale (`float`, defaults to `0.0`): + Rescaling factor to prevent overexposure from high guidance scales. Based on [Common Diffusion Noise + Schedules and Sample Steps are Flawed](https://huggingface.co/papers/2305.08891). Range: 0.0 (no rescaling) + to 1.0 (full rescaling). + spatio_temporal_guidance_blocks (`list[int]`, *optional*): + Transformer blocks to perturb on the STG pass. How the value is consumed is up to the denoiser block: + LTX-2's perturbs whole blocks in a single forward that feeds every modality, so it reads this off the video + guider alone — setting it on the audio guider has no effect, and that guider joins the same STG forward + through its own `stg_scale`. + start (`float`, defaults to `0.0`): + Fraction of denoising steps (0.0-1.0) after which guidance starts. + stop (`float`, defaults to `1.0`): + Fraction of denoising steps (0.0-1.0) after which guidance stops. + enabled (`bool`, defaults to `True`): + Whether this guider applies guidance at all. When `False`, only the conditional pass runs. """ # `pred_cond` is the base; the others are the guidance passes. @@ -68,9 +76,6 @@ def __init__( stg_scale: float = 0.0, modality_scale: float = 1.0, guidance_rescale: float = 0.0, - # STG perturbs whole transformer blocks, and one forward feeds *both* modalities, so the block list is a - # shared/transformer-level knob rather than a per-modality one; only the video guider carries it (the audio - # guider reuses the same STG forward via its own `stg_scale`). The denoiser reads it off the video guider. spatio_temporal_guidance_blocks: list[int] | None = None, start: float = 0.0, stop: float = 1.0, diff --git a/src/diffusers/modular_pipelines/ltx2/denoise.py b/src/diffusers/modular_pipelines/ltx2/denoise.py index 9742a6c7f3bc..b1c4657d4d04 100644 --- a/src/diffusers/modular_pipelines/ltx2/denoise.py +++ b/src/diffusers/modular_pipelines/ltx2/denoise.py @@ -391,6 +391,14 @@ def __call__(self, components, block_state: BlockState, i: int, t: torch.Tensor) # One single-batch forward per pass; store each modality's x0 prediction on the batch. `prepare_models` / # `cleanup_models` are the standard per-pass hook points -- no-ops here, since LTX-2 carries its # perturbations as transformer flags (set above) rather than hooks. + # + # Parity note. Running every pass (cond/uncond included) as its own single-batch forward -- rather than the + # batched `torch.cat([latents] * 2)` the standard `LTX2Pipeline` uses -- means this does NOT match the + # reference bitwise. GPU matmul is not batch-invariant, so `cond` computed alone differs from `cond` inside + # a batch-of-2: ~1e-6/op in fp32, but ~1e-2/op in bf16, which the CFG delta and the sampler amplify to ~10% + # mean-relative latent divergence. A batched cond+uncond forward would be fp32-bitwise but cannot drive the + # guider API per-pass, so this trades bitwiseness for using the guider API end-to-end. Gate any parity check + # against `LTX2Pipeline` on fp32 and treat bf16 as close-but-not-bitwise. for batch in guider_state: components.guider.prepare_models(components.transformer) cond_kwargs = {name: getattr(batch, name) for name in self._guider_input_fields}