resolve conflicts

Author: sayakpaul

.ai/references/testing.md

@@ -34,6 +34,7 @@ Follow the style introduced in [#14113](https://github.com/huggingface/diffusers
   - `torch.nn.MultiheadAttention` is the common instance: it passes `self.out_proj.weight` straight to `torch.nn.functional.multi_head_attention_forward` instead of calling `self.out_proj`, so the hook on `out_proj` never fires. `SiglipVisionModel`'s attention pooling head wraps one — see `tests/pipelines/hunyuan_video/test_hunyuan_video_framepack.py`, whose `image_encoder` is excluded for this reason.
   - `HunyuanDiTAttentionPool` (`src/diffusers/models/embeddings.py`) shows the same failure without an MHA module: a plain `nn.Module` that hands its `q_proj` / `k_proj` / `v_proj` / `c_proj` weights to `torch.nn.functional.multi_head_attention_forward`, so all four projections stay offloaded rather than just one. `HunyuanDiT2DModel` opts out of group offloading entirely with `_supports_group_offloading = False`.
   - Before adding a skip or an exclusion, confirm the failure still reproduces — several existing skips are stale, having outlived the upstream cause.
+- **`encode_prompt` reading a component that isn't a text encoder or tokenizer?** `test_encode_prompt_works_in_isolation` rebuilds the pipeline with only the components whose names contain `text` or `tokenizer`. When `encode_prompt` also needs another one — a `processor` used for chat templating, say — list it in `text_stack_component_names` on the config class rather than re-implementing the test.
 - **IP-Adapter tests** live in their own class decorated with `@is_ip_adapter`, subclassing only the config (not `PipelineTesterMixin`). UNet pipelines that load adapters through the standard `IPAdapterMixin` API compose the shared `IPAdapterTesterMixin` (`tests/pipelines/testing_utils/ip_adapter.py`, exported from `..testing_utils`); pipelines whose IP-Adapter API differs (Flux, for example) keep a bespoke mixin next to their own tests.
 
 #### LoRA tests

src/diffusers/models/transformers/transformer_skyreels_v2.py

@@ -563,7 +563,7 @@ class SkyReelsV2Transformer3DModel(
     _supports_gradient_checkpointing = True
     _skip_layerwise_casting_patterns = ["patch_embedding", "condition_embedder", "norm"]
     _no_split_modules = ["SkyReelsV2TransformerBlock"]
-    _keep_in_fp32_modules = ["time_embedder", "scale_shift_table", "norm1", "norm2", "norm3"]
+    _keep_in_fp32_modules = ["rope", "time_embedder", "scale_shift_table", "norm1", "norm2", "norm3"]
     _keys_to_ignore_on_load_unexpected = ["norm_added_q"]
     _repeated_blocks = ["SkyReelsV2TransformerBlock"]
 

src/diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion.py

@@ -113,7 +113,8 @@ def __call__(
                 generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
                 tensor is generated by sampling using the supplied random `generator`.
             output_type (`str`, *optional*, defaults to `"pil"`):
-                The output format of the generated image. Choose between `PIL.Image` or `np.array`.
+                The output format of the generated image. Choose between `"pil"` (`PIL.Image`), `"np"` (`np.array`) or
+                `"pt"` (`torch.Tensor`).
             return_dict (`bool`, *optional*, defaults to `True`):
                 Whether or not to return a [`ImagePipelineOutput`] instead of a plain tuple.
 
@@ -220,9 +221,12 @@ def __call__(
         image = self.vqvae.decode(latents).sample
 
         image = (image / 2 + 0.5).clamp(0, 1)
-        image = image.cpu().permute(0, 2, 3, 1).numpy()
-        if output_type == "pil":
-            image = self.numpy_to_pil(image)
+
+        if output_type != "pt":
+            image = image.cpu().permute(0, 2, 3, 1).numpy()
+
+            if output_type == "pil":
+                image = self.numpy_to_pil(image)
 
         if not return_dict:
             return (image,)

src/diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion_superresolution.py

@@ -97,7 +97,8 @@ def __call__(
                 A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make
                 generation deterministic.
             output_type (`str`, *optional*, defaults to `"pil"`):
-                The output format of the generated image. Choose between `PIL.Image` or `np.array`.
+                The output format of the generated image. Choose between `"pil"` (`PIL.Image`), `"np"` (`np.array`) or
+                `"pt"` (`torch.Tensor`).
             return_dict (`bool`, *optional*, defaults to `True`):
                 Whether or not to return a [`ImagePipelineOutput`] instead of a plain tuple.
 
@@ -185,10 +186,12 @@ def __call__(
         image = self.vqvae.decode(latents).sample
         image = torch.clamp(image, -1.0, 1.0)
         image = image / 2 + 0.5
-        image = image.cpu().permute(0, 2, 3, 1).numpy()
 
-        if output_type == "pil":
-            image = self.numpy_to_pil(image)
+        if output_type != "pt":
+            image = image.cpu().permute(0, 2, 3, 1).numpy()
+
+            if output_type == "pil":
+                image = self.numpy_to_pil(image)
 
         if not return_dict:
             return (image,)

src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2_diffusion_forcing_i2v.py

@@ -1035,7 +1035,8 @@ def __call__(
         # Final decoding step - convert latents to pixels
         if not output_type == "latent":
             if last_image is not None:
-                latents = latents[:, :, :-prefix_video_latents_frames, :, :].to(self.vae.dtype)
+                latents = latents[:, :, :-prefix_video_latents_frames, :, :]
+            latents = latents.to(self.vae.dtype)
             latents_mean = (
                 torch.tensor(self.vae.config.latents_mean)
                 .view(1, self.vae.config.z_dim, 1, 1, 1)

src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2_diffusion_forcing_v2v.py

@@ -441,6 +441,8 @@ def prepare_latents(
         latent_width = width // self.vae_scale_factor_spatial
 
         if long_video_iter == 0:
+            # `video` is preprocessed in float32; the VAE may be running in another dtype (fp16/bf16).
+            video = video.to(self.vae.dtype)
             prefix_video_latents = [
                 retrieve_latents(
                     self.vae.encode(
@@ -1049,7 +1051,9 @@ def __call__(
             )
             latents = latents / latents_std + latents_mean
             video_generated = self.vae.decode(latents, return_dict=False)[0]
-            video = torch.cat([video_original, video_generated], dim=2)
+            # `video_original` is kept in float32 by `preprocess_video`; promote the decoded frames to match it
+            # so the two halves can be concatenated when the VAE runs in fp16/bf16.
+            video = torch.cat([video_original, video_generated.to(video_original.dtype)], dim=2)
             video = self.video_processor.postprocess_video(video, output_type=output_type)
         else:
             video = latents

src/diffusers/pipelines/visualcloze/pipeline_visualcloze_combined.py

@@ -146,6 +146,9 @@ def __init__(
             transformer=transformer,
             scheduler=scheduler,
         )
+        # `resolution` is not a module, so it has to be registered explicitly to survive a
+        # `save_pretrained` / `from_pretrained` round-trip.
+        self.register_to_config(resolution=resolution)
 
         self.generation_pipe = VisualClozeGenerationPipeline(
             vae=vae,
@@ -376,6 +379,10 @@ def __call__(
             output_type=output_type if upsampling_strength == 0 else "pil",
         )
         if upsampling_strength == 0:
+            # Offload all models. The inner pipelines free their own (empty) hooks, so the ones installed on this
+            # pipeline by `enable_model_cpu_offload` have to be freed here.
+            self.maybe_free_model_hooks()
+
             if not return_dict:
                 return (generation_output,)
 
@@ -434,6 +441,9 @@ def __call__(
         else:
             output = image
 
+        # Offload all models
+        self.maybe_free_model_hooks()
+
         if not return_dict:
             return (output,)
 

src/diffusers/pipelines/visualcloze/pipeline_visualcloze_generation.py

@@ -175,6 +175,9 @@ def __init__(
             transformer=transformer,
             scheduler=scheduler,
         )
+        # `resolution` is not a module, so it has to be registered explicitly to survive a
+        # `save_pretrained` / `from_pretrained` round-trip.
+        self.register_to_config(resolution=resolution)
         self.resolution = resolution
         self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8
         # Flux latents are turned into 2x2 patches and packed. This means the latent width and height has to be divisible
@@ -715,8 +718,9 @@ def __call__(
                 Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.
                 If not provided, pooled text embeddings will be generated from `prompt` input argument.
             output_type (`str`, *optional*, defaults to `"pil"`):
-                The output format of the generate image. Choose between
-                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
+                The output format of the generate image. Choose between `"pil"`
+                ([PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image`), `"np"` (`np.array`) or `"pt"`
+                (`torch.Tensor`).
             return_dict (`bool`, *optional*, defaults to `True`):
                 Whether or not to return a [`~pipelines.flux.FluxPipelineOutput`] instead of a plain tuple.
             joint_attention_kwargs (`dict`, *optional*):
@@ -907,11 +911,17 @@ def __call__(
                     if cur_target_position[i]:
                         if output_type == "pil":
                             cropped.append(cur_image.crop((start, 0, start + size[1], size[0])))
+                        elif output_type == "pt":
+                            # `"pt"` images are `(channels, height, width)`, unlike the `(height, width, channels)`
+                            # layout of `"np"`, so the spatial crop applies to the last two axes.
+                            cropped.append(cur_image[:, 0 : size[0], start : start + size[1]])
                         else:
                             cropped.append(cur_image[0 : size[0], start : start + size[1]])
                     start += size[1]
                 image.append(cropped)
-            if output_type != "pil":
+            if output_type == "pt":
+                image = torch.stack([arr for sub_image in image for arr in sub_image], dim=0)
+            elif output_type != "pil":
                 image = np.concatenate([arr[None] for sub_image in image for arr in sub_image], axis=0)
 
         # Offload all models

tests/modular_pipelines/test_modular_pipelines_custom_blocks.py

@@ -36,7 +36,7 @@
 )
 from diffusers.utils import logging
 
-from ..testing_utils import CaptureLogger, nightly, require_torch, require_torch_accelerator, slow, torch_device
+from ..testing_utils import CaptureLogger, nightly, require_torch_accelerator, require_torch_gpu, slow, torch_device
 
 
 def _create_tiny_model_dir(model_dir):
@@ -722,7 +722,7 @@ def test_kwargs_type_input_in_pipeline_call_params(self):
 
 @slow
 @nightly
-@require_torch
+@require_torch_gpu
 class TestKreaCustomBlocksIntegration:
     repo_id = "krea/krea-realtime-video"