Fix LoRA hot-swapping recompilation with different_shapes_for_compilation (#14297)

[tests] fix hot-swapping recompilation with different_shapes_for_compilation

`test_hotswapping_compiled_model_linear` and
`test_hotswapping_compiled_model_both_linear_and_other` failed for every model
setting `different_shapes_for_compilation`, e.g.:

    RecompileError: tensor 'hidden_states' size mismatch at index 1.
    expected 16, actual 32

The dummy inputs are `(batch, height * width, channels)` with `channels = 16`
and the first traced shape is `(4, 4)`, so the image sequence length equals the
channel count. Duck shaping assigns both dims the same symbol, which `img_in`
(an `nn.Linear` with constant `in_features`) then specializes to 16, forcing a
recompile on the next shape.

`use_duck_shape = False` was already set in `test_compile_on_different_shapes`
and `test_hotswapping_compile_on_different_shapes` (#11327) but the other two
multi-shape hot-swapping tests were missed. Move it into
`_check_model_hotswap` so all of them are covered, and drop the now-redundant
duplicate.

Also drop the two `xfail(strict=True)` markers on
`TestQwenImageTransformerLoRAHotSwap` that were masking this bug.

Co-authored-by: Sayak Paul <spsayakpaul@gmail.com>

Author: jiqing-feng

tests/models/testing_utils/lora.py

@@ -321,6 +321,12 @@ def _check_model_hotswap(
         fine.
         """
         different_shapes = self.different_shapes_for_compilation
+        if different_shapes is not None:
+            # Specifying `use_duck_shape=False` instructs the compiler if it should use the same symbolic
+            # variable to represent input sizes that are the same. For more details,
+            # check out this [comment](https://github.com/huggingface/diffusers/pull/11327#discussion_r2047659790).
+            torch.fx.experimental._config.use_duck_shape = False
+
         # create 2 adapters with different ranks and alphas
         torch.manual_seed(0)
         init_dict = self.get_init_dict()
@@ -557,10 +563,6 @@ def test_hotswapping_compile_on_different_shapes(self, tmp_path, rank0, rank1):
         different_shapes_for_compilation = self.different_shapes_for_compilation
         if different_shapes_for_compilation is None:
             pytest.skip(f"Skipping as `different_shapes_for_compilation` is not set for {self.__class__.__name__}.")
-        # Specifying `use_duck_shape=False` instructs the compiler if it should use the same symbolic
-        # variable to represent input sizes that are the same. For more details,
-        # check out this [comment](https://github.com/huggingface/diffusers/pull/11327#discussion_r2047659790).
-        torch.fx.experimental._config.use_duck_shape = False
 
         target_modules = ["to_q", "to_k", "to_v", "to_out.0"]
         with torch._dynamo.config.patch(error_on_recompile=True):

tests/models/transformers/test_models_transformer_qwenimage.py

@@ -305,14 +305,6 @@ class TestQwenImageTransformerLoRA(QwenImageTransformerTesterConfig, LoraTesterM
 class TestQwenImageTransformerLoRAHotSwap(QwenImageTransformerTesterConfig, LoraHotSwappingForModelTesterMixin):
     """LoRA hot-swapping tests for QwenImage Transformer."""
 
-    @pytest.mark.xfail(True, reason="Recompilation issues.", strict=True)
-    def test_hotswapping_compiled_model_linear(self):
-        super().test_hotswapping_compiled_model_linear()
-
-    @pytest.mark.xfail(True, reason="Recompilation issues.", strict=True)
-    def test_hotswapping_compiled_model_both_linear_and_other(self):
-        super().test_hotswapping_compiled_model_both_linear_and_other()
-
     @property
     def different_shapes_for_compilation(self):
         return [(4, 4), (4, 8), (8, 8)]