From 0bb5845a5335e29ae88231bb4937f151244baab6 Mon Sep 17 00:00:00 2001 From: ErenAta16 Date: Fri, 17 Jul 2026 19:20:25 +0300 Subject: [PATCH 1/7] Fix _merged_adapters bookkeeping desync on partial unfuse_lora(components=...) _merged_adapters is a single set shared across the whole pipeline, while actual merge state is tracked per component by PEFT. When the same adapter is fused into multiple components (the default when fuse_lora() is called with no adapter_names) and later only some of those components are unfused via unfuse_lora(components=[...]), the old code removed the adapter name from _merged_adapters unconditionally, even when it was still physically merged into the base weights of the untouched component(s). num_fused_loras/fused_loras would then report the pipeline as having nothing (or less) fused than it actually does. Track unmerge candidates during the per-component unmerge loop, then only drop an adapter from the pipeline-wide set once confirmed unmerged in every _lora_loadable_modules component via the new _is_adapter_merged_in_any_component helper. Also adds a regression test (test_fuse_unfuse_partial_components_keeps_merged_adapter_bookkeeping in tests/lora/utils.py) covering the partial-components case, since the existing fuse/unfuse tests only ever pass every loadable component at once. Fixes #14214. --- src/diffusers/loaders/lora_base.py | 30 ++++++++++++++++++++--- tests/lora/utils.py | 39 ++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 3 deletions(-) diff --git a/src/diffusers/loaders/lora_base.py b/src/diffusers/loaders/lora_base.py index dc01475ac8de..402f7beda285 100644 --- a/src/diffusers/loaders/lora_base.py +++ b/src/diffusers/loaders/lora_base.py @@ -664,6 +664,13 @@ def unfuse_lora(self, components: list[str] | None = None, **kwargs): if len(components) == 0: raise ValueError("`components` cannot be an empty list.") + # self._merged_adapters is shared across the whole pipeline (all _lora_loadable_modules), while + # merge state is tracked per component by PEFT. The same adapter name can be fused into multiple + # components independently, so unmerging it in only some of them doesn't mean it's fully unfused. + # Track candidates here and only drop them from the pipeline-wide set once confirmed unmerged + # everywhere, below. + unmerge_candidates: set[str] = set() + for fuse_component in components: if fuse_component not in self._lora_loadable_modules: raise ValueError(f"{fuse_component} is not found in {self._lora_loadable_modules=}.") @@ -673,11 +680,28 @@ def unfuse_lora(self, components: list[str] | None = None, **kwargs): if issubclass(model.__class__, (ModelMixin, PreTrainedModel)): for module in model.modules(): if isinstance(module, BaseTunerLayer): - for adapter in set(module.merged_adapters): - if adapter and adapter in self._merged_adapters: - self._merged_adapters = self._merged_adapters - {adapter} + unmerge_candidates.update(module.merged_adapters) module.unmerge() + for adapter in unmerge_candidates: + if adapter not in self._merged_adapters: + continue + if not self._is_adapter_merged_in_any_component(adapter): + self._merged_adapters = self._merged_adapters - {adapter} + + def _is_adapter_merged_in_any_component(self, adapter_name: str) -> bool: + """Whether `adapter_name` is still merged into the base weights of any `_lora_loadable_modules` + component, used to keep the pipeline-wide `_merged_adapters` bookkeeping in sync with the real, + per-component PEFT merge state after a partial `unfuse_lora(components=...)` call.""" + for component in self._lora_loadable_modules: + model = getattr(self, component, None) + if model is None or not issubclass(model.__class__, (ModelMixin, PreTrainedModel)): + continue + for module in model.modules(): + if isinstance(module, BaseTunerLayer) and adapter_name in module.merged_adapters: + return True + return False + def set_adapters( self, adapter_names: list[str] | str, diff --git a/tests/lora/utils.py b/tests/lora/utils.py index 38aec8ce4807..ad5aaf1ab868 100644 --- a/tests/lora/utils.py +++ b/tests/lora/utils.py @@ -1711,6 +1711,45 @@ def test_simple_inference_with_text_lora_denoiser_fused_multi( pipe.unfuse_lora(components=self.pipeline_class._lora_loadable_modules) self.assertTrue(pipe.num_fused_loras == 0, f"{pipe.num_fused_loras=}, {pipe.fused_loras=}") + def test_fuse_unfuse_partial_components_keeps_merged_adapter_bookkeeping(self): + """ + `_merged_adapters` (backing `num_fused_loras`/`fused_loras`) is shared across the whole pipeline, + while actual merge state is tracked per component by PEFT. Fusing an adapter into every loadable + component and then unfusing only *some* of them should not report the adapter as fully unfused, + it's still merged into the untouched component(s). + """ + if "text_encoder" not in self.pipeline_class._lora_loadable_modules: + return + + components, text_lora_config, denoiser_lora_config = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + pipe.text_encoder.add_adapter(text_lora_config, "adapter-1") + self.assertTrue(check_if_lora_correctly_set(pipe.text_encoder), "Lora not correctly set in text encoder") + + denoiser = pipe.transformer if self.unet_kwargs is None else pipe.unet + denoiser_component_name = "unet" if self.unet_kwargs is not None else "transformer" + denoiser.add_adapter(denoiser_lora_config, "adapter-1") + self.assertTrue(check_if_lora_correctly_set(denoiser), "Lora not correctly set in denoiser.") + + pipe.fuse_lora(components=["text_encoder", denoiser_component_name], adapter_names=["adapter-1"]) + self.assertTrue(pipe.num_fused_loras == 1, f"{pipe.num_fused_loras=}, {pipe.fused_loras=}") + + # Only unfuse the text encoder; the denoiser still has "adapter-1" merged into its base weights. + pipe.unfuse_lora(components=["text_encoder"]) + self.assertTrue( + pipe.num_fused_loras == 1, + f"adapter-1 is still merged into {denoiser_component_name}, but num_fused_loras dropped to " + f"{pipe.num_fused_loras=}, {pipe.fused_loras=}", + ) + self.assertIn("adapter-1", pipe.fused_loras) + + # Now unfuse the remaining component; bookkeeping should correctly drop to 0. + pipe.unfuse_lora(components=[denoiser_component_name]) + self.assertTrue(pipe.num_fused_loras == 0, f"{pipe.num_fused_loras=}, {pipe.fused_loras=}") + def test_lora_scale_kwargs_match_fusion(self, expected_atol: float = 1e-3, expected_rtol: float = 1e-3): attention_kwargs_name = determine_attention_kwargs_name(self.pipeline_class) From cacdeee5fc3c4e412e49ed59238171014e2e104a Mon Sep 17 00:00:00 2001 From: ErenAta16 Date: Sat, 18 Jul 2026 11:53:17 +0300 Subject: [PATCH 2/7] Use pytest.skip convention for unsupported text-encoder LoRA case Matches the established skip pattern used elsewhere in this file instead of a silent early return, per review feedback. --- tests/lora/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/lora/utils.py b/tests/lora/utils.py index ad5aaf1ab868..a7385602a4cf 100644 --- a/tests/lora/utils.py +++ b/tests/lora/utils.py @@ -1718,8 +1718,8 @@ def test_fuse_unfuse_partial_components_keeps_merged_adapter_bookkeeping(self): component and then unfusing only *some* of them should not report the adapter as fully unfused, it's still merged into the untouched component(s). """ - if "text_encoder" not in self.pipeline_class._lora_loadable_modules: - return + if not self.supports_text_encoder_loras: + pytest.skip("Skipping test as text encoder LoRAs are not currently supported.") components, text_lora_config, denoiser_lora_config = self.get_dummy_components() pipe = self.pipeline_class(**components) From f6c54bc19a6d33e322bb4be273e8d0cc1cb2b74d Mon Sep 17 00:00:00 2001 From: ErenAta16 Date: Sat, 18 Jul 2026 12:27:56 +0300 Subject: [PATCH 3/7] Document pipeline-wide fused-adapter tracking; strengthen bookkeeping test - unfuse_lora docstring now notes num_fused_loras/fused_loras are tracked pipeline-wide, not per component. - Regression test now also asserts the adapter is absent from fused_loras after it's been unfused from every component. --- src/diffusers/loaders/lora_base.py | 5 +++++ tests/lora/utils.py | 1 + 2 files changed, 6 insertions(+) diff --git a/src/diffusers/loaders/lora_base.py b/src/diffusers/loaders/lora_base.py index 402f7beda285..166755f6ec63 100644 --- a/src/diffusers/loaders/lora_base.py +++ b/src/diffusers/loaders/lora_base.py @@ -635,6 +635,11 @@ def unfuse_lora(self, components: list[str] | None = None, **kwargs): unfuse_text_encoder (`bool`, defaults to `True`): Whether to unfuse the text encoder LoRA parameters. If the text encoder wasn't monkey-patched with the LoRA parameters then it won't have any effect. + + Note that `num_fused_loras`/`fused_loras` (backed by `self._merged_adapters`) are tracked pipeline-wide, + not per component. An adapter fused into multiple `_lora_loadable_modules` components only drops out of + `fused_loras` once it has been unfused from all of them; unfusing it from only some components leaves it + listed as fused. """ if components is None: components = [] diff --git a/tests/lora/utils.py b/tests/lora/utils.py index a7385602a4cf..370f01c0e414 100644 --- a/tests/lora/utils.py +++ b/tests/lora/utils.py @@ -1749,6 +1749,7 @@ def test_fuse_unfuse_partial_components_keeps_merged_adapter_bookkeeping(self): # Now unfuse the remaining component; bookkeeping should correctly drop to 0. pipe.unfuse_lora(components=[denoiser_component_name]) self.assertTrue(pipe.num_fused_loras == 0, f"{pipe.num_fused_loras=}, {pipe.fused_loras=}") + self.assertNotIn("adapter-1", pipe.fused_loras) def test_lora_scale_kwargs_match_fusion(self, expected_atol: float = 1e-3, expected_rtol: float = 1e-3): attention_kwargs_name = determine_attention_kwargs_name(self.pipeline_class) From e11c893ed647d0bd4aade10983a247c4e1318d30 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 18 Jul 2026 09:59:35 +0000 Subject: [PATCH 4/7] Apply style fixes --- src/diffusers/loaders/lora_base.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/diffusers/loaders/lora_base.py b/src/diffusers/loaders/lora_base.py index 166755f6ec63..f31a30b33916 100644 --- a/src/diffusers/loaders/lora_base.py +++ b/src/diffusers/loaders/lora_base.py @@ -636,10 +636,10 @@ def unfuse_lora(self, components: list[str] | None = None, **kwargs): Whether to unfuse the text encoder LoRA parameters. If the text encoder wasn't monkey-patched with the LoRA parameters then it won't have any effect. - Note that `num_fused_loras`/`fused_loras` (backed by `self._merged_adapters`) are tracked pipeline-wide, - not per component. An adapter fused into multiple `_lora_loadable_modules` components only drops out of - `fused_loras` once it has been unfused from all of them; unfusing it from only some components leaves it - listed as fused. + Note that `num_fused_loras`/`fused_loras` (backed by `self._merged_adapters`) are tracked pipeline-wide, not + per component. An adapter fused into multiple `_lora_loadable_modules` components only drops out of + `fused_loras` once it has been unfused from all of them; unfusing it from only some components leaves it listed + as fused. """ if components is None: components = [] @@ -696,8 +696,8 @@ def unfuse_lora(self, components: list[str] | None = None, **kwargs): def _is_adapter_merged_in_any_component(self, adapter_name: str) -> bool: """Whether `adapter_name` is still merged into the base weights of any `_lora_loadable_modules` - component, used to keep the pipeline-wide `_merged_adapters` bookkeeping in sync with the real, - per-component PEFT merge state after a partial `unfuse_lora(components=...)` call.""" + component, used to keep the pipeline-wide `_merged_adapters` bookkeeping in sync with the real, per-component + PEFT merge state after a partial `unfuse_lora(components=...)` call.""" for component in self._lora_loadable_modules: model = getattr(self, component, None) if model is None or not issubclass(model.__class__, (ModelMixin, PreTrainedModel)): From b52aa289ad1c710f615a258019954541f8b2aa44 Mon Sep 17 00:00:00 2001 From: ErenAta16 Date: Sat, 18 Jul 2026 15:24:27 +0300 Subject: [PATCH 5/7] Trim unfuse_lora docstring to user-facing behavior only Drop the _merged_adapters implementation detail per review feedback; fused_loras is what's actually exposed to callers. --- src/diffusers/loaders/lora_base.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/diffusers/loaders/lora_base.py b/src/diffusers/loaders/lora_base.py index f31a30b33916..51b30698118e 100644 --- a/src/diffusers/loaders/lora_base.py +++ b/src/diffusers/loaders/lora_base.py @@ -636,10 +636,9 @@ def unfuse_lora(self, components: list[str] | None = None, **kwargs): Whether to unfuse the text encoder LoRA parameters. If the text encoder wasn't monkey-patched with the LoRA parameters then it won't have any effect. - Note that `num_fused_loras`/`fused_loras` (backed by `self._merged_adapters`) are tracked pipeline-wide, not - per component. An adapter fused into multiple `_lora_loadable_modules` components only drops out of - `fused_loras` once it has been unfused from all of them; unfusing it from only some components leaves it listed - as fused. + Note that `fused_loras` is tracked pipeline-wide, not per component. An adapter fused into multiple + components only drops out of `fused_loras` once it has been unfused from all of them; unfusing it from + only some components leaves it listed as fused. """ if components is None: components = [] From 266daa149964bb2c5ba9953e8a4ace616285f28a Mon Sep 17 00:00:00 2001 From: ErenAta16 <149434812+ErenAta16@users.noreply.github.com> Date: Wed, 19 Aug 2026 10:19:06 +0300 Subject: [PATCH 6/7] Drop the lora_base fix; an equivalent one landed on main --- src/diffusers/loaders/lora_base.py | 64 +++++++++++------------------- 1 file changed, 24 insertions(+), 40 deletions(-) diff --git a/src/diffusers/loaders/lora_base.py b/src/diffusers/loaders/lora_base.py index 51b30698118e..0009f04df9e2 100644 --- a/src/diffusers/loaders/lora_base.py +++ b/src/diffusers/loaders/lora_base.py @@ -37,7 +37,6 @@ get_adapter_name, is_accelerate_available, is_peft_available, - is_peft_version, is_transformers_available, is_transformers_version, logging, @@ -47,7 +46,7 @@ set_weights_and_activate_adapters, ) from ..utils.peft_utils import _create_lora_config -from ..utils.state_dict_utils import _load_sft_state_dict_metadata +from ..utils.state_dict_utils import _load_sft_file_metadata, _load_sft_state_dict_metadata if is_transformers_available(): @@ -209,7 +208,15 @@ def _fetch_state_dict( user_agent, allow_pickle, metadata=None, + return_file_metadata=False, ): + """ + `metadata` is diffusers' own LoRA adapter metadata, parsed out of the file's `__metadata__`. With + `return_file_metadata`, that `__metadata__` is returned in full as a third element. It is `None` whenever the + weights did not come from a safetensors file — a state dict passed in memory, or a pickled checkpoint — since there + is nowhere else for a header to live. + """ + file_metadata = None model_file = None if not isinstance(pretrained_model_name_or_path_or_dict, dict): # Let's first try to load .safetensors weights @@ -240,6 +247,8 @@ def _fetch_state_dict( ) state_dict = safetensors.torch.load_file(model_file, device="cpu") metadata = _load_sft_state_dict_metadata(model_file) + if return_file_metadata: + file_metadata = _load_sft_file_metadata(model_file) except (IOError, safetensors.SafetensorError) as e: if not allow_pickle: @@ -247,6 +256,7 @@ def _fetch_state_dict( # try loading non-safetensors weights model_file = None metadata = None + file_metadata = None pass if model_file is None: @@ -271,6 +281,8 @@ def _fetch_state_dict( else: state_dict = pretrained_model_name_or_path_or_dict + if return_file_metadata: + return state_dict, metadata, file_metadata return state_dict, metadata @@ -340,10 +352,6 @@ def _load_lora_into_text_encoder( peft_kwargs = {} if low_cpu_mem_usage: - if not is_peft_version(">=", "0.13.1"): - raise ValueError( - "`low_cpu_mem_usage=True` is not compatible with this `peft` version. Please update it with `pip install -U peft`." - ) if not is_transformers_version(">", "4.45.2"): # Note from sayakpaul: It's not in `transformers` stable yet. # https://github.com/huggingface/transformers/pull/33725/ @@ -544,8 +552,6 @@ def fuse_lora( r""" Fuses the LoRA parameters into the original parameters of the corresponding blocks. - > [!WARNING] > This is an experimental API. - Args: components: (`list[str]`): list of LoRA-injectable components to fuse the LoRAs into. lora_scale (`float`, defaults to 1.0): @@ -627,18 +633,12 @@ def unfuse_lora(self, components: list[str] | None = None, **kwargs): Reverses the effect of [`pipe.fuse_lora()`](https://huggingface.co/docs/diffusers/main/en/api/loaders#diffusers.loaders.LoraBaseMixin.fuse_lora). - > [!WARNING] > This is an experimental API. - Args: components (`list[str]`): list of LoRA-injectable components to unfuse LoRA from. unfuse_unet (`bool`, defaults to `True`): Whether to unfuse the UNet LoRA parameters. unfuse_text_encoder (`bool`, defaults to `True`): Whether to unfuse the text encoder LoRA parameters. If the text encoder wasn't monkey-patched with the LoRA parameters then it won't have any effect. - - Note that `fused_loras` is tracked pipeline-wide, not per component. An adapter fused into multiple - components only drops out of `fused_loras` once it has been unfused from all of them; unfusing it from - only some components leaves it listed as fused. """ if components is None: components = [] @@ -668,13 +668,6 @@ def unfuse_lora(self, components: list[str] | None = None, **kwargs): if len(components) == 0: raise ValueError("`components` cannot be an empty list.") - # self._merged_adapters is shared across the whole pipeline (all _lora_loadable_modules), while - # merge state is tracked per component by PEFT. The same adapter name can be fused into multiple - # components independently, so unmerging it in only some of them doesn't mean it's fully unfused. - # Track candidates here and only drop them from the pipeline-wide set once confirmed unmerged - # everywhere, below. - unmerge_candidates: set[str] = set() - for fuse_component in components: if fuse_component not in self._lora_loadable_modules: raise ValueError(f"{fuse_component} is not found in {self._lora_loadable_modules=}.") @@ -684,27 +677,18 @@ def unfuse_lora(self, components: list[str] | None = None, **kwargs): if issubclass(model.__class__, (ModelMixin, PreTrainedModel)): for module in model.modules(): if isinstance(module, BaseTunerLayer): - unmerge_candidates.update(module.merged_adapters) module.unmerge() - for adapter in unmerge_candidates: - if adapter not in self._merged_adapters: - continue - if not self._is_adapter_merged_in_any_component(adapter): - self._merged_adapters = self._merged_adapters - {adapter} - - def _is_adapter_merged_in_any_component(self, adapter_name: str) -> bool: - """Whether `adapter_name` is still merged into the base weights of any `_lora_loadable_modules` - component, used to keep the pipeline-wide `_merged_adapters` bookkeeping in sync with the real, per-component - PEFT merge state after a partial `unfuse_lora(components=...)` call.""" - for component in self._lora_loadable_modules: - model = getattr(self, component, None) - if model is None or not issubclass(model.__class__, (ModelMixin, PreTrainedModel)): - continue - for module in model.modules(): - if isinstance(module, BaseTunerLayer) and adapter_name in module.merged_adapters: - return True - return False + # Only remove an adapter from _merged_adapters once it is no longer + # physically merged in any remaining loadable component. + remaining_merged: set[str] = set() + for component_name in self._lora_loadable_modules: + component_model = getattr(self, component_name, None) + if isinstance(component_model, nn.Module): + for module in component_model.modules(): + if isinstance(module, BaseTunerLayer): + remaining_merged.update(module.merged_adapters) + self._merged_adapters = self._merged_adapters & remaining_merged def set_adapters( self, From 3466f53b22b86ab03fdf501c7d964549a6425821 Mon Sep 17 00:00:00 2001 From: ErenAta16 <149434812+ErenAta16@users.noreply.github.com> Date: Wed, 19 Aug 2026 10:19:25 +0300 Subject: [PATCH 7/7] Keep only the partial-unfuse regression test --- tests/lora/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/lora/utils.py b/tests/lora/utils.py index 370f01c0e414..999c06523ff9 100644 --- a/tests/lora/utils.py +++ b/tests/lora/utils.py @@ -1715,8 +1715,8 @@ def test_fuse_unfuse_partial_components_keeps_merged_adapter_bookkeeping(self): """ `_merged_adapters` (backing `num_fused_loras`/`fused_loras`) is shared across the whole pipeline, while actual merge state is tracked per component by PEFT. Fusing an adapter into every loadable - component and then unfusing only *some* of them should not report the adapter as fully unfused, - it's still merged into the untouched component(s). + component and then unfusing only *some* of them must not report the adapter as fully unfused, + it is still merged into the untouched component(s). """ if not self.supports_text_encoder_loras: pytest.skip("Skipping test as text encoder LoRAs are not currently supported.")