From bf304186b4d669d4d5c93b80361e0326b63c02f5 Mon Sep 17 00:00:00 2001 From: Shira Sassoon Date: Mon, 9 Feb 2026 12:44:06 +0200 Subject: [PATCH 01/18] fix selective folder publish and add include option --- src/fabric_cicd/_common/_validate_input.py | 26 +++++++++----- src/fabric_cicd/constants.py | 2 ++ src/fabric_cicd/fabric_workspace.py | 41 +++++++++++++++++----- src/fabric_cicd/publish.py | 32 ++++++++++++++--- 4 files changed, 81 insertions(+), 20 deletions(-) diff --git a/src/fabric_cicd/_common/_validate_input.py b/src/fabric_cicd/_common/_validate_input.py index cc9b411e..86ccafbd 100644 --- a/src/fabric_cicd/_common/_validate_input.py +++ b/src/fabric_cicd/_common/_validate_input.py @@ -177,8 +177,6 @@ def validate_experimental_param( Raises: InputError: If required feature flags are not enabled. """ - from fabric_cicd.constants import FeatureFlag - if param_value is None: return @@ -204,8 +202,6 @@ def validate_items_to_include(items_to_include: Optional[list[str]], operation: Raises: InputError: If required feature flags are not enabled. """ - from fabric_cicd.constants import FeatureFlag - validate_experimental_param( param_value=items_to_include, required_flag=FeatureFlag.ENABLE_ITEMS_TO_INCLUDE, @@ -224,8 +220,6 @@ def validate_folder_path_exclude_regex(folder_path_exclude_regex: Optional[str]) Raises: InputError: If required feature flags are not enabled. """ - from fabric_cicd.constants import FeatureFlag - validate_experimental_param( param_value=folder_path_exclude_regex, required_flag=FeatureFlag.ENABLE_EXCLUDE_FOLDER, @@ -234,6 +228,24 @@ def validate_folder_path_exclude_regex(folder_path_exclude_regex: Optional[str]) ) +def validate_folder_path_to_include(folder_path_to_include: Optional[str]) -> None: + """ + Validate folder_path_to_include parameter and check required feature flags. + + Args: + folder_path_to_include: List of folder paths (with items) to publish. + + Raises: + InputError: If required feature flags are not enabled. + """ + validate_experimental_param( + param_value=folder_path_to_include, + required_flag=FeatureFlag.ENABLE_INCLUDE_FOLDER, + warning_message="Folder path inclusion is enabled.", + risk_warning="Using folder_path_to_include is risky as it can prevent needed dependencies from being deployed. Use at your own risk.", + ) + + def validate_shortcut_exclude_regex(shortcut_exclude_regex: Optional[str]) -> None: """ Validate shortcut_exclude_regex parameter and check required feature flags. @@ -244,8 +256,6 @@ def validate_shortcut_exclude_regex(shortcut_exclude_regex: Optional[str]) -> No Raises: InputError: If required feature flags are not enabled. """ - from fabric_cicd.constants import FeatureFlag - validate_experimental_param( param_value=shortcut_exclude_regex, required_flag=FeatureFlag.ENABLE_SHORTCUT_EXCLUDE, diff --git a/src/fabric_cicd/constants.py b/src/fabric_cicd/constants.py index 9b4e15de..c4950604 100644 --- a/src/fabric_cicd/constants.py +++ b/src/fabric_cicd/constants.py @@ -123,6 +123,8 @@ class FeatureFlag(str, Enum): """Set to enable selective publishing/unpublishing of items.""" ENABLE_EXCLUDE_FOLDER = "enable_exclude_folder" """Set to enable folder-based exclusion during publish operations.""" + ENABLE_INCLUDE_FOLDER = "enable_include_folder" + """Set to enable folder-based inclusion during publish operations.""" ENABLE_SHORTCUT_EXCLUDE = "enable_shortcut_exclude" """Set to enable selective publishing of shortcuts in a Lakehouse.""" ENABLE_CONFIG_DEPLOY = "enable_config_deploy" diff --git a/src/fabric_cicd/fabric_workspace.py b/src/fabric_cicd/fabric_workspace.py index 600d5031..3bd81f3b 100644 --- a/src/fabric_cicd/fabric_workspace.py +++ b/src/fabric_cicd/fabric_workspace.py @@ -129,6 +129,7 @@ def __init__( self.environment = validate_environment(environment) self.publish_item_name_exclude_regex = None self.publish_folder_path_exclude_regex = None + self.publish_folder_path_to_include = None self.shortcut_exclude_regex = None self.items_to_include = None self.responses = None @@ -591,15 +592,29 @@ def _publish_item( logger.info(f"Skipping publishing of {item_type} '{item_name}' due to exclusion regex.") return - # Skip publishing if the item's folder path is excluded by the regex - if self.publish_folder_path_exclude_regex: - regex_pattern = check_regex(self.publish_folder_path_exclude_regex) + # Skip publishing if the folder path is excluded by the regex or not in the include list + if self.publish_folder_path_exclude_regex or self.publish_folder_path_to_include: relative_path = item.path.relative_to(Path(self.repository_directory)) - relative_path_str = relative_path.as_posix() - if regex_pattern.search(relative_path_str): - item.skip_publish = True - logger.info(f"Skipping publishing of {item_type} '{item_name}' due to folder path exclusion regex.") - return + # Build the folder path in the same format as repository_folders keys (/folder_name) + relative_parts = relative_path.as_posix().split("/") + # Remove the last part (item folder name) to get the parent folder path + folder_path = "/" + "/".join(relative_parts[:-1]) if len(relative_parts) > 1 else "" + if folder_path: + if self.publish_folder_path_exclude_regex: + regex_pattern = check_regex(self.publish_folder_path_exclude_regex) + if regex_pattern.search(folder_path): + item.skip_publish = True + logger.info( + f"Skipping publishing of {item_type} '{item_name}' due to folder path exclusion regex." + ) + return + + if self.publish_folder_path_to_include and folder_path not in self.publish_folder_path_to_include: + item.skip_publish = True + logger.info( + f"Skipping publishing of {item_type} '{item_name}' under {folder_path} as it is not in the include list." + ) + return # Skip publishing if the item is not in the include list if self.items_to_include: @@ -830,6 +845,16 @@ def _publish_folders(self) -> None: print_header("Publishing Workspace Folders") logger.info("Publishing Workspace Folders") for folder_path in sorted_folders: + # Skip folders not in the include list + if self.publish_folder_path_to_include and folder_path not in self.publish_folder_path_to_include: + logger.info(f"Skipping publishing of folder '{folder_path}' as it is not in the include list.") + continue + # Skip folders matching the exclusion regex + if self.publish_folder_path_exclude_regex: + regex_pattern = check_regex(self.publish_folder_path_exclude_regex) + if regex_pattern.search(folder_path): + logger.info(f"Skipping publishing of folder '{folder_path}' due to folder path exclusion regex.") + continue if folder_path in self.deployed_folders: # Folder already deployed, update local hierarchy self.repository_folders[folder_path] = self.deployed_folders[folder_path] diff --git a/src/fabric_cicd/publish.py b/src/fabric_cicd/publish.py index 1884dd02..ee7d6d5b 100644 --- a/src/fabric_cicd/publish.py +++ b/src/fabric_cicd/publish.py @@ -24,6 +24,7 @@ validate_environment, validate_fabric_workspace_obj, validate_folder_path_exclude_regex, + validate_folder_path_to_include, validate_items_to_include, validate_shortcut_exclude_regex, ) @@ -37,6 +38,7 @@ def publish_all_items( fabric_workspace_obj: FabricWorkspace, item_name_exclude_regex: Optional[str] = None, folder_path_exclude_regex: Optional[str] = None, + folder_path_to_include: Optional[list[str]] = None, items_to_include: Optional[list[str]] = None, shortcut_exclude_regex: Optional[str] = None, ) -> Optional[dict]: @@ -47,6 +49,7 @@ def publish_all_items( fabric_workspace_obj: The FabricWorkspace object containing the items to be published. item_name_exclude_regex: Regex pattern to exclude specific items from being published. folder_path_exclude_regex: Regex pattern to exclude items based on their folder path. + folder_path_to_include: List of folder paths (with items) that should be published. items_to_include: List of items in the format "item_name.item_type" that should be published. shortcut_exclude_regex: Regex pattern to exclude specific shortcuts from being published in lakehouses. @@ -58,6 +61,11 @@ def publish_all_items( not recommended due to item dependencies. To enable this feature, see How To -> Optional Features for information on which flags to enable. + folder_path_to_include: + This is an experimental feature in fabric-cicd. Use at your own risk as selective deployments are + not recommended due to item dependencies. To enable this feature, see How To -> Optional Features + for information on which flags to enable. + items_to_include: This is an experimental feature in fabric-cicd. Use at your own risk as selective deployments are not recommended due to item dependencies. To enable this feature, see How To -> Optional Features @@ -100,6 +108,18 @@ def publish_all_items( >>> folder_exclude_regex = "^legacy/" >>> publish_all_items(workspace, folder_path_exclude_regex=folder_exclude_regex) + With folder inclusion + >>> from fabric_cicd import FabricWorkspace, publish_all_items, append_feature_flag + >>> append_feature_flag("enable_experimental_features") + >>> append_feature_flag("enable_include_folder") + >>> workspace = FabricWorkspace( + ... workspace_id="your-workspace-id", + ... repository_directory="/path/to/repo", + ... item_type_in_scope=["Environment", "Notebook", "DataPipeline"] + ... ) + >>> folder_path_to_include = ["/subfolder"] + >>> publish_all_items(workspace, folder_path_to_include=folder_path_to_include) + With items to include >>> from fabric_cicd import FabricWorkspace, publish_all_items, append_feature_flag >>> append_feature_flag("enable_experimental_features") @@ -161,6 +181,14 @@ def publish_all_items( raise FailedPublishedItemStatusError(msg, logger) if FeatureFlag.DISABLE_WORKSPACE_FOLDER_PUBLISH.value not in constants.FEATURE_FLAG: + if folder_path_exclude_regex: + validate_folder_path_exclude_regex(folder_path_exclude_regex) + fabric_workspace_obj.publish_folder_path_exclude_regex = folder_path_exclude_regex + + if folder_path_to_include: + validate_folder_path_to_include(folder_path_to_include) + fabric_workspace_obj.publish_folder_path_to_include = folder_path_to_include + fabric_workspace_obj._refresh_deployed_folders() fabric_workspace_obj._refresh_repository_folders() fabric_workspace_obj._publish_folders() @@ -174,10 +202,6 @@ def publish_all_items( ) fabric_workspace_obj.publish_item_name_exclude_regex = item_name_exclude_regex - if folder_path_exclude_regex: - validate_folder_path_exclude_regex(folder_path_exclude_regex) - fabric_workspace_obj.publish_folder_path_exclude_regex = folder_path_exclude_regex - if items_to_include: validate_items_to_include(items_to_include, operation=constants.OperationType.PUBLISH) fabric_workspace_obj.items_to_include = items_to_include From 8a0e302c6b14a443053e1347252b69c1eb89c831 Mon Sep 17 00:00:00 2001 From: Shira Sassoon Date: Tue, 10 Feb 2026 10:22:15 +0200 Subject: [PATCH 02/18] update unit tests and config --- docs/how_to/optional_feature.md | 1 + src/fabric_cicd/_common/_config_utils.py | 1 + src/fabric_cicd/_common/_config_validator.py | 94 ++++++++---- src/fabric_cicd/constants.py | 5 + src/fabric_cicd/fabric_workspace.py | 1 + src/fabric_cicd/publish.py | 1 + tests/test_config_validator.py | 151 ++++++++++++++++++- tests/test_deploy_with_config.py | 122 +++++++++++++++ tests/test_publish.py | 132 +++++++++++++++- 9 files changed, 472 insertions(+), 36 deletions(-) diff --git a/docs/how_to/optional_feature.md b/docs/how_to/optional_feature.md index 5d367a1c..9b89dcac 100644 --- a/docs/how_to/optional_feature.md +++ b/docs/how_to/optional_feature.md @@ -20,6 +20,7 @@ For scenarios that aren't supported by default, fabric-cicd offers `feature-flag | `enable_experimental_features` | Set to enable experimental features, such as selective deployments | | | `enable_items_to_include` | Set to enable selective publishing/unpublishing of items | ☑️ | | `enable_exclude_folder` | Set to enable folder-based exclusion during publish operations | ☑️ | +| `enable_include_folder` | Set to enable folder-based inclusion during publish operations | ☑️ | | `enable_shortcut_exclude` | Set to enable selective publishing of shortcuts in a Lakehouse | ☑️ | | `enable_config_deploy` | Set to enable config file-based deployment | ☑️ | | `enable_response_collection` | Set to enable collection of API responses during publish operations | | diff --git a/src/fabric_cicd/_common/_config_utils.py b/src/fabric_cicd/_common/_config_utils.py index 555fbbc5..02efa664 100644 --- a/src/fabric_cicd/_common/_config_utils.py +++ b/src/fabric_cicd/_common/_config_utils.py @@ -113,6 +113,7 @@ def extract_publish_settings(config: dict, environment: str) -> dict: settings_to_update = [ "exclude_regex", "folder_exclude_regex", + "folders_to_include", "items_to_include", "shortcut_exclude_regex", ] diff --git a/src/fabric_cicd/_common/_config_validator.py b/src/fabric_cicd/_common/_config_validator.py index 449b89c3..167a1e20 100644 --- a/src/fabric_cicd/_common/_config_validator.py +++ b/src/fabric_cicd/_common/_config_validator.py @@ -690,6 +690,7 @@ def _validate_operation_section(self, section: dict[str, Any], section_name: str # Validate exclude_regex if present if "exclude_regex" in section: exclude_regex = section["exclude_regex"] + if isinstance(exclude_regex, str): if not exclude_regex.strip(): self.errors.append( @@ -705,15 +706,8 @@ def _validate_operation_section(self, section: dict[str, Any], section_name: str # Validate each environment's regex pattern for env, regex_pattern in exclude_regex.items(): - if not regex_pattern.strip(): - self.errors.append( - constants.CONFIG_VALIDATION_MSGS["field"]["empty_value"].format( - f"{section_name}.exclude_regex.{env}" - ) - ) - continue - self._validate_regex(regex_pattern, f"{section_name}.exclude_regex.{env}") + else: self.errors.append( constants.CONFIG_VALIDATION_MSGS["field"]["string_or_dict"].format( @@ -742,18 +736,13 @@ def _validate_operation_section(self, section: dict[str, Any], section_name: str # Validate each environment's items list for env, items_list in items.items(): - if not items_list: - self.errors.append( - constants.CONFIG_VALIDATION_MSGS["field"]["empty_list"].format( - f"{section_name}.items_to_include.{env}" - ) - ) - continue self._validate_items_list(items_list, f"{section_name}.items_to_include.{env}") else: self.errors.append( - constants.CONFIG_VALIDATION_MSGS["field"]["item_types_list_or_dict"].format(type(items).__name__) + constants.CONFIG_VALIDATION_MSGS["field"]["list_or_dict"].format( + f"{section_name}.items_to_include", type(items).__name__ + ) ) # Validate folder_exclude_regex if present (publish only) @@ -785,15 +774,8 @@ def _validate_operation_section(self, section: dict[str, Any], section_name: str # Validate each environment's regex pattern for env, regex_pattern in folder_exclude_regex.items(): - if not regex_pattern.strip(): - self.errors.append( - constants.CONFIG_VALIDATION_MSGS["field"]["empty_value"].format( - f"{section_name}.folder_exclude_regex.{env}" - ) - ) - continue - self._validate_regex(regex_pattern, f"{section_name}.folder_exclude_regex.{env}") + else: self.errors.append( constants.CONFIG_VALIDATION_MSGS["field"]["string_or_dict"].format( @@ -801,6 +783,42 @@ def _validate_operation_section(self, section: dict[str, Any], section_name: str ) ) + # Validate folders_to_include if present (publish only) + if "folders_to_include" in section: + if section_name != "publish": + self.errors.append( + constants.CONFIG_VALIDATION_MSGS["operation"]["unsupported_field"].format( + "folders_to_include", section_name + ) + ) + + folders = section["folders_to_include"] + if isinstance(folders, list): + if not folders: + self.errors.append( + constants.CONFIG_VALIDATION_MSGS["field"]["empty_list"].format( + f"{section_name}.folders_to_include" + ) + ) + else: + self._validate_folders_list(folders, f"{section_name}.folders_to_include") + + elif isinstance(folders, dict): + # Validate environment mapping + if not self._validate_environment_mapping(folders, f"{section_name}.folders_to_include", list): + return + + # Validate each environment's folders list + for env, folders_list in folders.items(): + self._validate_folders_list(folders_list, f"{section_name}.folders_to_include.{env}") + + else: + self.errors.append( + constants.CONFIG_VALIDATION_MSGS["field"]["list_or_dict"].format( + f"{section_name}.folders_to_include", type(folders).__name__ + ) + ) + # Validate shortcut_exclude_regex if present (publish only) if "shortcut_exclude_regex" in section: if section_name != "publish": @@ -828,15 +846,8 @@ def _validate_operation_section(self, section: dict[str, Any], section_name: str # Validate each environment's regex pattern for env, regex_pattern in shortcut_exclude_regex.items(): - if not regex_pattern.strip(): - self.errors.append( - constants.CONFIG_VALIDATION_MSGS["field"]["empty_value"].format( - f"{section_name}.shortcut_exclude_regex.{env}" - ) - ) - continue - self._validate_regex(regex_pattern, f"{section_name}.shortcut_exclude_regex.{env}") + else: self.errors.append( constants.CONFIG_VALIDATION_MSGS["field"]["string_or_dict"].format( @@ -885,6 +896,24 @@ def _validate_items_list(self, items_list: list, context: str) -> None: elif not item.strip(): self.errors.append(constants.CONFIG_VALIDATION_MSGS["operation"]["items_list_empty"].format(context, i)) + def _validate_folders_list(self, folders_list: list, context: str) -> None: + """Validate a list of folder paths with proper context for error messages.""" + for i, folder in enumerate(folders_list): + if not isinstance(folder, str): + self.errors.append( + constants.CONFIG_VALIDATION_MSGS["operation"]["folders_list_type"].format( + context, i, type(folder).__name__ + ) + ) + elif not folder.strip(): + self.errors.append( + constants.CONFIG_VALIDATION_MSGS["operation"]["folders_list_empty"].format(context, i) + ) + elif not folder.startswith("/"): + self.errors.append( + constants.CONFIG_VALIDATION_MSGS["operation"]["folders_list_prefix"].format(context, i, folder) + ) + def _validate_features_section(self, features: any) -> None: """Validate features section.""" if isinstance(features, list): @@ -989,6 +1018,7 @@ def _get_config_fields(config: dict) -> list[tuple[dict, str, str, bool, bool]]: # Publish section fields - optional (debug if missing) (config.get("publish", {}), "exclude_regex", "publish.exclude_regex", False, False), (config.get("publish", {}), "folder_exclude_regex", "publish.folder_exclude_regex", False, False), + (config.get("publish", {}), "folders_to_include", "publish.folders_to_include", False, False), (config.get("publish", {}), "shortcut_exclude_regex", "publish.shortcut_exclude_regex", False, False), (config.get("publish", {}), "items_to_include", "publish.items_to_include", False, False), (config.get("publish", {}), "skip", "publish.skip", False, False), diff --git a/src/fabric_cicd/constants.py b/src/fabric_cicd/constants.py index c4950604..c5ca0278 100644 --- a/src/fabric_cicd/constants.py +++ b/src/fabric_cicd/constants.py @@ -375,6 +375,7 @@ class OperationType(str, Enum): # Field validation "field": { "string_or_dict": "'{}' must be either a string or environment mapping dictionary (e.g., {{dev: 'dev_value', prod: 'prod_value'}}), got type {}", + "list_or_dict": "'{}' must be a list or environment mapping dictionary (e.g., {{dev: 'dev_value', prod: 'prod_value'}}), got type {}", "empty_value": "'{}' cannot be empty", "empty_list": "'{}' cannot be empty if specified", "invalid_guid": "'{}' must be a valid GUID format: {}", @@ -396,6 +397,7 @@ class OperationType(str, Enum): }, # Operation section validation "operation": { + "unsupported_field": "'{}' field is not supported in '{}' section", "not_dict": "'{}' section must be a dictionary, got {}", "invalid_regex": "'{}' in {} is not a valid regex pattern: {}", "items_list_type": "'{}[{}]' must be a string, got {}", @@ -405,6 +407,9 @@ class OperationType(str, Enum): "empty_section_env": "'{}.{}' cannot be empty if specified", "invalid_constant_key": "Constant key in '{}' must be a non-empty string, got: {}", "unknown_constant": "Unknown constant '{}' in '{}' - this constant does not exist in fabric_cicd.constants", + "folders_list_type": "'{}[{}]' must be a string, got {}", + "folders_list_empty": "'{}[{}]' cannot be empty", + "folders_list_prefix": "'{}[{}]' entry must start with '/' (got '{}')", }, # Log messages "log": { diff --git a/src/fabric_cicd/fabric_workspace.py b/src/fabric_cicd/fabric_workspace.py index 6b0820ec..eb7e3911 100644 --- a/src/fabric_cicd/fabric_workspace.py +++ b/src/fabric_cicd/fabric_workspace.py @@ -855,6 +855,7 @@ def _publish_folders(self) -> None: if regex_pattern.search(folder_path): logger.info(f"Skipping publishing of folder '{folder_path}' due to folder path exclusion regex.") continue + logger.debug(f"Folder path '{folder_path}' does not match the exclusion regex pattern.") if folder_path in self.deployed_folders: # Folder already deployed, update local hierarchy self.repository_folders[folder_path] = self.deployed_folders[folder_path] diff --git a/src/fabric_cicd/publish.py b/src/fabric_cicd/publish.py index 657589bb..b77d6811 100644 --- a/src/fabric_cicd/publish.py +++ b/src/fabric_cicd/publish.py @@ -418,6 +418,7 @@ def deploy_with_config( workspace, item_name_exclude_regex=publish_settings.get("exclude_regex"), folder_path_exclude_regex=publish_settings.get("folder_exclude_regex"), + folder_path_to_include=publish_settings.get("folders_to_include"), items_to_include=publish_settings.get("items_to_include"), shortcut_exclude_regex=publish_settings.get("shortcut_exclude_regex"), ) diff --git a/tests/test_config_validator.py b/tests/test_config_validator.py index 0192d3ee..2fc01aef 100644 --- a/tests/test_config_validator.py +++ b/tests/test_config_validator.py @@ -1375,7 +1375,8 @@ def test_get_config_fields_complete_config(self): }, "publish": { "exclude_regex": ".*_test", - "folder_exclude_regex": "^temp/", + "folder_exclude_regex": "^/temp", + "folders_to_include": ["/subfolder"], "shortcut_exclude_regex": "^shortcut_temp/", "items_to_include": ["item1"], "skip": False, @@ -1388,7 +1389,7 @@ def test_get_config_fields_complete_config(self): fields = _get_config_fields(config) # Should return all fields from all sections - assert len(fields) == 15 # Updated count with folder_exclude_regex and shortcut_exclude_regex + assert len(fields) == 16 # Updated count with folder_exclude_regex and shortcut_exclude_regex # Check some specific fields field_names = [field[1] for field in fields] @@ -1694,7 +1695,10 @@ def test_validate_operation_section_items_to_include_invalid_type(self): self.validator._validate_operation_section(section, "publish") assert len(self.validator.errors) == 1 - assert "must be either a list or environment mapping dictionary" in self.validator.errors[0] + assert ( + "'publish.items_to_include' must be a list or environment mapping dictionary (e.g., {dev: 'dev_value', prod: 'prod_value'}), got type str" + in self.validator.errors[0] + ) def test_validate_operation_section_skip_boolean(self): """Test _validate_operation_section with skip as boolean.""" @@ -1765,6 +1769,147 @@ def test_folder_exclude_regex_restricted_to_publish_section(self): # We can't test the negative case (unpublish) directly due to missing error message key # So we'll just document that the feature should be restricted to publish section + def test_validate_operation_section_folders_to_include_valid_list(self): + """Test _validate_operation_section with valid folders_to_include list.""" + section = {"folders_to_include": ["/FolderA", "/FolderB"]} + + self.validator._validate_operation_section(section, "publish") + + assert len(self.validator.errors) == 0 + + def test_validate_operation_section_folders_to_include_valid_env_mapping(self): + """Test _validate_operation_section with valid folders_to_include environment mapping.""" + section = {"folders_to_include": {"dev": ["/FolderA"], "prod": ["/FolderA", "/FolderB"]}} + + self.validator._validate_operation_section(section, "publish") + + assert len(self.validator.errors) == 0 + + def test_validate_operation_section_folders_to_include_empty_list(self): + """Test _validate_operation_section with empty folders_to_include list.""" + section = {"folders_to_include": []} + + self.validator._validate_operation_section(section, "publish") + + assert len(self.validator.errors) == 1 + assert ( + constants.CONFIG_VALIDATION_MSGS["field"]["empty_list"].format("publish.folders_to_include") + in self.validator.errors[0] + ) + + def test_validate_operation_section_folders_to_include_invalid_type(self): + """Test _validate_operation_section with folders_to_include invalid type.""" + section = {"folders_to_include": "not a list or dict"} + + self.validator._validate_operation_section(section, "publish") + + assert len(self.validator.errors) == 1 + assert ( + constants.CONFIG_VALIDATION_MSGS["field"]["list_or_dict"].format("publish.folders_to_include", "str") + in self.validator.errors[0] + ) + + def test_validate_operation_section_folders_to_include_unsupported_in_unpublish(self): + """Test _validate_operation_section with folders_to_include in unpublish section.""" + section = {"folders_to_include": ["/FolderA"]} + + self.validator._validate_operation_section(section, "unpublish") + + assert len(self.validator.errors) == 1 + assert ( + constants.CONFIG_VALIDATION_MSGS["operation"]["unsupported_field"].format("folders_to_include", "unpublish") + in self.validator.errors[0] + ) + + def test_validate_operation_section_folders_to_include_entry_not_string(self): + """Test _validate_operation_section with non-string entry in folders_to_include.""" + section = {"folders_to_include": ["/FolderA", 123, "/FolderB"]} + + self.validator._validate_operation_section(section, "publish") + + assert len(self.validator.errors) == 1 + assert ( + constants.CONFIG_VALIDATION_MSGS["operation"]["folders_list_type"].format( + "publish.folders_to_include", 1, "int" + ) + in self.validator.errors[0] + ) + + def test_validate_operation_section_folders_to_include_empty_string_entry(self): + """Test _validate_operation_section with empty string entry in folders_to_include.""" + section = {"folders_to_include": ["/FolderA", "", "/FolderB"]} + + self.validator._validate_operation_section(section, "publish") + + assert len(self.validator.errors) == 1 + assert ( + constants.CONFIG_VALIDATION_MSGS["operation"]["folders_list_empty"].format("publish.folders_to_include", 1) + in self.validator.errors[0] + ) + + def test_validate_operation_section_folders_to_include_missing_prefix(self): + """Test _validate_operation_section with folder entry missing leading slash.""" + section = {"folders_to_include": ["/FolderA", "FolderB"]} + + self.validator._validate_operation_section(section, "publish") + + assert len(self.validator.errors) == 1 + assert ( + constants.CONFIG_VALIDATION_MSGS["operation"]["folders_list_prefix"].format( + "publish.folders_to_include", 1, "FolderB" + ) + in self.validator.errors[0] + ) + + def test_validate_operation_section_folders_to_include_env_mapping_empty_list(self): + """Test _validate_operation_section with empty list in environment mapping for folders_to_include.""" + section = {"folders_to_include": {"dev": ["/FolderA"], "prod": []}} + + self.validator._validate_operation_section(section, "publish") + + assert len(self.validator.errors) == 1 + print(self.validator.errors[0]) + assert ( + constants.CONFIG_VALIDATION_MSGS["environment"]["empty_env_value"].format( + "publish.folders_to_include", "prod" + ) + in self.validator.errors[0] + ) + + def test_validate_operation_section_folders_to_include_env_mapping_invalid_entry(self): + """Test _validate_operation_section with invalid entry in environment mapping for folders_to_include.""" + section = {"folders_to_include": {"dev": ["/FolderA", "NoSlash"]}} + + self.validator._validate_operation_section(section, "publish") + + assert len(self.validator.errors) == 1 + assert ( + constants.CONFIG_VALIDATION_MSGS["operation"]["folders_list_prefix"].format( + "publish.folders_to_include.dev", 1, "NoSlash" + ) + in self.validator.errors[0] + ) + + def test_validate_operation_section_folders_to_include_nested_path(self): + """Test _validate_operation_section with nested folder paths in folders_to_include.""" + section = {"folders_to_include": ["/FolderA/SubFolder", "/FolderB/Sub1/Sub2"]} + + self.validator._validate_operation_section(section, "publish") + + assert len(self.validator.errors) == 0 + + def test_validate_operation_section_folders_to_include_whitespace_entry(self): + """Test _validate_operation_section with whitespace-only entry in folders_to_include.""" + section = {"folders_to_include": ["/FolderA", " "]} + + self.validator._validate_operation_section(section, "publish") + + assert len(self.validator.errors) == 1 + assert ( + constants.CONFIG_VALIDATION_MSGS["operation"]["folders_list_empty"].format("publish.folders_to_include", 1) + in self.validator.errors[0] + ) + def test_validate_operation_section_with_shortcut_exclude_regex(self): """Test _validate_operation_section with shortcut_exclude_regex.""" section = {"shortcut_exclude_regex": "^temp_.*"} diff --git a/tests/test_deploy_with_config.py b/tests/test_deploy_with_config.py index 974953b5..4acee948 100644 --- a/tests/test_deploy_with_config.py +++ b/tests/test_deploy_with_config.py @@ -459,6 +459,7 @@ def test_deploy_with_config_full_deployment(self, mock_unpublish, mock_publish, mock_workspace_instance, item_name_exclude_regex="^DONT_DEPLOY.*", folder_path_exclude_regex=None, + folder_path_to_include=None, items_to_include=None, shortcut_exclude_regex=None, ) @@ -643,12 +644,97 @@ def test_deploy_with_config_shortcut_exclude_regex(self, mock_unpublish, mock_pu mock_workspace_instance, item_name_exclude_regex=None, folder_path_exclude_regex=None, + folder_path_to_include=None, items_to_include=None, shortcut_exclude_regex="^temp_.*", ) # Verify unpublish was also called (but without shortcut_exclude_regex since it's publish-only) mock_unpublish.assert_called_once() + @patch("fabric_cicd.publish.FabricWorkspace") + @patch("fabric_cicd.publish.publish_all_items") + @patch("fabric_cicd.publish.unpublish_all_orphan_items") + @patch("fabric_cicd.constants.FEATURE_FLAG", set(["enable_experimental_features", "enable_config_deploy"])) + def test_folder_path_to_include_passed_to_publish(self, _mock_unpublish, mock_publish, mock_workspace, tmp_path): + """Test that folder_path_to_include from config is passed to publish_all_items.""" + test_repo_dir = tmp_path / "repo" + test_repo_dir.mkdir(parents=True) + + config_data = { + "core": { + "workspace_id": "11111111-1111-1111-1111-111111111111", + "repository_directory": str(test_repo_dir), + }, + "publish": { + "folders_to_include": ["/my/folder/path"], + }, + } + config_file = tmp_path / "config.yml" + with Path.open(config_file, "w") as f: + yaml.dump(config_data, f) + + mock_workspace.return_value = MagicMock() + deploy_with_config(str(config_file), "dev") + + call_args = mock_publish.call_args[1] + assert call_args["folder_path_to_include"] == ["/my/folder/path"] + + @patch("fabric_cicd.publish.FabricWorkspace") + @patch("fabric_cicd.publish.publish_all_items") + @patch("fabric_cicd.publish.unpublish_all_orphan_items") + @patch("fabric_cicd.constants.FEATURE_FLAG", set(["enable_experimental_features", "enable_config_deploy"])) + def test_folder_path_to_include_defaults_to_none(self, _mock_unpublish, mock_publish, mock_workspace, tmp_path): + """Test that folder_path_to_include defaults to None when not specified.""" + test_repo_dir = tmp_path / "repo" + test_repo_dir.mkdir(parents=True) + + config_data = { + "core": { + "workspace_id": "11111111-1111-1111-1111-111111111111", + "repository_directory": str(test_repo_dir), + }, + } + config_file = tmp_path / "config.yml" + with Path.open(config_file, "w") as f: + yaml.dump(config_data, f) + + mock_workspace.return_value = MagicMock() + deploy_with_config(str(config_file), "dev") + + call_args = mock_publish.call_args[1] + assert "folders_to_include" not in call_args + + @patch("fabric_cicd.publish.FabricWorkspace") + @patch("fabric_cicd.publish.publish_all_items") + @patch("fabric_cicd.publish.unpublish_all_orphan_items") + @patch("fabric_cicd.constants.FEATURE_FLAG", set(["enable_experimental_features", "enable_config_deploy"])) + def test_folder_path_to_include_environment_specific(self, _mock_unpublish, mock_publish, mock_workspace, tmp_path): + """Test that folder_path_to_include resolves environment-specific values.""" + test_repo_dir = tmp_path / "repo" + test_repo_dir.mkdir(parents=True) + + config_data = { + "core": { + "workspace_id": { + "dev": "11111111-1111-1111-1111-111111111111", + "prod": "22222222-2222-2222-2222-222222222222", + }, + "repository_directory": str(test_repo_dir), + }, + "publish": { + "folders_to_include": {"dev": ["/dev/folder"], "prod": ["/prod/folder"]}, + }, + } + config_file = tmp_path / "config.yml" + with Path.open(config_file, "w") as f: + yaml.dump(config_data, f) + + mock_workspace.return_value = MagicMock() + deploy_with_config(str(config_file), "dev") + + call_args = mock_publish.call_args[1] + assert call_args["folder_path_to_include"] == ["/dev/folder"] + class TestConfigIntegration: """Integration tests for config functionality.""" @@ -861,6 +947,42 @@ def test_extract_publish_settings_items_to_include_missing_environment(self): settings = extract_publish_settings(config, "prod") assert "items_to_include" not in settings + def test_extract_publish_settings_folders_to_include_list(self): + """Test extract_publish_settings returns folders_to_include as a list.""" + config = { + "publish": { + "folders_to_include": ["/my/folder/path"], + }, + } + result = extract_publish_settings(config, "dev") + assert result["folders_to_include"] == ["/my/folder/path"] + + def test_extract_publish_settings_folders_to_include_env_specific(self): + """Test extract_publish_settings resolves folders_to_include per environment.""" + config = { + "publish": { + "folders_to_include": {"dev": ["/dev/folder"], "prod": ["/prod/folder"]}, + }, + } + result = extract_publish_settings(config, "dev") + assert result["folders_to_include"] == ["/dev/folder"] + + def test_extract_publish_settings_folders_to_include_missing(self): + """Test extract_publish_settings defaults folders_to_include to None.""" + config = { + "publish": { + "exclude_regex": "^SKIP.*", + }, + } + settings = extract_publish_settings(config, "dev") + assert "folders_to_include" not in settings + + def test_extract_publish_settings_no_publish_section_folders_to_include(self): + """Test extract_publish_settings defaults folders_to_include to None when no publish section.""" + config = {} + settings = extract_publish_settings(config, "dev") + assert "folders_to_include" not in settings + class TestGetConfigValue: """Test the get_config_value utility function.""" diff --git a/tests/test_publish.py b/tests/test_publish.py index 3133b088..aee639f4 100644 --- a/tests/test_publish.py +++ b/tests/test_publish.py @@ -743,7 +743,7 @@ def test_legacy_folder_exclusion_example(mock_endpoint): ) # Test: Exclude all items in 'legacy' folder using the folder path regex pattern - exclude_regex = r"^legacy/" # Match items that start with 'legacy/' + exclude_regex = r"^/legacy" # Match items that start with 'legacy/' publish.publish_all_items(workspace, folder_path_exclude_regex=exclude_regex) # Verify that legacy items were excluded @@ -757,3 +757,133 @@ def test_legacy_folder_exclusion_example(mock_endpoint): # Restore original feature flags constants.FEATURE_FLAG.clear() constants.FEATURE_FLAG.update(original_flags) + + +def test_folder_inclusion_with_folder_path_to_include(mock_endpoint): + """Test that folder_path_to_include only publishes items in specified folders.""" + + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + + # Create items in 'active' folder (should be included) + active_notebook_dir = temp_path / "active" / "ActiveNotebook.Notebook" + active_notebook_dir.mkdir(parents=True, exist_ok=True) + + active_notebook_platform = active_notebook_dir / ".platform" + active_notebook_metadata = { + "metadata": { + "type": "Notebook", + "displayName": "ActiveNotebook", + "description": "Active notebook to be included", + }, + "config": {"logicalId": "active-notebook-id"}, + } + + with active_notebook_platform.open("w", encoding="utf-8") as f: + json.dump(active_notebook_metadata, f) + + with (active_notebook_dir / "dummy.txt").open("w", encoding="utf-8") as f: + f.write("Dummy file content") + + active_model_dir = temp_path / "active" / "ActiveModel.SemanticModel" + active_model_dir.mkdir(parents=True, exist_ok=True) + + active_model_platform = active_model_dir / ".platform" + active_model_metadata = { + "metadata": { + "type": "SemanticModel", + "displayName": "ActiveModel", + "description": "Active semantic model to be included", + }, + "config": {"logicalId": "active-model-id"}, + } + + with active_model_platform.open("w", encoding="utf-8") as f: + json.dump(active_model_metadata, f) + + with (active_model_dir / "dummy.txt").open("w", encoding="utf-8") as f: + f.write("Dummy file content") + + # Create items in 'archive' folder (should be excluded) + archive_notebook_dir = temp_path / "archive" / "ArchivedNotebook.Notebook" + archive_notebook_dir.mkdir(parents=True, exist_ok=True) + + archive_notebook_platform = archive_notebook_dir / ".platform" + archive_notebook_metadata = { + "metadata": { + "type": "Notebook", + "displayName": "ArchivedNotebook", + "description": "Archived notebook to be excluded", + }, + "config": {"logicalId": "archived-notebook-id"}, + } + + with archive_notebook_platform.open("w", encoding="utf-8") as f: + json.dump(archive_notebook_metadata, f) + + with (archive_notebook_dir / "dummy.txt").open("w", encoding="utf-8") as f: + f.write("Dummy file content") + + # Create root-level item (should be excluded since not in included folder) + root_notebook_dir = temp_path / "RootNotebook.Notebook" + root_notebook_dir.mkdir(parents=True, exist_ok=True) + + root_notebook_platform = root_notebook_dir / ".platform" + root_notebook_metadata = { + "metadata": { + "type": "Notebook", + "displayName": "RootNotebook", + "description": "Root level notebook to be excluded", + }, + "config": {"logicalId": "root-notebook-id"}, + } + + with root_notebook_platform.open("w", encoding="utf-8") as f: + json.dump(root_notebook_metadata, f) + + with (root_notebook_dir / "dummy.txt").open("w", encoding="utf-8") as f: + f.write("Dummy file content") + + with ( + patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint), + patch.object( + FabricWorkspace, "_refresh_deployed_items", new=lambda self: setattr(self, "deployed_items", {}) + ), + patch.object( + FabricWorkspace, "_refresh_deployed_folders", new=lambda self: setattr(self, "deployed_folders", {}) + ), + ): + # Enable experimental feature flags for folder inclusion + original_flags = constants.FEATURE_FLAG.copy() + constants.FEATURE_FLAG.add("enable_experimental_features") + constants.FEATURE_FLAG.add("enable_include_folder") + + try: + workspace = FabricWorkspace( + workspace_id="12345678-1234-5678-abcd-1234567890ab", + repository_directory=str(temp_path), + item_type_in_scope=["Notebook", "SemanticModel"], + ) + + # Test: Only include items in 'active' folder + publish.publish_all_items(workspace, folder_path_to_include=["/active"]) + + # Verify that repository_items are populated correctly + assert "Notebook" in workspace.repository_items + assert "SemanticModel" in workspace.repository_items + + # Check that active items were NOT marked for exclusion (skip_publish = False) + assert workspace.repository_items["Notebook"]["ActiveNotebook"].skip_publish is False + assert workspace.repository_items["SemanticModel"]["ActiveModel"].skip_publish is False + + # Check that archive items were marked for exclusion (skip_publish = True) + assert workspace.repository_items["Notebook"]["ArchivedNotebook"].skip_publish is True + + # Root-level items have an empty folder_path so they are not evaluated + # against folder_path_to_include and remain publishable + assert workspace.repository_items["Notebook"]["RootNotebook"].skip_publish is False + + finally: + # Restore original feature flags + constants.FEATURE_FLAG.clear() + constants.FEATURE_FLAG.update(original_flags) From 32b61e11ce0a43bba3d44005e34f3499b1d1ffe2 Mon Sep 17 00:00:00 2001 From: Shira Sassoon Date: Tue, 10 Feb 2026 12:33:39 +0200 Subject: [PATCH 03/18] fix linting --- tests/test_deploy_with_config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_deploy_with_config.py b/tests/test_deploy_with_config.py index 4acee948..a82f432f 100644 --- a/tests/test_deploy_with_config.py +++ b/tests/test_deploy_with_config.py @@ -655,7 +655,7 @@ def test_deploy_with_config_shortcut_exclude_regex(self, mock_unpublish, mock_pu @patch("fabric_cicd.publish.publish_all_items") @patch("fabric_cicd.publish.unpublish_all_orphan_items") @patch("fabric_cicd.constants.FEATURE_FLAG", set(["enable_experimental_features", "enable_config_deploy"])) - def test_folder_path_to_include_passed_to_publish(self, _mock_unpublish, mock_publish, mock_workspace, tmp_path): + def test_folder_path_to_include_passed_to_publish(self, _mock_unpublish, mock_publish, mock_workspace, tmp_path): # noqa: PT019 """Test that folder_path_to_include from config is passed to publish_all_items.""" test_repo_dir = tmp_path / "repo" test_repo_dir.mkdir(parents=True) @@ -683,7 +683,7 @@ def test_folder_path_to_include_passed_to_publish(self, _mock_unpublish, mock_pu @patch("fabric_cicd.publish.publish_all_items") @patch("fabric_cicd.publish.unpublish_all_orphan_items") @patch("fabric_cicd.constants.FEATURE_FLAG", set(["enable_experimental_features", "enable_config_deploy"])) - def test_folder_path_to_include_defaults_to_none(self, _mock_unpublish, mock_publish, mock_workspace, tmp_path): + def test_folder_path_to_include_defaults_to_none(self, _mock_unpublish, mock_publish, mock_workspace, tmp_path): # noqa: PT019 """Test that folder_path_to_include defaults to None when not specified.""" test_repo_dir = tmp_path / "repo" test_repo_dir.mkdir(parents=True) @@ -708,7 +708,7 @@ def test_folder_path_to_include_defaults_to_none(self, _mock_unpublish, mock_pub @patch("fabric_cicd.publish.publish_all_items") @patch("fabric_cicd.publish.unpublish_all_orphan_items") @patch("fabric_cicd.constants.FEATURE_FLAG", set(["enable_experimental_features", "enable_config_deploy"])) - def test_folder_path_to_include_environment_specific(self, _mock_unpublish, mock_publish, mock_workspace, tmp_path): + def test_folder_path_to_include_environment_specific(self, _mock_unpublish, mock_publish, mock_workspace, tmp_path): # noqa: PT019 """Test that folder_path_to_include resolves environment-specific values.""" test_repo_dir = tmp_path / "repo" test_repo_dir.mkdir(parents=True) From 519271993a4471e54583d5172e69e9a7fd45915e Mon Sep 17 00:00:00 2001 From: Shira Sassoon Date: Tue, 10 Feb 2026 13:52:55 +0200 Subject: [PATCH 04/18] add sample --- sample/workspace/config.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sample/workspace/config.yml b/sample/workspace/config.yml index c1988fe4..d4abcb41 100644 --- a/sample/workspace/config.yml +++ b/sample/workspace/config.yml @@ -28,10 +28,16 @@ publish: # Publish configuration (optional) exclude_regex: "^DONT_DEPLOY.*" # Regex pattern to exclude items from publishing # folder_exclude_regex: "^DONT_DEPLOY_FOLDER/" # Regex pattern to exclude folder with items from publishing (requires feature flags) + + # folders_to_include: # Optional list of specific folders (with items) to publish (requires feature flags) + # - "/subfolderA" + # - "/subfolderA/subfolderB" # items_to_include: # Optional list of specific items to publish (requires feature flags) # - "Hello World.Notebook" # - "Run Hello World.DataPipeline" + + # shortcut_exclude_regex: "^DONT_DEPLOY_SHORTCUT.*" # Regex pattern to exclude Lakehouse shortcuts from publishing (requires feature flags) skip: # Skip publishing for specific environments dev: true # Skip publishing in dev environment From c84617806a2012f8503fdc8108c0ea9472f20025 Mon Sep 17 00:00:00 2001 From: Shira Sassoon Date: Tue, 10 Feb 2026 14:32:45 +0200 Subject: [PATCH 05/18] Rename config field, add field to constant, fix path regex --- docs/how_to/config_deployment.md | 57 +++++++++-- sample/workspace/config.yml | 4 +- src/fabric_cicd/_common/_config_utils.py | 2 +- src/fabric_cicd/_common/_config_validator.py | 20 ++-- src/fabric_cicd/constants.py | 8 +- src/fabric_cicd/publish.py | 4 +- tests/test_config_validator.py | 102 ++++++++++--------- tests/test_deploy_with_config.py | 46 ++++----- 8 files changed, 149 insertions(+), 94 deletions(-) diff --git a/docs/how_to/config_deployment.md b/docs/how_to/config_deployment.md index 9cf55523..9d60f2ff 100644 --- a/docs/how_to/config_deployment.md +++ b/docs/how_to/config_deployment.md @@ -115,19 +115,30 @@ core: `publish` is optional and can be used to control item publishing behavior. It includes various optional settings to enable/disable publishing operations or selectively publish items. +**Important:** To effectively use folder exclusion/inclusion, ensure the folder path contains a `/` preceding the folder name — for example, `/folder_name` for a single folder or `/folder_name/nested_folder_name` for nested folders. For regex patterns, ensure the pattern matches folders with a `/` preceding the folder name. For folder inclusion lists, ensure the path exactly matches this format. + ```yaml publish: # Optional - pattern to exclude items from publishing exclude_regex: - # Optional - pattern to exclude items in specific folders from publishing + # Optional - pattern to exclude specific folder paths with items from publishing (requires feature flags) folder_exclude_regex: + # Optional - specific folder paths with items to publish (requires feature flags) + folder_path_to_include: + - + - + - + # Optional - specific items to publish (requires feature flags) items_to_include: - - + # Optional - pattern to exclude Lakehouse shortcuts from publishing (requires feature flags) + shortcut_exclude_regex: + # Optional - control publishing by environment skip: ``` @@ -141,15 +152,32 @@ publish: : : - # Optional - pattern to exclude items in specific folders from publishing + # Optional - pattern to exclude specific folder paths with items from publishing (requires feature flags) folder_exclude_regex: : : + # Optional - specific folder paths with items to publish (requires feature flags) + folder_path_to_include: + : + - + - + : + - + # Optional - specific items to publish (requires feature flags) items_to_include: - - - - + : + - + - + : + - + - + + # Optional - pattern to exclude Lakehouse Shortcuts from publishing (requires feature flags) + shortcut_exclude_regex: + : + : # Optional - control publishing by environment skip: @@ -188,7 +216,9 @@ unpublish: items_to_include: : - + - : + - - # Optional - control unpublishing by environment @@ -215,7 +245,9 @@ features: features: : - + - : + - - ``` @@ -262,6 +294,7 @@ Fields are categorized as **required** or **optional**, which affects how missin | `publish.exclude_regex` | ❌ | Debug logged, setting skipped | | `publish.folder_exclude_regex` | ❌ | Debug logged, setting skipped | | `publish.shortcut_exclude_regex` | ❌ | Debug logged, setting skipped | +| `publish.folder_path_to_include` | ❌ | Debug logged, setting skipped | | `publish.items_to_include` | ❌ | Debug logged, setting skipped | | `publish.skip` | ❌ | Defaults to `False` | | `unpublish.exclude_regex` | ❌ | Debug logged, setting skipped | @@ -285,7 +318,7 @@ core: publish: # Only exclude legacy folders in prod environment folder_exclude_regex: - prod: "^legacy_.*" + prod: "^/legacy_.*" # dev and test not specified - no folder exclusion applied # Skip publish in dev, run in test and prod @@ -298,7 +331,7 @@ In this example: - Deploying to `dev`: No folder exclusion applied, `skip` = `true` - Deploying to `test`: No folder exclusion applied, `skip` = `false` -- Deploying to `prod`: `folder_exclude_regex` = `"^legacy_.*"`, `skip` = `false` +- Deploying to `prod`: `folder_exclude_regex` = `"^/legacy_.*"`, `skip` = `false` ### Logging Behavior @@ -347,12 +380,19 @@ publish: # Don't publish items matching this pattern exclude_regex: "^DONT_DEPLOY.*" - folder_exclude_regex: "^DONT_DEPLOY_FOLDER/" + folder_exclude_regex: "^/DONT_DEPLOY_FOLDER" + + folder_path_to_include: + - "/DEPLOY_FOLDER" + - "/DEPLOY_FOLDER/DEPLOY_NESTED_FOLDER" items_to_include: - "Hello World.Notebook" - "Run Hello World.DataPipeline" + shortcut_exclude_regex: + test: "^temp_.*" + skip: dev: true test: false @@ -371,6 +411,9 @@ features: - enable_shortcut_publish - enable_experimental_features - enable_items_to_include + - enable_exclude_folder + - enable_include_folder + - enable_shortcut_exclude constants: DEFAULT_API_ROOT_URL: "https://api.fabric.microsoft.com" diff --git a/sample/workspace/config.yml b/sample/workspace/config.yml index d4abcb41..0f4a8adf 100644 --- a/sample/workspace/config.yml +++ b/sample/workspace/config.yml @@ -27,9 +27,9 @@ core: # Core configurations publish: # Publish configuration (optional) exclude_regex: "^DONT_DEPLOY.*" # Regex pattern to exclude items from publishing - # folder_exclude_regex: "^DONT_DEPLOY_FOLDER/" # Regex pattern to exclude folder with items from publishing (requires feature flags) + # folder_exclude_regex: "^/DONT_DEPLOY_FOLDER" # Regex pattern to exclude folder paths with items from publishing (requires feature flags) - # folders_to_include: # Optional list of specific folders (with items) to publish (requires feature flags) + # folder_path_to_include: # Optional list of specific folder paths with items to publish (requires feature flags) # - "/subfolderA" # - "/subfolderA/subfolderB" diff --git a/src/fabric_cicd/_common/_config_utils.py b/src/fabric_cicd/_common/_config_utils.py index 02efa664..e517efac 100644 --- a/src/fabric_cicd/_common/_config_utils.py +++ b/src/fabric_cicd/_common/_config_utils.py @@ -113,7 +113,7 @@ def extract_publish_settings(config: dict, environment: str) -> dict: settings_to_update = [ "exclude_regex", "folder_exclude_regex", - "folders_to_include", + "folder_path_to_include", "items_to_include", "shortcut_exclude_regex", ] diff --git a/src/fabric_cicd/_common/_config_validator.py b/src/fabric_cicd/_common/_config_validator.py index 167a1e20..7b6af830 100644 --- a/src/fabric_cicd/_common/_config_validator.py +++ b/src/fabric_cicd/_common/_config_validator.py @@ -783,39 +783,39 @@ def _validate_operation_section(self, section: dict[str, Any], section_name: str ) ) - # Validate folders_to_include if present (publish only) - if "folders_to_include" in section: + # Validate folder_path_to_include if present (publish only) + if "folder_path_to_include" in section: if section_name != "publish": self.errors.append( constants.CONFIG_VALIDATION_MSGS["operation"]["unsupported_field"].format( - "folders_to_include", section_name + "folder_path_to_include", section_name ) ) - folders = section["folders_to_include"] + folders = section["folder_path_to_include"] if isinstance(folders, list): if not folders: self.errors.append( constants.CONFIG_VALIDATION_MSGS["field"]["empty_list"].format( - f"{section_name}.folders_to_include" + f"{section_name}.folder_path_to_include" ) ) else: - self._validate_folders_list(folders, f"{section_name}.folders_to_include") + self._validate_folders_list(folders, f"{section_name}.folder_path_to_include") elif isinstance(folders, dict): # Validate environment mapping - if not self._validate_environment_mapping(folders, f"{section_name}.folders_to_include", list): + if not self._validate_environment_mapping(folders, f"{section_name}.folder_path_to_include", list): return # Validate each environment's folders list for env, folders_list in folders.items(): - self._validate_folders_list(folders_list, f"{section_name}.folders_to_include.{env}") + self._validate_folders_list(folders_list, f"{section_name}.folder_path_to_include.{env}") else: self.errors.append( constants.CONFIG_VALIDATION_MSGS["field"]["list_or_dict"].format( - f"{section_name}.folders_to_include", type(folders).__name__ + f"{section_name}.folder_path_to_include", type(folders).__name__ ) ) @@ -1018,7 +1018,7 @@ def _get_config_fields(config: dict) -> list[tuple[dict, str, str, bool, bool]]: # Publish section fields - optional (debug if missing) (config.get("publish", {}), "exclude_regex", "publish.exclude_regex", False, False), (config.get("publish", {}), "folder_exclude_regex", "publish.folder_exclude_regex", False, False), - (config.get("publish", {}), "folders_to_include", "publish.folders_to_include", False, False), + (config.get("publish", {}), "folder_path_to_include", "publish.folder_path_to_include", False, False), (config.get("publish", {}), "shortcut_exclude_regex", "publish.shortcut_exclude_regex", False, False), (config.get("publish", {}), "items_to_include", "publish.items_to_include", False, False), (config.get("publish", {}), "skip", "publish.skip", False, False), diff --git a/src/fabric_cicd/constants.py b/src/fabric_cicd/constants.py index 94e55204..e8e855c8 100644 --- a/src/fabric_cicd/constants.py +++ b/src/fabric_cicd/constants.py @@ -323,7 +323,13 @@ class OperationType(str, Enum): }, "publish": { "type": dict, - "settings": ["exclude_regex", "folder_exclude_regex", "items_to_include", "shortcut_exclude_regex", "skip"], + "settings": [ + "exclude_regex", + "folder_exclude_regex", + "folder_path_to_include", + "shortcut_exclude_regex", + "skip", + ], }, "unpublish": {"type": dict, "settings": ["exclude_regex", "items_to_include", "skip"]}, "features": {"type": (list, dict), "settings": []}, diff --git a/src/fabric_cicd/publish.py b/src/fabric_cicd/publish.py index dec34dee..5cd6d589 100644 --- a/src/fabric_cicd/publish.py +++ b/src/fabric_cicd/publish.py @@ -105,7 +105,7 @@ def publish_all_items( ... repository_directory="/path/to/repo", ... item_type_in_scope=["Environment", "Notebook", "DataPipeline"] ... ) - >>> folder_exclude_regex = "^legacy/" + >>> folder_exclude_regex = "^/legacy" >>> publish_all_items(workspace, folder_path_exclude_regex=folder_exclude_regex) With folder inclusion @@ -410,7 +410,7 @@ def deploy_with_config( workspace, item_name_exclude_regex=publish_settings.get("exclude_regex"), folder_path_exclude_regex=publish_settings.get("folder_exclude_regex"), - folder_path_to_include=publish_settings.get("folders_to_include"), + folder_path_to_include=publish_settings.get("folder_path_to_include"), items_to_include=publish_settings.get("items_to_include"), shortcut_exclude_regex=publish_settings.get("shortcut_exclude_regex"), ) diff --git a/tests/test_config_validator.py b/tests/test_config_validator.py index 2fc01aef..4a5d661c 100644 --- a/tests/test_config_validator.py +++ b/tests/test_config_validator.py @@ -1376,7 +1376,7 @@ def test_get_config_fields_complete_config(self): "publish": { "exclude_regex": ".*_test", "folder_exclude_regex": "^/temp", - "folders_to_include": ["/subfolder"], + "folder_path_to_include": ["/subfolder"], "shortcut_exclude_regex": "^shortcut_temp/", "items_to_include": ["item1"], "skip": False, @@ -1732,7 +1732,7 @@ def test_validate_operation_section_skip_invalid_type(self): def test_validate_operation_section_with_folder_exclude_regex(self): """Test _validate_operation_section with folder_exclude_regex.""" - section = {"folder_exclude_regex": "^DONT_DEPLOY_FOLDER/"} + section = {"folder_exclude_regex": "^/DONT_DEPLOY_FOLDER"} self.validator._validate_operation_section(section, "publish") @@ -1749,7 +1749,7 @@ def test_validate_operation_section_with_invalid_folder_exclude_regex(self): def test_validate_operation_section_with_folder_exclude_regex_environment_mapping(self): """Test _validate_operation_section with folder_exclude_regex environment mapping.""" - section = {"folder_exclude_regex": {"dev": "^DEV_FOLDER/", "prod": "^PROD_FOLDER/"}} + section = {"folder_exclude_regex": {"dev": "^/DEV_FOLDER", "prod": "^/PROD_FOLDER"}} self.validator._validate_operation_section(section, "publish") @@ -1761,7 +1761,7 @@ def test_folder_exclude_regex_restricted_to_publish_section(self): # This test doesn't directly test the implementation or error message # Test that it's allowed in publish - section_publish = {"folder_exclude_regex": "^DONT_DEPLOY_FOLDER/"} + section_publish = {"folder_exclude_regex": "^/DONT_DEPLOY_FOLDER"} self.validator.errors = [] # Reset errors self.validator._validate_operation_section(section_publish, "publish") assert len(self.validator.errors) == 0 # Should be valid in publish @@ -1769,101 +1769,105 @@ def test_folder_exclude_regex_restricted_to_publish_section(self): # We can't test the negative case (unpublish) directly due to missing error message key # So we'll just document that the feature should be restricted to publish section - def test_validate_operation_section_folders_to_include_valid_list(self): - """Test _validate_operation_section with valid folders_to_include list.""" - section = {"folders_to_include": ["/FolderA", "/FolderB"]} + def test_validate_operation_section_folder_path_to_include_valid_list(self): + """Test _validate_operation_section with valid folder_path_to_include list.""" + section = {"folder_path_to_include": ["/FolderA", "/FolderB"]} self.validator._validate_operation_section(section, "publish") assert len(self.validator.errors) == 0 - def test_validate_operation_section_folders_to_include_valid_env_mapping(self): - """Test _validate_operation_section with valid folders_to_include environment mapping.""" - section = {"folders_to_include": {"dev": ["/FolderA"], "prod": ["/FolderA", "/FolderB"]}} + def test_validate_operation_section_folder_path_to_include_valid_env_mapping(self): + """Test _validate_operation_section with valid folder_path_to_include environment mapping.""" + section = {"folder_path_to_include": {"dev": ["/FolderA"], "prod": ["/FolderA", "/FolderB"]}} self.validator._validate_operation_section(section, "publish") assert len(self.validator.errors) == 0 - def test_validate_operation_section_folders_to_include_empty_list(self): - """Test _validate_operation_section with empty folders_to_include list.""" - section = {"folders_to_include": []} + def test_validate_operation_section_folder_path_to_include_empty_list(self): + """Test _validate_operation_section with empty folder_path_to_include list.""" + section = {"folder_path_to_include": []} self.validator._validate_operation_section(section, "publish") assert len(self.validator.errors) == 1 assert ( - constants.CONFIG_VALIDATION_MSGS["field"]["empty_list"].format("publish.folders_to_include") + constants.CONFIG_VALIDATION_MSGS["field"]["empty_list"].format("publish.folder_path_to_include") in self.validator.errors[0] ) - def test_validate_operation_section_folders_to_include_invalid_type(self): - """Test _validate_operation_section with folders_to_include invalid type.""" - section = {"folders_to_include": "not a list or dict"} + def test_validate_operation_section_folder_path_to_include_invalid_type(self): + """Test _validate_operation_section with folder_path_to_include invalid type.""" + section = {"folder_path_to_include": "not a list or dict"} self.validator._validate_operation_section(section, "publish") assert len(self.validator.errors) == 1 assert ( - constants.CONFIG_VALIDATION_MSGS["field"]["list_or_dict"].format("publish.folders_to_include", "str") + constants.CONFIG_VALIDATION_MSGS["field"]["list_or_dict"].format("publish.folder_path_to_include", "str") in self.validator.errors[0] ) - def test_validate_operation_section_folders_to_include_unsupported_in_unpublish(self): - """Test _validate_operation_section with folders_to_include in unpublish section.""" - section = {"folders_to_include": ["/FolderA"]} + def test_validate_operation_section_folder_path_to_include_unsupported_in_unpublish(self): + """Test _validate_operation_section with folder_path_to_include in unpublish section.""" + section = {"folder_path_to_include": ["/FolderA"]} self.validator._validate_operation_section(section, "unpublish") assert len(self.validator.errors) == 1 assert ( - constants.CONFIG_VALIDATION_MSGS["operation"]["unsupported_field"].format("folders_to_include", "unpublish") + constants.CONFIG_VALIDATION_MSGS["operation"]["unsupported_field"].format( + "folder_path_to_include", "unpublish" + ) in self.validator.errors[0] ) - def test_validate_operation_section_folders_to_include_entry_not_string(self): - """Test _validate_operation_section with non-string entry in folders_to_include.""" - section = {"folders_to_include": ["/FolderA", 123, "/FolderB"]} + def test_validate_operation_section_folder_path_to_include_entry_not_string(self): + """Test _validate_operation_section with non-string entry in folder_path_to_include.""" + section = {"folder_path_to_include": ["/FolderA", 123, "/FolderB"]} self.validator._validate_operation_section(section, "publish") assert len(self.validator.errors) == 1 assert ( constants.CONFIG_VALIDATION_MSGS["operation"]["folders_list_type"].format( - "publish.folders_to_include", 1, "int" + "publish.folder_path_to_include", 1, "int" ) in self.validator.errors[0] ) - def test_validate_operation_section_folders_to_include_empty_string_entry(self): - """Test _validate_operation_section with empty string entry in folders_to_include.""" - section = {"folders_to_include": ["/FolderA", "", "/FolderB"]} + def test_validate_operation_section_folder_path_to_include_empty_string_entry(self): + """Test _validate_operation_section with empty string entry in folder_path_to_include.""" + section = {"folder_path_to_include": ["/FolderA", "", "/FolderB"]} self.validator._validate_operation_section(section, "publish") assert len(self.validator.errors) == 1 assert ( - constants.CONFIG_VALIDATION_MSGS["operation"]["folders_list_empty"].format("publish.folders_to_include", 1) + constants.CONFIG_VALIDATION_MSGS["operation"]["folders_list_empty"].format( + "publish.folder_path_to_include", 1 + ) in self.validator.errors[0] ) - def test_validate_operation_section_folders_to_include_missing_prefix(self): + def test_validate_operation_section_folder_path_to_include_missing_prefix(self): """Test _validate_operation_section with folder entry missing leading slash.""" - section = {"folders_to_include": ["/FolderA", "FolderB"]} + section = {"folder_path_to_include": ["/FolderA", "FolderB"]} self.validator._validate_operation_section(section, "publish") assert len(self.validator.errors) == 1 assert ( constants.CONFIG_VALIDATION_MSGS["operation"]["folders_list_prefix"].format( - "publish.folders_to_include", 1, "FolderB" + "publish.folder_path_to_include", 1, "FolderB" ) in self.validator.errors[0] ) - def test_validate_operation_section_folders_to_include_env_mapping_empty_list(self): - """Test _validate_operation_section with empty list in environment mapping for folders_to_include.""" - section = {"folders_to_include": {"dev": ["/FolderA"], "prod": []}} + def test_validate_operation_section_folder_path_to_include_env_mapping_empty_list(self): + """Test _validate_operation_section with empty list in environment mapping for folder_path_to_include.""" + section = {"folder_path_to_include": {"dev": ["/FolderA"], "prod": []}} self.validator._validate_operation_section(section, "publish") @@ -1871,42 +1875,44 @@ def test_validate_operation_section_folders_to_include_env_mapping_empty_list(se print(self.validator.errors[0]) assert ( constants.CONFIG_VALIDATION_MSGS["environment"]["empty_env_value"].format( - "publish.folders_to_include", "prod" + "publish.folder_path_to_include", "prod" ) in self.validator.errors[0] ) - def test_validate_operation_section_folders_to_include_env_mapping_invalid_entry(self): - """Test _validate_operation_section with invalid entry in environment mapping for folders_to_include.""" - section = {"folders_to_include": {"dev": ["/FolderA", "NoSlash"]}} + def test_validate_operation_section_folder_path_to_include_env_mapping_invalid_entry(self): + """Test _validate_operation_section with invalid entry in environment mapping for folder_path_to_include.""" + section = {"folder_path_to_include": {"dev": ["/FolderA", "NoSlash"]}} self.validator._validate_operation_section(section, "publish") assert len(self.validator.errors) == 1 assert ( constants.CONFIG_VALIDATION_MSGS["operation"]["folders_list_prefix"].format( - "publish.folders_to_include.dev", 1, "NoSlash" + "publish.folder_path_to_include.dev", 1, "NoSlash" ) in self.validator.errors[0] ) - def test_validate_operation_section_folders_to_include_nested_path(self): - """Test _validate_operation_section with nested folder paths in folders_to_include.""" - section = {"folders_to_include": ["/FolderA/SubFolder", "/FolderB/Sub1/Sub2"]} + def test_validate_operation_section_folder_path_to_include_nested_path(self): + """Test _validate_operation_section with nested folder paths in folder_path_to_include.""" + section = {"folder_path_to_include": ["/FolderA/SubFolder", "/FolderB/Sub1/Sub2"]} self.validator._validate_operation_section(section, "publish") assert len(self.validator.errors) == 0 - def test_validate_operation_section_folders_to_include_whitespace_entry(self): - """Test _validate_operation_section with whitespace-only entry in folders_to_include.""" - section = {"folders_to_include": ["/FolderA", " "]} + def test_validate_operation_section_folder_path_to_include_whitespace_entry(self): + """Test _validate_operation_section with whitespace-only entry in folder_path_to_include.""" + section = {"folder_path_to_include": ["/FolderA", " "]} self.validator._validate_operation_section(section, "publish") assert len(self.validator.errors) == 1 assert ( - constants.CONFIG_VALIDATION_MSGS["operation"]["folders_list_empty"].format("publish.folders_to_include", 1) + constants.CONFIG_VALIDATION_MSGS["operation"]["folders_list_empty"].format( + "publish.folder_path_to_include", 1 + ) in self.validator.errors[0] ) diff --git a/tests/test_deploy_with_config.py b/tests/test_deploy_with_config.py index 0cd93631..5dcce730 100644 --- a/tests/test_deploy_with_config.py +++ b/tests/test_deploy_with_config.py @@ -660,7 +660,7 @@ def test_folder_path_to_include_passed_to_publish(self, _mock_unpublish, mock_pu "repository_directory": str(test_repo_dir), }, "publish": { - "folders_to_include": ["/my/folder/path"], + "folder_path_to_include": ["/my/folder/path"], }, } config_file = tmp_path / "config.yml" @@ -696,7 +696,7 @@ def test_folder_path_to_include_defaults_to_none(self, _mock_unpublish, mock_pub deploy_with_config(str(config_file), "dev") call_args = mock_publish.call_args[1] - assert "folders_to_include" not in call_args + assert "folder_path_to_include" not in call_args @patch("fabric_cicd.publish.FabricWorkspace") @patch("fabric_cicd.publish.publish_all_items") @@ -716,7 +716,7 @@ def test_folder_path_to_include_environment_specific(self, _mock_unpublish, mock "repository_directory": str(test_repo_dir), }, "publish": { - "folders_to_include": {"dev": ["/dev/folder"], "prod": ["/prod/folder"]}, + "folder_path_to_include": {"dev": ["/dev/folder"], "prod": ["/prod/folder"]}, }, } config_file = tmp_path / "config.yml" @@ -825,33 +825,33 @@ def test_extract_publish_settings_with_folder_exclude_regex(self): """Test extracting publish settings with folder_exclude_regex.""" config = { "publish": { - "folder_exclude_regex": "^DONT_DEPLOY_FOLDER/", + "folder_exclude_regex": "^/DONT_DEPLOY_FOLDER", } } settings = extract_publish_settings(config, "dev") - assert settings["folder_exclude_regex"] == "^DONT_DEPLOY_FOLDER/" + assert settings["folder_exclude_regex"] == "^/DONT_DEPLOY_FOLDER" def test_extract_publish_settings_with_environment_specific_folder_exclude_regex(self): """Test extracting publish settings with environment-specific folder_exclude_regex.""" config = { "publish": { - "folder_exclude_regex": {"dev": "^DEV_FOLDER/", "prod": "^PROD_FOLDER/"}, + "folder_exclude_regex": {"dev": "^/DEV_FOLDER", "prod": "^/PROD_FOLDER"}, } } settings = extract_publish_settings(config, "dev") - assert settings["folder_exclude_regex"] == "^DEV_FOLDER/" + assert settings["folder_exclude_regex"] == "^/DEV_FOLDER" settings = extract_publish_settings(config, "prod") - assert settings["folder_exclude_regex"] == "^PROD_FOLDER/" + assert settings["folder_exclude_regex"] == "^/PROD_FOLDER" def test_extract_publish_settings_missing_environment_skips_setting(self): """Test that missing environment in optional publish settings skips the setting.""" config = { "publish": { "exclude_regex": {"dev": "^DEV.*"}, # Only dev defined - "folder_exclude_regex": {"dev": "^DEV_FOLDER/"}, # Only dev defined + "folder_exclude_regex": {"dev": "^/DEV_FOLDER"}, # Only dev defined } } @@ -941,41 +941,41 @@ def test_extract_publish_settings_items_to_include_missing_environment(self): settings = extract_publish_settings(config, "prod") assert "items_to_include" not in settings - def test_extract_publish_settings_folders_to_include_list(self): - """Test extract_publish_settings returns folders_to_include as a list.""" + def test_extract_publish_settings_folder_path_to_include_list(self): + """Test extract_publish_settings returns folder_path_to_include as a list.""" config = { "publish": { - "folders_to_include": ["/my/folder/path"], + "folder_path_to_include": ["/my/folder/path"], }, } result = extract_publish_settings(config, "dev") - assert result["folders_to_include"] == ["/my/folder/path"] + assert result["folder_path_to_include"] == ["/my/folder/path"] - def test_extract_publish_settings_folders_to_include_env_specific(self): - """Test extract_publish_settings resolves folders_to_include per environment.""" + def test_extract_publish_settings_folder_path_to_include_env_specific(self): + """Test extract_publish_settings resolves folder_path_to_include per environment.""" config = { "publish": { - "folders_to_include": {"dev": ["/dev/folder"], "prod": ["/prod/folder"]}, + "folder_path_to_include": {"dev": ["/dev/folder"], "prod": ["/prod/folder"]}, }, } result = extract_publish_settings(config, "dev") - assert result["folders_to_include"] == ["/dev/folder"] + assert result["folder_path_to_include"] == ["/dev/folder"] - def test_extract_publish_settings_folders_to_include_missing(self): - """Test extract_publish_settings defaults folders_to_include to None.""" + def test_extract_publish_settings_folder_path_to_include_missing(self): + """Test extract_publish_settings defaults folder_path_to_include to None.""" config = { "publish": { "exclude_regex": "^SKIP.*", }, } settings = extract_publish_settings(config, "dev") - assert "folders_to_include" not in settings + assert "folder_path_to_include" not in settings - def test_extract_publish_settings_no_publish_section_folders_to_include(self): - """Test extract_publish_settings defaults folders_to_include to None when no publish section.""" + def test_extract_publish_settings_no_publish_section_folder_path_to_include(self): + """Test extract_publish_settings defaults folder_path_to_include to None when no publish section.""" config = {} settings = extract_publish_settings(config, "dev") - assert "folders_to_include" not in settings + assert "folder_path_to_include" not in settings class TestGetConfigValue: From b8ff3c6bb2b874a47c09d7b774fb1f52cb856378 Mon Sep 17 00:00:00 2001 From: Shira Sassoon Date: Tue, 10 Feb 2026 14:44:21 +0200 Subject: [PATCH 06/18] fix wording --- docs/how_to/config_deployment.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/how_to/config_deployment.md b/docs/how_to/config_deployment.md index 9d60f2ff..28d99191 100644 --- a/docs/how_to/config_deployment.md +++ b/docs/how_to/config_deployment.md @@ -33,7 +33,7 @@ Raise a [feature request](https://github.com/microsoft/fabric-cicd/issues/new?te The configuration file includes several sections with configurable settings for different aspects of the deployment process. -**Note**: Configuration values can be specified in two ways: as a single value (applied to any environment provided) or as an environment mapping. Both approaches can be used within the same configuration file - for example, using environment mappings for workspace IDs while keeping a single value for repository directory. +**Note**: Configuration values can be specified in two ways: as a single value (applied to any target environment provided) or as an environment mapping. Both approaches can be used within the same configuration file - for example, using environment mappings for workspace IDs while keeping a single value for repository directory. ### Core Settings @@ -115,7 +115,7 @@ core: `publish` is optional and can be used to control item publishing behavior. It includes various optional settings to enable/disable publishing operations or selectively publish items. -**Important:** To effectively use folder exclusion/inclusion, ensure the folder path contains a `/` preceding the folder name — for example, `/folder_name` for a single folder or `/folder_name/nested_folder_name` for nested folders. For regex patterns, ensure the pattern matches folders with a `/` preceding the folder name. For folder inclusion lists, ensure the path exactly matches this format. +**Note:** To effectively apply selective folder publishing via `folder_exclude_regex` and/or `folder_path_to_include`, ensure that the provided folder path starts with a `/` — for example, `/folder_name` for a top-level folder or `/folder_name/nested_folder_name` for nested folders. For regex patterns, ensure the pattern accounts for the leading `/`. Paths without a leading `/` (e.g., `folder_name/`) will not match correctly. ```yaml publish: @@ -163,7 +163,7 @@ publish: - - : - - + - # Optional - specific items to publish (requires feature flags) items_to_include: From 73ba2016a93ea3b2d58ffd23a20d6b1029494c1c Mon Sep 17 00:00:00 2001 From: Shira Sassoon Date: Tue, 10 Feb 2026 15:17:06 +0200 Subject: [PATCH 07/18] fix bugs --- src/fabric_cicd/constants.py | 1 + tests/test_deploy_with_config.py | 5 +---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/fabric_cicd/constants.py b/src/fabric_cicd/constants.py index e8e855c8..5d7d54f9 100644 --- a/src/fabric_cicd/constants.py +++ b/src/fabric_cicd/constants.py @@ -327,6 +327,7 @@ class OperationType(str, Enum): "exclude_regex", "folder_exclude_regex", "folder_path_to_include", + "items_to_include", "shortcut_exclude_regex", "skip", ], diff --git a/tests/test_deploy_with_config.py b/tests/test_deploy_with_config.py index 5dcce730..86068841 100644 --- a/tests/test_deploy_with_config.py +++ b/tests/test_deploy_with_config.py @@ -648,7 +648,6 @@ def test_deploy_with_config_shortcut_exclude_regex(self, mock_unpublish, mock_pu @patch("fabric_cicd.publish.FabricWorkspace") @patch("fabric_cicd.publish.publish_all_items") @patch("fabric_cicd.publish.unpublish_all_orphan_items") - @patch("fabric_cicd.constants.FEATURE_FLAG", set(["enable_experimental_features", "enable_config_deploy"])) def test_folder_path_to_include_passed_to_publish(self, _mock_unpublish, mock_publish, mock_workspace, tmp_path): # noqa: PT019 """Test that folder_path_to_include from config is passed to publish_all_items.""" test_repo_dir = tmp_path / "repo" @@ -676,7 +675,6 @@ def test_folder_path_to_include_passed_to_publish(self, _mock_unpublish, mock_pu @patch("fabric_cicd.publish.FabricWorkspace") @patch("fabric_cicd.publish.publish_all_items") @patch("fabric_cicd.publish.unpublish_all_orphan_items") - @patch("fabric_cicd.constants.FEATURE_FLAG", set(["enable_experimental_features", "enable_config_deploy"])) def test_folder_path_to_include_defaults_to_none(self, _mock_unpublish, mock_publish, mock_workspace, tmp_path): # noqa: PT019 """Test that folder_path_to_include defaults to None when not specified.""" test_repo_dir = tmp_path / "repo" @@ -696,12 +694,11 @@ def test_folder_path_to_include_defaults_to_none(self, _mock_unpublish, mock_pub deploy_with_config(str(config_file), "dev") call_args = mock_publish.call_args[1] - assert "folder_path_to_include" not in call_args + assert call_args["folder_path_to_include"] is None @patch("fabric_cicd.publish.FabricWorkspace") @patch("fabric_cicd.publish.publish_all_items") @patch("fabric_cicd.publish.unpublish_all_orphan_items") - @patch("fabric_cicd.constants.FEATURE_FLAG", set(["enable_experimental_features", "enable_config_deploy"])) def test_folder_path_to_include_environment_specific(self, _mock_unpublish, mock_publish, mock_workspace, tmp_path): # noqa: PT019 """Test that folder_path_to_include resolves environment-specific values.""" test_repo_dir = tmp_path / "repo" From 1af7fd0fed6ef4efed685e9189534d8b60457787 Mon Sep 17 00:00:00 2001 From: Shira Sassoon Date: Tue, 10 Feb 2026 15:19:05 +0200 Subject: [PATCH 08/18] add change --- .changes/unreleased/added-20260210-151850.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .changes/unreleased/added-20260210-151850.yaml diff --git a/.changes/unreleased/added-20260210-151850.yaml b/.changes/unreleased/added-20260210-151850.yaml new file mode 100644 index 00000000..6b00d3cd --- /dev/null +++ b/.changes/unreleased/added-20260210-151850.yaml @@ -0,0 +1,8 @@ +kind: added +body: Support selective folder deployment using inclusion list +time: 2026-02-10T15:18:50.3316666+02:00 +custom: + Author: shirasassoon + AuthorLink: https://github.com/shirasassoon + Issue: "757" + IssueLink: https://github.com/microsoft/fabric-cicd/issues/757 From a19478409db3e6fcc884571a9c063533ef626cf4 Mon Sep 17 00:00:00 2001 From: Shira Sassoon Date: Wed, 11 Feb 2026 12:56:21 +0200 Subject: [PATCH 09/18] address feedback --- src/fabric_cicd/_common/_item.py | 1 + src/fabric_cicd/_common/_validate_input.py | 4 ++-- src/fabric_cicd/constants.py | 2 +- src/fabric_cicd/fabric_workspace.py | 20 +++++++++++-------- tests/test_config_validator.py | 3 +-- tests/test_publish.py | 23 ++++++++++++---------- 6 files changed, 30 insertions(+), 23 deletions(-) diff --git a/src/fabric_cicd/_common/_item.py b/src/fabric_cicd/_common/_item.py index f4caf427..9806d7a6 100644 --- a/src/fabric_cicd/_common/_item.py +++ b/src/fabric_cicd/_common/_item.py @@ -23,6 +23,7 @@ class Item: path: Path = field(default_factory=Path) item_files: list = field(default_factory=list) folder_id: str = field(default="") + folder_path: str = field(default="") IMMUTABLE_FIELDS: ClassVar[set] = {"type", "name", "description"} skip_publish: bool = field(default=False) diff --git a/src/fabric_cicd/_common/_validate_input.py b/src/fabric_cicd/_common/_validate_input.py index 86ccafbd..2d6dfdef 100644 --- a/src/fabric_cicd/_common/_validate_input.py +++ b/src/fabric_cicd/_common/_validate_input.py @@ -228,12 +228,12 @@ def validate_folder_path_exclude_regex(folder_path_exclude_regex: Optional[str]) ) -def validate_folder_path_to_include(folder_path_to_include: Optional[str]) -> None: +def validate_folder_path_to_include(folder_path_to_include: Optional[list[str]]) -> None: """ Validate folder_path_to_include parameter and check required feature flags. Args: - folder_path_to_include: List of folder paths (with items) to publish. + folder_path_to_include: List of folder paths with format ["/folder1", "/folder2", ...], or None. Raises: InputError: If required feature flags are not enabled. diff --git a/src/fabric_cicd/constants.py b/src/fabric_cicd/constants.py index dcf5b981..08e255fe 100644 --- a/src/fabric_cicd/constants.py +++ b/src/fabric_cicd/constants.py @@ -380,7 +380,7 @@ class OperationType(str, Enum): # Field validation "field": { "string_or_dict": "'{}' must be either a string or environment mapping dictionary (e.g., {{dev: 'dev_value', prod: 'prod_value'}}), got type {}", - "list_or_dict": "'{}' must be a list or environment mapping dictionary (e.g., {{dev: 'dev_value', prod: 'prod_value'}}), got type {}", + "list_or_dict": "'{}' must be a list or environment mapping dictionary (e.g., {{dev: ['dev_value1', 'dev_value2'], prod: ['prod_value']}}), got type {}", "empty_value": "'{}' cannot be empty", "empty_list": "'{}' cannot be empty if specified", "invalid_guid": "'{}' must be a valid GUID format: {}", diff --git a/src/fabric_cicd/fabric_workspace.py b/src/fabric_cicd/fabric_workspace.py index eb7e3911..7578953e 100644 --- a/src/fabric_cicd/fabric_workspace.py +++ b/src/fabric_cicd/fabric_workspace.py @@ -343,6 +343,7 @@ def _refresh_repository_items(self) -> None: logical_id=item_logical_id, path=item_path, folder_id=item_folder_id, + folder_path=relative_parent_path, ) self.repository_items[item_type][item_name].collect_item_files() @@ -594,11 +595,7 @@ def _publish_item( # Skip publishing if the folder path is excluded by the regex or not in the include list if self.publish_folder_path_exclude_regex or self.publish_folder_path_to_include: - relative_path = item.path.relative_to(Path(self.repository_directory)) - # Build the folder path in the same format as repository_folders keys (/folder_name) - relative_parts = relative_path.as_posix().split("/") - # Remove the last part (item folder name) to get the parent folder path - folder_path = "/" + "/".join(relative_parts[:-1]) if len(relative_parts) > 1 else "" + folder_path = item.folder_path or "" if folder_path: if self.publish_folder_path_exclude_regex: regex_pattern = check_regex(self.publish_folder_path_exclude_regex) @@ -846,9 +843,16 @@ def _publish_folders(self) -> None: logger.info("Publishing Workspace Folders") for folder_path in sorted_folders: # Skip folders not in the include list - if self.publish_folder_path_to_include and folder_path not in self.publish_folder_path_to_include: - logger.info(f"Skipping publishing of folder '{folder_path}' as it is not in the include list.") - continue + # Ancestor folders must be published to preserve the correct hierarchy + # (e.g., if /A/B is included, /A must also be published). + if self.publish_folder_path_to_include: + is_included = folder_path in self.publish_folder_path_to_include + is_ancestor_of_included = any( + included.startswith(folder_path + "/") for included in self.publish_folder_path_to_include + ) + if not is_included and not is_ancestor_of_included: + logger.info(f"Skipping publishing of folder '{folder_path}' as it is not in the include list.") + continue # Skip folders matching the exclusion regex if self.publish_folder_path_exclude_regex: regex_pattern = check_regex(self.publish_folder_path_exclude_regex) diff --git a/tests/test_config_validator.py b/tests/test_config_validator.py index 4a5d661c..2ecb466c 100644 --- a/tests/test_config_validator.py +++ b/tests/test_config_validator.py @@ -1696,7 +1696,7 @@ def test_validate_operation_section_items_to_include_invalid_type(self): assert len(self.validator.errors) == 1 assert ( - "'publish.items_to_include' must be a list or environment mapping dictionary (e.g., {dev: 'dev_value', prod: 'prod_value'}), got type str" + constants.CONFIG_VALIDATION_MSGS["field"]["list_or_dict"].format("publish.items_to_include", "str") in self.validator.errors[0] ) @@ -1872,7 +1872,6 @@ def test_validate_operation_section_folder_path_to_include_env_mapping_empty_lis self.validator._validate_operation_section(section, "publish") assert len(self.validator.errors) == 1 - print(self.validator.errors[0]) assert ( constants.CONFIG_VALIDATION_MSGS["environment"]["empty_env_value"].format( "publish.folder_path_to_include", "prod" diff --git a/tests/test_publish.py b/tests/test_publish.py index aee639f4..821e1469 100644 --- a/tests/test_publish.py +++ b/tests/test_publish.py @@ -743,7 +743,7 @@ def test_legacy_folder_exclusion_example(mock_endpoint): ) # Test: Exclude all items in 'legacy' folder using the folder path regex pattern - exclude_regex = r"^/legacy" # Match items that start with 'legacy/' + exclude_regex = r"^/legacy" # Match items that start with '/legacy' publish.publish_all_items(workspace, folder_path_exclude_regex=exclude_regex) # Verify that legacy items were excluded @@ -760,12 +760,14 @@ def test_legacy_folder_exclusion_example(mock_endpoint): def test_folder_inclusion_with_folder_path_to_include(mock_endpoint): - """Test that folder_path_to_include only publishes items in specified folders.""" + """Test that folder_path_to_include only filters items found within a Fabric folder. + Root-level items (not located within any subfolder) are always published, + as folder inclusion can only apply to items that reside inside a folder.""" with tempfile.TemporaryDirectory() as temp_dir: temp_path = Path(temp_dir) - # Create items in 'active' folder (should be included) + # Create items in 'active' folder (should be included via folder_path_to_include) active_notebook_dir = temp_path / "active" / "ActiveNotebook.Notebook" active_notebook_dir.mkdir(parents=True, exist_ok=True) @@ -804,7 +806,7 @@ def test_folder_inclusion_with_folder_path_to_include(mock_endpoint): with (active_model_dir / "dummy.txt").open("w", encoding="utf-8") as f: f.write("Dummy file content") - # Create items in 'archive' folder (should be excluded) + # Create items in 'archive' folder (should be excluded - folder not in inclusion list) archive_notebook_dir = temp_path / "archive" / "ArchivedNotebook.Notebook" archive_notebook_dir.mkdir(parents=True, exist_ok=True) @@ -824,7 +826,7 @@ def test_folder_inclusion_with_folder_path_to_include(mock_endpoint): with (archive_notebook_dir / "dummy.txt").open("w", encoding="utf-8") as f: f.write("Dummy file content") - # Create root-level item (should be excluded since not in included folder) + # Create root-level item (not inside any folder - always published) root_notebook_dir = temp_path / "RootNotebook.Notebook" root_notebook_dir.mkdir(parents=True, exist_ok=True) @@ -833,7 +835,7 @@ def test_folder_inclusion_with_folder_path_to_include(mock_endpoint): "metadata": { "type": "Notebook", "displayName": "RootNotebook", - "description": "Root level notebook to be excluded", + "description": "Root level notebook - always published regardless of folder inclusion", }, "config": {"logicalId": "root-notebook-id"}, } @@ -872,15 +874,16 @@ def test_folder_inclusion_with_folder_path_to_include(mock_endpoint): assert "Notebook" in workspace.repository_items assert "SemanticModel" in workspace.repository_items - # Check that active items were NOT marked for exclusion (skip_publish = False) + # Check that items in the included folder are published (skip_publish = False) assert workspace.repository_items["Notebook"]["ActiveNotebook"].skip_publish is False assert workspace.repository_items["SemanticModel"]["ActiveModel"].skip_publish is False - # Check that archive items were marked for exclusion (skip_publish = True) + # Check that items in a non-included folder are excluded (skip_publish = True) assert workspace.repository_items["Notebook"]["ArchivedNotebook"].skip_publish is True - # Root-level items have an empty folder_path so they are not evaluated - # against folder_path_to_include and remain publishable + # Root-level items are not located within any Fabric folder, so + # folder_path_to_include does not apply to them. They are always + # published regardless of the folder inclusion filter. assert workspace.repository_items["Notebook"]["RootNotebook"].skip_publish is False finally: From b7b4bc5ca7bf00772688bc0e81b735f206786148 Mon Sep 17 00:00:00 2001 From: Shira Sassoon Date: Wed, 11 Feb 2026 14:15:56 +0200 Subject: [PATCH 10/18] fix exclude regex behavior --- docs/how_to/config_deployment.md | 6 +- src/fabric_cicd/fabric_workspace.py | 33 ++++- tests/test_publish.py | 216 +++++++++++++++++++++++++++- 3 files changed, 245 insertions(+), 10 deletions(-) diff --git a/docs/how_to/config_deployment.md b/docs/how_to/config_deployment.md index 28d99191..c65b57c9 100644 --- a/docs/how_to/config_deployment.md +++ b/docs/how_to/config_deployment.md @@ -115,7 +115,11 @@ core: `publish` is optional and can be used to control item publishing behavior. It includes various optional settings to enable/disable publishing operations or selectively publish items. -**Note:** To effectively apply selective folder publishing via `folder_exclude_regex` and/or `folder_path_to_include`, ensure that the provided folder path starts with a `/` — for example, `/folder_name` for a top-level folder or `/folder_name/nested_folder_name` for nested folders. For regex patterns, ensure the pattern accounts for the leading `/`. Paths without a leading `/` (e.g., `folder_name/`) will not match correctly. +**Note:** Folder-level filtering only applies to items that reside within a Fabric folder (subfolder). To effectively apply selective folder publishing via `folder_exclude_regex` and/or `folder_path_to_include`, ensure that the provided folder path starts with a `/` — for example, `/folder_name` for a top-level folder or `/folder_name/nested_folder_name` for nested folders. For regex patterns, ensure the pattern accounts for the leading `/`. Paths without a leading `/` (e.g., `folder_name/`) will not match correctly. + +When using `folder_exclude_regex`, the pattern is matched using `search()` (substring match), so a pattern like `/subfolder1` will match both `/subfolder1` and `/subfolder1/subfolder2`. If you use anchored patterns (e.g., `^/subfolder1$`), only the exact folder will match the pattern directly — however, child folders like `/subfolder1/subfolder2` will also be excluded automatically since their parent folder was excluded, preserving a consistent folder hierarchy. + +When using `folder_path_to_include` with nested paths (e.g., `/subfolder1/subfolder2`), ancestor folders (e.g., `/subfolder1`) are automatically created to preserve the correct folder hierarchy, but items directly under the ancestor folder are **not** published unless the ancestor folder is also explicitly included in the list. ```yaml publish: diff --git a/src/fabric_cicd/fabric_workspace.py b/src/fabric_cicd/fabric_workspace.py index 7578953e..a54c8e12 100644 --- a/src/fabric_cicd/fabric_workspace.py +++ b/src/fabric_cicd/fabric_workspace.py @@ -599,12 +599,19 @@ def _publish_item( if folder_path: if self.publish_folder_path_exclude_regex: regex_pattern = check_regex(self.publish_folder_path_exclude_regex) - if regex_pattern.search(folder_path): - item.skip_publish = True - logger.info( - f"Skipping publishing of {item_type} '{item_name}' due to folder path exclusion regex." - ) - return + # Check if the folder path itself or any ancestor matches the exclusion regex + path_to_check = folder_path + while path_to_check: + if regex_pattern.search(path_to_check): + item.skip_publish = True + logger.info( + f"Skipping publishing of {item_type} '{item_name}' due to folder path exclusion regex." + ) + return + if "/" in path_to_check and path_to_check != "": + path_to_check = path_to_check.rsplit("/", 1)[0] + else: + break if self.publish_folder_path_to_include and folder_path not in self.publish_folder_path_to_include: item.skip_publish = True @@ -859,6 +866,20 @@ def _publish_folders(self) -> None: if regex_pattern.search(folder_path): logger.info(f"Skipping publishing of folder '{folder_path}' due to folder path exclusion regex.") continue + # If any ancestor folder was excluded by the regex, skip this + # descendant folder too to preserve a consistent hierarchy + ancestor_path = folder_path + ancestor_excluded = False + while "/" in ancestor_path and ancestor_path != "": + ancestor_path = ancestor_path.rsplit("/", 1)[0] + if ancestor_path and regex_pattern.search(ancestor_path): + ancestor_excluded = True + break + if ancestor_excluded: + logger.info( + f"Skipping publishing of folder '{folder_path}' because ancestor folder was excluded by regex." + ) + continue logger.debug(f"Folder path '{folder_path}' does not match the exclusion regex pattern.") if folder_path in self.deployed_folders: # Folder already deployed, update local hierarchy diff --git a/tests/test_publish.py b/tests/test_publish.py index 821e1469..cd5acac5 100644 --- a/tests/test_publish.py +++ b/tests/test_publish.py @@ -583,6 +583,113 @@ def test_folder_exclusion_with_regex(mock_endpoint): constants.FEATURE_FLAG.update(original_flags) +def test_folder_exclusion_with_anchored_regex(mock_endpoint): + """Test that excluding a parent folder with an anchored regex also excludes + items in child folders, preserving consistent hierarchy behavior.""" + + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + + # Create item directly under /legacy (should be excluded - direct match) + legacy_notebook_dir = temp_path / "legacy" / "LegacyNotebook.Notebook" + legacy_notebook_dir.mkdir(parents=True, exist_ok=True) + + legacy_notebook_platform = legacy_notebook_dir / ".platform" + legacy_notebook_metadata = { + "metadata": { + "type": "Notebook", + "displayName": "LegacyNotebook", + "description": "Legacy notebook in excluded parent folder", + }, + "config": {"logicalId": "legacy-notebook-id"}, + } + + with legacy_notebook_platform.open("w", encoding="utf-8") as f: + json.dump(legacy_notebook_metadata, f) + + with (legacy_notebook_dir / "dummy.txt").open("w", encoding="utf-8") as f: + f.write("Dummy file content") + + # Create item in /legacy/archived (should also be excluded - ancestor excluded) + archived_notebook_dir = temp_path / "legacy" / "archived" / "ArchivedNotebook.Notebook" + archived_notebook_dir.mkdir(parents=True, exist_ok=True) + + archived_notebook_platform = archived_notebook_dir / ".platform" + archived_notebook_metadata = { + "metadata": { + "type": "Notebook", + "displayName": "ArchivedNotebook", + "description": "Notebook in child folder of excluded parent - should also be excluded", + }, + "config": {"logicalId": "archived-notebook-id"}, + } + + with archived_notebook_platform.open("w", encoding="utf-8") as f: + json.dump(archived_notebook_metadata, f) + + with (archived_notebook_dir / "dummy.txt").open("w", encoding="utf-8") as f: + f.write("Dummy file content") + + # Create item in /current (should NOT be excluded) + current_notebook_dir = temp_path / "current" / "CurrentNotebook.Notebook" + current_notebook_dir.mkdir(parents=True, exist_ok=True) + + current_notebook_platform = current_notebook_dir / ".platform" + current_notebook_metadata = { + "metadata": { + "type": "Notebook", + "displayName": "CurrentNotebook", + "description": "Current notebook to be included", + }, + "config": {"logicalId": "current-notebook-id"}, + } + + with current_notebook_platform.open("w", encoding="utf-8") as f: + json.dump(current_notebook_metadata, f) + + with (current_notebook_dir / "dummy.txt").open("w", encoding="utf-8") as f: + f.write("Dummy file content") + + with ( + patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint), + patch.object( + FabricWorkspace, "_refresh_deployed_items", new=lambda self: setattr(self, "deployed_items", {}) + ), + patch.object( + FabricWorkspace, "_refresh_deployed_folders", new=lambda self: setattr(self, "deployed_folders", {}) + ), + ): + original_flags = constants.FEATURE_FLAG.copy() + constants.FEATURE_FLAG.add("enable_experimental_features") + constants.FEATURE_FLAG.add("enable_exclude_folder") + + try: + workspace = FabricWorkspace( + workspace_id="12345678-1234-5678-abcd-1234567890ab", + repository_directory=str(temp_path), + item_type_in_scope=["Notebook"], + ) + + # Use an anchored regex that only matches /legacy exactly, + # NOT /legacy/archived directly. The ancestor walk logic should + # still exclude /legacy/archived because its parent /legacy is excluded. + exclude_regex = r"^/legacy$" + publish.publish_all_items(workspace, folder_path_exclude_regex=exclude_regex) + + # Direct match: /legacy is excluded + assert workspace.repository_items["Notebook"]["LegacyNotebook"].skip_publish is True + + # Ancestor excluded: /legacy/archived is excluded because /legacy matches + assert workspace.repository_items["Notebook"]["ArchivedNotebook"].skip_publish is True + + # Unrelated folder: /current is NOT excluded + assert workspace.repository_items["Notebook"]["CurrentNotebook"].skip_publish is False + + finally: + constants.FEATURE_FLAG.clear() + constants.FEATURE_FLAG.update(original_flags) + + def test_item_name_exclusion_still_works(mock_endpoint): """Test that existing item name exclusion still works with the new folder exclusion feature.""" @@ -762,7 +869,10 @@ def test_legacy_folder_exclusion_example(mock_endpoint): def test_folder_inclusion_with_folder_path_to_include(mock_endpoint): """Test that folder_path_to_include only filters items found within a Fabric folder. Root-level items (not located within any subfolder) are always published, - as folder inclusion can only apply to items that reside inside a folder.""" + as folder inclusion can only apply to items that reside inside a folder. + Items in ancestor folders of included paths are excluded even though the + ancestor folder itself is created to preserve hierarchy. + When an ancestor folder is explicitly included, its items are also published.""" with tempfile.TemporaryDirectory() as temp_dir: temp_path = Path(temp_dir) @@ -846,6 +956,88 @@ def test_folder_inclusion_with_folder_path_to_include(mock_endpoint): with (root_notebook_dir / "dummy.txt").open("w", encoding="utf-8") as f: f.write("Dummy file content") + # Create nested folder structure: /projects/team1/NestedNotebook.Notebook + # Only /projects/team1 is in the include list (not /projects), so: + # - /projects folder is created (ancestor) but items directly under it are excluded + # - /projects/team1 items are included + projects_notebook_dir = temp_path / "projects" / "ProjectNotebook.Notebook" + projects_notebook_dir.mkdir(parents=True, exist_ok=True) + + projects_notebook_platform = projects_notebook_dir / ".platform" + projects_notebook_metadata = { + "metadata": { + "type": "Notebook", + "displayName": "ProjectNotebook", + "description": "Item directly under ancestor folder - should be excluded", + }, + "config": {"logicalId": "projects-notebook-id"}, + } + + with projects_notebook_platform.open("w", encoding="utf-8") as f: + json.dump(projects_notebook_metadata, f) + + with (projects_notebook_dir / "dummy.txt").open("w", encoding="utf-8") as f: + f.write("Dummy file content") + + nested_notebook_dir = temp_path / "projects" / "team1" / "NestedNotebook.Notebook" + nested_notebook_dir.mkdir(parents=True, exist_ok=True) + + nested_notebook_platform = nested_notebook_dir / ".platform" + nested_notebook_metadata = { + "metadata": { + "type": "Notebook", + "displayName": "NestedNotebook", + "description": "Notebook in nested included folder", + }, + "config": {"logicalId": "nested-notebook-id"}, + } + + with nested_notebook_platform.open("w", encoding="utf-8") as f: + json.dump(nested_notebook_metadata, f) + + with (nested_notebook_dir / "dummy.txt").open("w", encoding="utf-8") as f: + f.write("Dummy file content") + + # Create structure where ancestor IS in the include list: /dept/eng/EngNotebook.Notebook + # Both /dept and /dept/eng are in the include list, so items under both should be published + dept_notebook_dir = temp_path / "dept" / "DeptNotebook.Notebook" + dept_notebook_dir.mkdir(parents=True, exist_ok=True) + + dept_notebook_platform = dept_notebook_dir / ".platform" + dept_notebook_metadata = { + "metadata": { + "type": "Notebook", + "displayName": "DeptNotebook", + "description": "Item under explicitly included ancestor folder - should be published", + }, + "config": {"logicalId": "dept-notebook-id"}, + } + + with dept_notebook_platform.open("w", encoding="utf-8") as f: + json.dump(dept_notebook_metadata, f) + + with (dept_notebook_dir / "dummy.txt").open("w", encoding="utf-8") as f: + f.write("Dummy file content") + + eng_notebook_dir = temp_path / "dept" / "eng" / "EngNotebook.Notebook" + eng_notebook_dir.mkdir(parents=True, exist_ok=True) + + eng_notebook_platform = eng_notebook_dir / ".platform" + eng_notebook_metadata = { + "metadata": { + "type": "Notebook", + "displayName": "EngNotebook", + "description": "Item in nested folder under explicitly included ancestor", + }, + "config": {"logicalId": "eng-notebook-id"}, + } + + with eng_notebook_platform.open("w", encoding="utf-8") as f: + json.dump(eng_notebook_metadata, f) + + with (eng_notebook_dir / "dummy.txt").open("w", encoding="utf-8") as f: + f.write("Dummy file content") + with ( patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint), patch.object( @@ -867,8 +1059,11 @@ def test_folder_inclusion_with_folder_path_to_include(mock_endpoint): item_type_in_scope=["Notebook", "SemanticModel"], ) - # Test: Only include items in 'active' folder - publish.publish_all_items(workspace, folder_path_to_include=["/active"]) + # Test: Include items in 'active', 'projects/team1', 'dept', and 'dept/eng' folders + publish.publish_all_items( + workspace, + folder_path_to_include=["/active", "/projects/team1", "/dept", "/dept/eng"], + ) # Verify that repository_items are populated correctly assert "Notebook" in workspace.repository_items @@ -886,6 +1081,21 @@ def test_folder_inclusion_with_folder_path_to_include(mock_endpoint): # published regardless of the folder inclusion filter. assert workspace.repository_items["Notebook"]["RootNotebook"].skip_publish is False + # Nested folder: items in the included nested path are published + assert workspace.repository_items["Notebook"]["NestedNotebook"].skip_publish is False + + # Ancestor folder: /projects is NOT in the include list (only /projects/team1 is), + # so items directly under /projects are excluded + assert workspace.repository_items["Notebook"]["ProjectNotebook"].skip_publish is True + + # Explicitly included ancestor: /dept IS in the include list, + # so items directly under /dept are published + assert workspace.repository_items["Notebook"]["DeptNotebook"].skip_publish is False + + # Nested folder under explicitly included ancestor: /dept/eng is also + # in the include list, so its items are published too + assert workspace.repository_items["Notebook"]["EngNotebook"].skip_publish is False + finally: # Restore original feature flags constants.FEATURE_FLAG.clear() From 66ef285f5ec271dba55e9d6dee4477e450a16f37 Mon Sep 17 00:00:00 2001 From: Shira Sassoon Date: Sun, 15 Feb 2026 14:32:57 +0200 Subject: [PATCH 11/18] Fix logic based on testing --- src/fabric_cicd/_common/_validate_input.py | 16 +++ src/fabric_cicd/fabric_workspace.py | 22 +-- src/fabric_cicd/publish.py | 10 +- tests/test_publish.py | 147 +++++++++++++++++++++ 4 files changed, 182 insertions(+), 13 deletions(-) diff --git a/src/fabric_cicd/_common/_validate_input.py b/src/fabric_cicd/_common/_validate_input.py index 2d6dfdef..ae5d68a3 100644 --- a/src/fabric_cicd/_common/_validate_input.py +++ b/src/fabric_cicd/_common/_validate_input.py @@ -227,6 +227,14 @@ def validate_folder_path_exclude_regex(folder_path_exclude_regex: Optional[str]) risk_warning="Using folder_path_exclude_regex is risky as it can prevent needed dependencies from being deployed. Use at your own risk.", ) + if not isinstance(folder_path_exclude_regex, str): + msg = "folder_path_exclude_regex must be a string." + raise InputError(msg, logger) + + if folder_path_exclude_regex == "": + msg = "folder_path_exclude_regex must not be an empty string. Provide a valid regex pattern or omit the parameter." + raise InputError(msg, logger) + def validate_folder_path_to_include(folder_path_to_include: Optional[list[str]]) -> None: """ @@ -245,6 +253,14 @@ def validate_folder_path_to_include(folder_path_to_include: Optional[list[str]]) risk_warning="Using folder_path_to_include is risky as it can prevent needed dependencies from being deployed. Use at your own risk.", ) + if not isinstance(folder_path_to_include, list): + msg = "folder_path_to_include must be a list of folder paths." + raise InputError(msg, logger) + + if not folder_path_to_include: + msg = "folder_path_to_include must not be an empty list. Provide folder paths or omit the parameter." + raise InputError(msg, logger) + def validate_shortcut_exclude_regex(shortcut_exclude_regex: Optional[str]) -> None: """ diff --git a/src/fabric_cicd/fabric_workspace.py b/src/fabric_cicd/fabric_workspace.py index a54c8e12..8cad9703 100644 --- a/src/fabric_cicd/fabric_workspace.py +++ b/src/fabric_cicd/fabric_workspace.py @@ -849,17 +849,6 @@ def _publish_folders(self) -> None: log_header(logger, "Publishing Workspace Folders") logger.info("Publishing Workspace Folders") for folder_path in sorted_folders: - # Skip folders not in the include list - # Ancestor folders must be published to preserve the correct hierarchy - # (e.g., if /A/B is included, /A must also be published). - if self.publish_folder_path_to_include: - is_included = folder_path in self.publish_folder_path_to_include - is_ancestor_of_included = any( - included.startswith(folder_path + "/") for included in self.publish_folder_path_to_include - ) - if not is_included and not is_ancestor_of_included: - logger.info(f"Skipping publishing of folder '{folder_path}' as it is not in the include list.") - continue # Skip folders matching the exclusion regex if self.publish_folder_path_exclude_regex: regex_pattern = check_regex(self.publish_folder_path_exclude_regex) @@ -881,6 +870,17 @@ def _publish_folders(self) -> None: ) continue logger.debug(f"Folder path '{folder_path}' does not match the exclusion regex pattern.") + # Skip folders not in the include list + # Ancestor folders must be published to preserve the correct hierarchy + # (e.g., if /A/B is included, /A must also be published). + if self.publish_folder_path_to_include: + is_included = folder_path in self.publish_folder_path_to_include + is_ancestor_of_included = any( + included.startswith(folder_path + "/") for included in self.publish_folder_path_to_include + ) + if not is_included and not is_ancestor_of_included: + logger.info(f"Skipping publishing of folder '{folder_path}' as it is not in the include list.") + continue if folder_path in self.deployed_folders: # Folder already deployed, update local hierarchy self.repository_folders[folder_path] = self.deployed_folders[folder_path] diff --git a/src/fabric_cicd/publish.py b/src/fabric_cicd/publish.py index 5cd6d589..8a75e897 100644 --- a/src/fabric_cicd/publish.py +++ b/src/fabric_cicd/publish.py @@ -181,14 +181,20 @@ def publish_all_items( raise FailedPublishedItemStatusError(msg, logger) if FeatureFlag.DISABLE_WORKSPACE_FOLDER_PUBLISH.value not in constants.FEATURE_FLAG: - if folder_path_exclude_regex: + if folder_path_exclude_regex is not None: validate_folder_path_exclude_regex(folder_path_exclude_regex) fabric_workspace_obj.publish_folder_path_exclude_regex = folder_path_exclude_regex - if folder_path_to_include: + if folder_path_to_include is not None: validate_folder_path_to_include(folder_path_to_include) fabric_workspace_obj.publish_folder_path_to_include = folder_path_to_include + if folder_path_exclude_regex is not None and folder_path_to_include is not None: + logger.warning( + "Both folder_path_exclude_regex and folder_path_to_include are defined. " + "Folder exclusion will be applied first, followed by inclusion filtering." + ) + fabric_workspace_obj._refresh_deployed_folders() fabric_workspace_obj._refresh_repository_folders() fabric_workspace_obj._publish_folders() diff --git a/tests/test_publish.py b/tests/test_publish.py index cd5acac5..187b4677 100644 --- a/tests/test_publish.py +++ b/tests/test_publish.py @@ -1100,3 +1100,150 @@ def test_folder_inclusion_with_folder_path_to_include(mock_endpoint): # Restore original feature flags constants.FEATURE_FLAG.clear() constants.FEATURE_FLAG.update(original_flags) + + +def test_folder_inclusion_and_exclusion_together(mock_endpoint, caplog): + """Test that both folder_path_to_include and folder_path_exclude_regex can be used together. + Exclusion is applied first, followed by inclusion filtering. A warning is logged.""" + import logging + + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + + # Create item in /deploy (included, not excluded) + deploy_notebook_dir = temp_path / "deploy" / "DeployNotebook.Notebook" + deploy_notebook_dir.mkdir(parents=True, exist_ok=True) + + deploy_platform = deploy_notebook_dir / ".platform" + deploy_metadata = { + "metadata": { + "type": "Notebook", + "displayName": "DeployNotebook", + "description": "Notebook in included folder", + }, + "config": {"logicalId": "deploy-notebook-id"}, + } + + with deploy_platform.open("w", encoding="utf-8") as f: + json.dump(deploy_metadata, f) + + with (deploy_notebook_dir / "dummy.txt").open("w", encoding="utf-8") as f: + f.write("Dummy file content") + + # Create item in /deploy/legacy (included but excluded by regex) + legacy_notebook_dir = temp_path / "deploy" / "legacy" / "LegacyNotebook.Notebook" + legacy_notebook_dir.mkdir(parents=True, exist_ok=True) + + legacy_platform = legacy_notebook_dir / ".platform" + legacy_metadata = { + "metadata": { + "type": "Notebook", + "displayName": "LegacyNotebook", + "description": "Notebook in excluded subfolder", + }, + "config": {"logicalId": "legacy-notebook-id"}, + } + + with legacy_platform.open("w", encoding="utf-8") as f: + json.dump(legacy_metadata, f) + + with (legacy_notebook_dir / "dummy.txt").open("w", encoding="utf-8") as f: + f.write("Dummy file content") + + # Create item in /archive (not included, not excluded) + archive_notebook_dir = temp_path / "archive" / "ArchiveNotebook.Notebook" + archive_notebook_dir.mkdir(parents=True, exist_ok=True) + + archive_platform = archive_notebook_dir / ".platform" + archive_metadata = { + "metadata": { + "type": "Notebook", + "displayName": "ArchiveNotebook", + "description": "Notebook in non-included folder", + }, + "config": {"logicalId": "archive-notebook-id"}, + } + + with archive_platform.open("w", encoding="utf-8") as f: + json.dump(archive_metadata, f) + + with (archive_notebook_dir / "dummy.txt").open("w", encoding="utf-8") as f: + f.write("Dummy file content") + + with ( + patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint), + patch.object( + FabricWorkspace, "_refresh_deployed_items", new=lambda self: setattr(self, "deployed_items", {}) + ), + patch.object( + FabricWorkspace, "_refresh_deployed_folders", new=lambda self: setattr(self, "deployed_folders", {}) + ), + caplog.at_level(logging.WARNING), + ): + original_flags = constants.FEATURE_FLAG.copy() + constants.FEATURE_FLAG.add("enable_experimental_features") + constants.FEATURE_FLAG.add("enable_include_folder") + constants.FEATURE_FLAG.add("enable_exclude_folder") + + try: + workspace = FabricWorkspace( + workspace_id="12345678-1234-5678-abcd-1234567890ab", + repository_directory=str(temp_path), + item_type_in_scope=["Notebook"], + ) + + publish.publish_all_items( + workspace, + folder_path_to_include=["/deploy", "/deploy/legacy"], + folder_path_exclude_regex=r"^/deploy/legacy", + ) + + # Verify warning was logged about both being defined + assert "Both folder_path_exclude_regex and folder_path_to_include are defined" in caplog.text + + # /deploy item is included and not excluded -> published + assert workspace.repository_items["Notebook"]["DeployNotebook"].skip_publish is False + + # /deploy/legacy item is excluded by regex (exclusion applied first) -> skipped + assert workspace.repository_items["Notebook"]["LegacyNotebook"].skip_publish is True + + # /archive item is not in the include list -> skipped + assert workspace.repository_items["Notebook"]["ArchiveNotebook"].skip_publish is True + + finally: + constants.FEATURE_FLAG.clear() + constants.FEATURE_FLAG.update(original_flags) + + +def test_empty_folder_path_to_include_raises_error(mock_endpoint): + """Test that passing an empty list for folder_path_to_include raises an InputError.""" + + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + + with ( + patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint), + patch.object( + FabricWorkspace, "_refresh_deployed_items", new=lambda self: setattr(self, "deployed_items", {}) + ), + patch.object( + FabricWorkspace, "_refresh_deployed_folders", new=lambda self: setattr(self, "deployed_folders", {}) + ), + ): + original_flags = constants.FEATURE_FLAG.copy() + constants.FEATURE_FLAG.add("enable_experimental_features") + constants.FEATURE_FLAG.add("enable_include_folder") + + try: + workspace = FabricWorkspace( + workspace_id="12345678-1234-5678-abcd-1234567890ab", + repository_directory=str(temp_path), + item_type_in_scope=["Notebook"], + ) + + with pytest.raises(InputError, match="folder_path_to_include must not be an empty list"): + publish.publish_all_items(workspace, folder_path_to_include=[]) + + finally: + constants.FEATURE_FLAG.clear() + constants.FEATURE_FLAG.update(original_flags) From 4203a0c7dc54331421498753f802a8d0fa35d4e7 Mon Sep 17 00:00:00 2001 From: Shira Sassoon Date: Sun, 15 Feb 2026 15:20:48 +0200 Subject: [PATCH 12/18] fix constants --- src/fabric_cicd/_common/_config_validator.py | 28 ++++++++++---------- src/fabric_cicd/constants.py | 8 +++--- tests/test_config_validator.py | 28 ++++++++++++++------ 3 files changed, 38 insertions(+), 26 deletions(-) diff --git a/src/fabric_cicd/_common/_config_validator.py b/src/fabric_cicd/_common/_config_validator.py index 7b6af830..ed7f099b 100644 --- a/src/fabric_cicd/_common/_config_validator.py +++ b/src/fabric_cicd/_common/_config_validator.py @@ -694,10 +694,12 @@ def _validate_operation_section(self, section: dict[str, Any], section_name: str if isinstance(exclude_regex, str): if not exclude_regex.strip(): self.errors.append( - constants.CONFIG_VALIDATION_MSGS["field"]["empty_value"].format(f"{section_name}.exclude_regex") + constants.CONFIG_VALIDATION_MSGS["operation"]["empty_string"].format( + f"{section_name}.exclude_regex" + ) ) else: - self._validate_regex(exclude_regex, section_name) + self._validate_regex(exclude_regex, f"{section_name}.exclude_regex") elif isinstance(exclude_regex, dict): # Validate environment mapping @@ -722,7 +724,7 @@ def _validate_operation_section(self, section: dict[str, Any], section_name: str if isinstance(items, list): if not items: self.errors.append( - constants.CONFIG_VALIDATION_MSGS["field"]["empty_list"].format( + constants.CONFIG_VALIDATION_MSGS["operation"]["empty_list"].format( f"{section_name}.items_to_include" ) ) @@ -758,7 +760,7 @@ def _validate_operation_section(self, section: dict[str, Any], section_name: str if isinstance(folder_exclude_regex, str): if not folder_exclude_regex.strip(): self.errors.append( - constants.CONFIG_VALIDATION_MSGS["field"]["empty_value"].format( + constants.CONFIG_VALIDATION_MSGS["operation"]["empty_string"].format( f"{section_name}.folder_exclude_regex" ) ) @@ -796,7 +798,7 @@ def _validate_operation_section(self, section: dict[str, Any], section_name: str if isinstance(folders, list): if not folders: self.errors.append( - constants.CONFIG_VALIDATION_MSGS["field"]["empty_list"].format( + constants.CONFIG_VALIDATION_MSGS["operation"]["empty_list"].format( f"{section_name}.folder_path_to_include" ) ) @@ -830,7 +832,7 @@ def _validate_operation_section(self, section: dict[str, Any], section_name: str if isinstance(shortcut_exclude_regex, str): if not shortcut_exclude_regex.strip(): self.errors.append( - constants.CONFIG_VALIDATION_MSGS["field"]["empty_value"].format( + constants.CONFIG_VALIDATION_MSGS["operation"]["empty_string"].format( f"{section_name}.shortcut_exclude_regex" ) ) @@ -889,26 +891,24 @@ def _validate_items_list(self, items_list: list, context: str) -> None: for i, item in enumerate(items_list): if not isinstance(item, str): self.errors.append( - constants.CONFIG_VALIDATION_MSGS["operation"]["items_list_type"].format( + constants.CONFIG_VALIDATION_MSGS["operation"]["list_entry_type"].format( context, i, type(item).__name__ ) ) elif not item.strip(): - self.errors.append(constants.CONFIG_VALIDATION_MSGS["operation"]["items_list_empty"].format(context, i)) + self.errors.append(constants.CONFIG_VALIDATION_MSGS["operation"]["list_entry_empty"].format(context, i)) def _validate_folders_list(self, folders_list: list, context: str) -> None: """Validate a list of folder paths with proper context for error messages.""" for i, folder in enumerate(folders_list): if not isinstance(folder, str): self.errors.append( - constants.CONFIG_VALIDATION_MSGS["operation"]["folders_list_type"].format( + constants.CONFIG_VALIDATION_MSGS["operation"]["list_entry_type"].format( context, i, type(folder).__name__ ) ) elif not folder.strip(): - self.errors.append( - constants.CONFIG_VALIDATION_MSGS["operation"]["folders_list_empty"].format(context, i) - ) + self.errors.append(constants.CONFIG_VALIDATION_MSGS["operation"]["list_entry_empty"].format(context, i)) elif not folder.startswith("/"): self.errors.append( constants.CONFIG_VALIDATION_MSGS["operation"]["folders_list_prefix"].format(context, i, folder) @@ -948,12 +948,12 @@ def _validate_features_list(self, features_list: list, context: str) -> None: for i, feature in enumerate(features_list): if not isinstance(feature, str): self.errors.append( - constants.CONFIG_VALIDATION_MSGS["operation"]["items_list_type"].format( + constants.CONFIG_VALIDATION_MSGS["operation"]["list_entry_type"].format( context, i, type(feature).__name__ ) ) elif not feature.strip(): - self.errors.append(constants.CONFIG_VALIDATION_MSGS["operation"]["items_list_empty"].format(context, i)) + self.errors.append(constants.CONFIG_VALIDATION_MSGS["operation"]["list_entry_empty"].format(context, i)) def _validate_constants_section(self, constants_section: any) -> None: """Validate constants section.""" diff --git a/src/fabric_cicd/constants.py b/src/fabric_cicd/constants.py index 306d0293..1ad866eb 100644 --- a/src/fabric_cicd/constants.py +++ b/src/fabric_cicd/constants.py @@ -406,15 +406,15 @@ class OperationType(str, Enum): "unsupported_field": "'{}' field is not supported in '{}' section", "not_dict": "'{}' section must be a dictionary, got {}", "invalid_regex": "'{}' in {} is not a valid regex pattern: {}", - "items_list_type": "'{}[{}]' must be a string, got {}", - "items_list_empty": "'{}[{}]' cannot be empty", + "empty_string": "'{}' cannot be an empty string", + "empty_list": "'{}' cannot be an empty list", + "list_entry_type": "'{}[{}]' must be a string, got {}", + "list_entry_empty": "'{}[{}]' cannot be an empty string", "features_type": "'features' section must be either a list or environment mapping dictionary, got {}", "empty_section": "'{}' section cannot be empty if specified", "empty_section_env": "'{}.{}' cannot be empty if specified", "invalid_constant_key": "Constant key in '{}' must be a non-empty string, got: {}", "unknown_constant": "Unknown constant '{}' in '{}' - this constant does not exist in fabric_cicd.constants", - "folders_list_type": "'{}[{}]' must be a string, got {}", - "folders_list_empty": "'{}[{}]' cannot be empty", "folders_list_prefix": "'{}[{}]' entry must start with '/' (got '{}')", }, # Log messages diff --git a/tests/test_config_validator.py b/tests/test_config_validator.py index 2ecb466c..03c82713 100644 --- a/tests/test_config_validator.py +++ b/tests/test_config_validator.py @@ -431,7 +431,10 @@ def test_validate_items_list_empty_item(self): self.validator._validate_items_list(items_list, "test_context") assert len(self.validator.errors) == 1 - assert "'test_context[1]' cannot be empty" in self.validator.errors[0] + assert ( + constants.CONFIG_VALIDATION_MSGS["operation"]["list_entry_empty"].format("test_context", 1) + in self.validator.errors[0] + ) def test_validate_features_list_valid(self): """Test _validate_features_list with valid features.""" @@ -457,7 +460,10 @@ def test_validate_features_list_empty_feature(self): self.validator._validate_features_list(features_list, "test_context") assert len(self.validator.errors) == 1 - assert "'test_context[1]' cannot be empty" in self.validator.errors[0] + assert ( + constants.CONFIG_VALIDATION_MSGS["operation"]["list_entry_empty"].format("test_context", 1) + in self.validator.errors[0] + ) def test_validate_constants_dict_valid(self): """Test _validate_constants_dict with valid constants.""" @@ -1643,7 +1649,10 @@ def test_validate_operation_section_empty_exclude_regex(self): self.validator._validate_operation_section(section, "publish") assert len(self.validator.errors) == 1 - assert "'publish.exclude_regex' cannot be empty" in self.validator.errors[0] + assert ( + constants.CONFIG_VALIDATION_MSGS["operation"]["empty_string"].format("publish.exclude_regex") + in self.validator.errors[0] + ) def test_validate_operation_section_invalid_regex(self): """Test _validate_operation_section with invalid regex.""" @@ -1678,7 +1687,10 @@ def test_validate_operation_section_empty_items_to_include(self): self.validator._validate_operation_section(section, "publish") assert len(self.validator.errors) == 1 - assert "'publish.items_to_include' cannot be empty if specified" in self.validator.errors[0] + assert ( + constants.CONFIG_VALIDATION_MSGS["operation"]["empty_list"].format("publish.items_to_include") + in self.validator.errors[0] + ) def test_validate_operation_section_items_to_include_environment_mapping(self): """Test _validate_operation_section with items_to_include environment mapping.""" @@ -1793,7 +1805,7 @@ def test_validate_operation_section_folder_path_to_include_empty_list(self): assert len(self.validator.errors) == 1 assert ( - constants.CONFIG_VALIDATION_MSGS["field"]["empty_list"].format("publish.folder_path_to_include") + constants.CONFIG_VALIDATION_MSGS["operation"]["empty_list"].format("publish.folder_path_to_include") in self.validator.errors[0] ) @@ -1831,7 +1843,7 @@ def test_validate_operation_section_folder_path_to_include_entry_not_string(self assert len(self.validator.errors) == 1 assert ( - constants.CONFIG_VALIDATION_MSGS["operation"]["folders_list_type"].format( + constants.CONFIG_VALIDATION_MSGS["operation"]["list_entry_type"].format( "publish.folder_path_to_include", 1, "int" ) in self.validator.errors[0] @@ -1845,7 +1857,7 @@ def test_validate_operation_section_folder_path_to_include_empty_string_entry(se assert len(self.validator.errors) == 1 assert ( - constants.CONFIG_VALIDATION_MSGS["operation"]["folders_list_empty"].format( + constants.CONFIG_VALIDATION_MSGS["operation"]["list_entry_empty"].format( "publish.folder_path_to_include", 1 ) in self.validator.errors[0] @@ -1909,7 +1921,7 @@ def test_validate_operation_section_folder_path_to_include_whitespace_entry(self assert len(self.validator.errors) == 1 assert ( - constants.CONFIG_VALIDATION_MSGS["operation"]["folders_list_empty"].format( + constants.CONFIG_VALIDATION_MSGS["operation"]["list_entry_empty"].format( "publish.folder_path_to_include", 1 ) in self.validator.errors[0] From 5ee827fef1fb007fab97fb428efbfa720b3f82c2 Mon Sep 17 00:00:00 2001 From: Shira Sassoon Date: Sun, 15 Feb 2026 16:27:50 +0200 Subject: [PATCH 13/18] add info --- docs/how_to/config_deployment.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/how_to/config_deployment.md b/docs/how_to/config_deployment.md index c65b57c9..678ebc99 100644 --- a/docs/how_to/config_deployment.md +++ b/docs/how_to/config_deployment.md @@ -115,7 +115,7 @@ core: `publish` is optional and can be used to control item publishing behavior. It includes various optional settings to enable/disable publishing operations or selectively publish items. -**Note:** Folder-level filtering only applies to items that reside within a Fabric folder (subfolder). To effectively apply selective folder publishing via `folder_exclude_regex` and/or `folder_path_to_include`, ensure that the provided folder path starts with a `/` — for example, `/folder_name` for a top-level folder or `/folder_name/nested_folder_name` for nested folders. For regex patterns, ensure the pattern accounts for the leading `/`. Paths without a leading `/` (e.g., `folder_name/`) will not match correctly. +**Note:** Folder-level filtering only applies to items that reside within a Fabric folder (subfolder). To effectively apply selective folder publishing via `folder_exclude_regex` and/or `folder_path_to_include`, ensure that the provided folder path starts with a `/` — for example, `/folder_name` for a top-level folder or `/folder_name/nested_folder_name` for nested folders. For regex patterns, ensure the pattern accounts for the leading `/`. Paths without a leading `/` (e.g., `folder_name/`) will not match correctly. If both parameters are provided, `folder_exclude_regex` is applied first. When using `folder_exclude_regex`, the pattern is matched using `search()` (substring match), so a pattern like `/subfolder1` will match both `/subfolder1` and `/subfolder1/subfolder2`. If you use anchored patterns (e.g., `^/subfolder1$`), only the exact folder will match the pattern directly — however, child folders like `/subfolder1/subfolder2` will also be excluded automatically since their parent folder was excluded, preserving a consistent folder hierarchy. From 8d57aa6bf49f02d5b073206050bc0dbe01efcc12 Mon Sep 17 00:00:00 2001 From: Shira Sassoon Date: Sun, 15 Feb 2026 16:47:57 +0200 Subject: [PATCH 14/18] clarify parameter --- src/fabric_cicd/publish.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fabric_cicd/publish.py b/src/fabric_cicd/publish.py index 8a75e897..99526d09 100644 --- a/src/fabric_cicd/publish.py +++ b/src/fabric_cicd/publish.py @@ -48,8 +48,8 @@ def publish_all_items( Args: fabric_workspace_obj: The FabricWorkspace object containing the items to be published. item_name_exclude_regex: Regex pattern to exclude specific items from being published. - folder_path_exclude_regex: Regex pattern to exclude items based on their folder path. - folder_path_to_include: List of folder paths (with items) that should be published. + folder_path_exclude_regex: Regex pattern matched against folder paths (e.g., "/folder_name") to exclude folders and their contents from being published. + folder_path_to_include: List of folder paths in the format "/folder_path"; only the specified folders and their contents will be published. items_to_include: List of items in the format "item_name.item_type" that should be published. shortcut_exclude_regex: Regex pattern to exclude specific shortcuts from being published in lakehouses. From c68241e412649f94c6f6365e6e98df53be3329b5 Mon Sep 17 00:00:00 2001 From: shirasassoon <66449905+shirasassoon@users.noreply.github.com> Date: Sun, 15 Feb 2026 16:50:02 +0200 Subject: [PATCH 15/18] Update publish.py --- src/fabric_cicd/publish.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fabric_cicd/publish.py b/src/fabric_cicd/publish.py index 99526d09..26dcdc88 100644 --- a/src/fabric_cicd/publish.py +++ b/src/fabric_cicd/publish.py @@ -49,7 +49,7 @@ def publish_all_items( fabric_workspace_obj: The FabricWorkspace object containing the items to be published. item_name_exclude_regex: Regex pattern to exclude specific items from being published. folder_path_exclude_regex: Regex pattern matched against folder paths (e.g., "/folder_name") to exclude folders and their contents from being published. - folder_path_to_include: List of folder paths in the format "/folder_path"; only the specified folders and their contents will be published. + folder_path_to_include: List of folder paths in the format "/folder_name"; only the specified folders and their contents will be published. items_to_include: List of items in the format "item_name.item_type" that should be published. shortcut_exclude_regex: Regex pattern to exclude specific shortcuts from being published in lakehouses. From 043ba0a2231e6b7d0baceb8ca6136e1ec58a252b Mon Sep 17 00:00:00 2001 From: Shira Sassoon Date: Mon, 16 Feb 2026 15:57:57 +0200 Subject: [PATCH 16/18] update logic --- docs/how_to/config_deployment.md | 13 ++- src/fabric_cicd/_common/_config_validator.py | 43 +++++++ src/fabric_cicd/constants.py | 2 + src/fabric_cicd/publish.py | 22 ++-- tests/test_config_validator.py | 111 +++++++++++++++++++ tests/test_publish.py | 75 ++----------- 6 files changed, 187 insertions(+), 79 deletions(-) diff --git a/docs/how_to/config_deployment.md b/docs/how_to/config_deployment.md index 678ebc99..9bacfb4f 100644 --- a/docs/how_to/config_deployment.md +++ b/docs/how_to/config_deployment.md @@ -115,9 +115,9 @@ core: `publish` is optional and can be used to control item publishing behavior. It includes various optional settings to enable/disable publishing operations or selectively publish items. -**Note:** Folder-level filtering only applies to items that reside within a Fabric folder (subfolder). To effectively apply selective folder publishing via `folder_exclude_regex` and/or `folder_path_to_include`, ensure that the provided folder path starts with a `/` — for example, `/folder_name` for a top-level folder or `/folder_name/nested_folder_name` for nested folders. For regex patterns, ensure the pattern accounts for the leading `/`. Paths without a leading `/` (e.g., `folder_name/`) will not match correctly. If both parameters are provided, `folder_exclude_regex` is applied first. +**Note:** Folder-level filtering only applies to items within a Fabric folder. Folder paths must start with `/` (e.g., `/folder_name` or `/folder_name/nested_folder`). `folder_exclude_regex` and `folder_path_to_include` are **mutually exclusive** — providing both for the same environment will result in a validation error. -When using `folder_exclude_regex`, the pattern is matched using `search()` (substring match), so a pattern like `/subfolder1` will match both `/subfolder1` and `/subfolder1/subfolder2`. If you use anchored patterns (e.g., `^/subfolder1$`), only the exact folder will match the pattern directly — however, child folders like `/subfolder1/subfolder2` will also be excluded automatically since their parent folder was excluded, preserving a consistent folder hierarchy. +When using `folder_exclude_regex`, the pattern is matched using `search()` (substring match), so a pattern like `subfolder1` will match any folder path containing "subfolder1" (e.g., `/subfolder1`, `/subfolder1/subfolder2`, `/other/subfolder1`). To target a specific folder, use an anchored pattern with a leading `/` (e.g., `^/subfolder1$`) — this ensures only the exact folder path matches directly. Note that child folders like `/subfolder1/subfolder2` will also be excluded automatically since their parent folder was excluded, preserving a consistent folder hierarchy. When using `folder_path_to_include` with nested paths (e.g., `/subfolder1/subfolder2`), ancestor folders (e.g., `/subfolder1`) are automatically created to preserve the correct folder hierarchy, but items directly under the ancestor folder are **not** published unless the ancestor folder is also explicitly included in the list. @@ -384,11 +384,14 @@ publish: # Don't publish items matching this pattern exclude_regex: "^DONT_DEPLOY.*" - folder_exclude_regex: "^/DONT_DEPLOY_FOLDER" + # Use folder_exclude_regex OR folder_path_to_include, not both for the same environment + folder_exclude_regex: + dev: "^/DONT_DEPLOY_FOLDER" folder_path_to_include: - - "/DEPLOY_FOLDER" - - "/DEPLOY_FOLDER/DEPLOY_NESTED_FOLDER" + prod: + - "/DEPLOY_FOLDER" + - "/DEPLOY_FOLDER/DEPLOY_NESTED_FOLDER" items_to_include: - "Hello World.Notebook" diff --git a/src/fabric_cicd/_common/_config_validator.py b/src/fabric_cicd/_common/_config_validator.py index ed7f099b..c705415a 100644 --- a/src/fabric_cicd/_common/_config_validator.py +++ b/src/fabric_cicd/_common/_config_validator.py @@ -877,6 +877,11 @@ def _validate_operation_section(self, section: dict[str, Any], section_name: str .replace("a string", "a boolean") ) + # Validate mutual exclusivity of folder filtering options + self._validate_mutually_exclusive_fields( + section, "folder_exclude_regex", "folder_path_to_include", section_name + ) + def _validate_regex(self, regex: str, section_name: str) -> None: """Validate regex value.""" try: @@ -914,6 +919,44 @@ def _validate_folders_list(self, folders_list: list, context: str) -> None: constants.CONFIG_VALIDATION_MSGS["operation"]["folders_list_prefix"].format(context, i, folder) ) + def _validate_mutually_exclusive_fields(self, section: dict, field1: str, field2: str, section_name: str) -> None: + """Validate that two fields are not both specified for the same environment.""" + if field1 not in section or field2 not in section: + return + + value1 = section[field1] + value2 = section[field2] + + # Both are direct values (not environment-specific), throw error + if not isinstance(value1, dict) and not isinstance(value2, dict): + self.errors.append( + constants.CONFIG_VALIDATION_MSGS["operation"]["mutually_exclusive"].format( + f"{section_name}.{field1}", f"{section_name}.{field2}" + ) + ) + return + + # Determine which environments each field contains (if they are environment mappings) + value1_envs = set(value1.keys()) if isinstance(value1, dict) else set() + value2_envs = set(value2.keys()) if isinstance(value2, dict) else set() + + # Determine if it is a direct value + value1_is_direct = not isinstance(value1, dict) + value2_is_direct = not isinstance(value2, dict) + + # Check if both fields would resolve for the target environment + value1_applies = value1_is_direct or self.environment in value1_envs + value2_applies = value2_is_direct or self.environment in value2_envs + + if value1_applies and value2_applies: + self.errors.append( + constants.CONFIG_VALIDATION_MSGS["operation"]["mutually_exclusive_env"].format( + f"{section_name}.{field1}", + f"{section_name}.{field2}", + [self.environment], + ) + ) + def _validate_features_section(self, features: any) -> None: """Validate features section.""" if isinstance(features, list): diff --git a/src/fabric_cicd/constants.py b/src/fabric_cicd/constants.py index 1ad866eb..6d0a2288 100644 --- a/src/fabric_cicd/constants.py +++ b/src/fabric_cicd/constants.py @@ -416,6 +416,8 @@ class OperationType(str, Enum): "invalid_constant_key": "Constant key in '{}' must be a non-empty string, got: {}", "unknown_constant": "Unknown constant '{}' in '{}' - this constant does not exist in fabric_cicd.constants", "folders_list_prefix": "'{}[{}]' entry must start with '/' (got '{}')", + "mutually_exclusive": "Cannot specify both '{}' and '{}'. Choose one filtering strategy.", + "mutually_exclusive_env": "Cannot specify both '{}' and '{}' for the same environment(s): {}. Choose one filtering strategy per environment.", }, # Log messages "log": { diff --git a/src/fabric_cicd/publish.py b/src/fabric_cicd/publish.py index 26dcdc88..83a64ec2 100644 --- a/src/fabric_cicd/publish.py +++ b/src/fabric_cicd/publish.py @@ -18,7 +18,7 @@ extract_workspace_settings, load_config_file, ) -from fabric_cicd._common._exceptions import FailedPublishedItemStatusError +from fabric_cicd._common._exceptions import FailedPublishedItemStatusError, InputError from fabric_cicd._common._logging import log_header from fabric_cicd._common._validate_input import ( validate_environment, @@ -58,13 +58,15 @@ def publish_all_items( folder_path_exclude_regex: This is an experimental feature in fabric-cicd. Use at your own risk as selective deployments are - not recommended due to item dependencies. To enable this feature, see How To -> Optional Features - for information on which flags to enable. + not recommended due to item dependencies. Cannot be used together with ``folder_path_to_include`` + for the same environment. To enable this feature, see How To -> Optional Features for information + on which flags to enable. folder_path_to_include: This is an experimental feature in fabric-cicd. Use at your own risk as selective deployments are - not recommended due to item dependencies. To enable this feature, see How To -> Optional Features - for information on which flags to enable. + not recommended due to item dependencies. Cannot be used together with ``folder_path_exclude_regex`` + for the same environment. To enable this feature, see How To -> Optional Features for information + on which flags to enable. items_to_include: This is an experimental feature in fabric-cicd. Use at your own risk as selective deployments are @@ -181,6 +183,10 @@ def publish_all_items( raise FailedPublishedItemStatusError(msg, logger) if FeatureFlag.DISABLE_WORKSPACE_FOLDER_PUBLISH.value not in constants.FEATURE_FLAG: + if folder_path_exclude_regex is not None and folder_path_to_include is not None: + msg = "Cannot use both 'folder_path_exclude_regex' and 'folder_path_to_include' simultaneously. Choose one filtering strategy." + raise InputError(msg, logger) + if folder_path_exclude_regex is not None: validate_folder_path_exclude_regex(folder_path_exclude_regex) fabric_workspace_obj.publish_folder_path_exclude_regex = folder_path_exclude_regex @@ -189,12 +195,6 @@ def publish_all_items( validate_folder_path_to_include(folder_path_to_include) fabric_workspace_obj.publish_folder_path_to_include = folder_path_to_include - if folder_path_exclude_regex is not None and folder_path_to_include is not None: - logger.warning( - "Both folder_path_exclude_regex and folder_path_to_include are defined. " - "Folder exclusion will be applied first, followed by inclusion filtering." - ) - fabric_workspace_obj._refresh_deployed_folders() fabric_workspace_obj._refresh_repository_folders() fabric_workspace_obj._publish_folders() diff --git a/tests/test_config_validator.py b/tests/test_config_validator.py index 03c82713..cd2d20d2 100644 --- a/tests/test_config_validator.py +++ b/tests/test_config_validator.py @@ -1952,6 +1952,117 @@ def test_validate_operation_section_with_shortcut_exclude_regex_environment_mapp assert self.validator.errors == [] + def test_validate_operation_section_mutually_exclusive_both_direct_values(self): + """Test that both folder_exclude_regex and folder_path_to_include as direct values raises error.""" + section = {"folder_exclude_regex": "^/legacy", "folder_path_to_include": ["/subfolder"]} + + self.validator._validate_operation_section(section, "publish") + + error_messages = " ".join(self.validator.errors) + assert ( + constants.CONFIG_VALIDATION_MSGS["operation"]["mutually_exclusive"].format( + "publish.folder_exclude_regex", "publish.folder_path_to_include" + ) + in error_messages + ) + + def test_validate_operation_section_mutually_exclusive_both_env_mapped_overlapping(self): + """Test that both fields with overlapping environment mappings raises error.""" + self.validator.environment = "dev" + section = { + "folder_exclude_regex": {"dev": "^/legacy"}, + "folder_path_to_include": {"dev": ["/subfolder"]}, + } + + self.validator._validate_operation_section(section, "publish") + + error_messages = " ".join(self.validator.errors) + assert ( + constants.CONFIG_VALIDATION_MSGS["operation"]["mutually_exclusive_env"].format( + "publish.folder_exclude_regex", "publish.folder_path_to_include", ["dev"] + ) + in error_messages + ) + + def test_validate_operation_section_mutually_exclusive_both_env_mapped_no_overlap(self): + """Test that both fields with non-overlapping environment mappings is valid.""" + self.validator.environment = "dev" + section = { + "folder_exclude_regex": {"dev": "^/legacy"}, + "folder_path_to_include": {"prod": ["/subfolder"]}, + } + + self.validator._validate_operation_section(section, "publish") + + # Filter errors to only mutual exclusivity errors + mutual_errors = [ + e for e in self.validator.errors if "mutually exclusive" in e.lower() or "Cannot specify both" in e + ] + assert len(mutual_errors) == 0 + + def test_validate_operation_section_mutually_exclusive_direct_and_env_mapped_conflict(self): + """Test that direct value + env-mapped value conflicts when target env matches.""" + self.validator.environment = "dev" + section = { + "folder_exclude_regex": "^/legacy", # direct, applies to all + "folder_path_to_include": {"dev": ["/subfolder"]}, # env-mapped, applies to dev + } + + self.validator._validate_operation_section(section, "publish") + + error_messages = " ".join(self.validator.errors) + assert ( + constants.CONFIG_VALIDATION_MSGS["operation"]["mutually_exclusive_env"].format( + "publish.folder_exclude_regex", "publish.folder_path_to_include", ["dev"] + ) + in error_messages + ) + + def test_validate_operation_section_mutually_exclusive_direct_and_env_mapped_no_conflict(self): + """Test that direct value + env-mapped value is valid when target env doesn't match.""" + self.validator.environment = "prod" + section = { + "folder_exclude_regex": "^/legacy", # direct, applies to all + "folder_path_to_include": {"dev": ["/subfolder"]}, # env-mapped, only dev + } + + self.validator._validate_operation_section(section, "publish") + + # The direct value applies to prod, but folder_path_to_include doesn't apply to prod + mutual_errors = [ + e for e in self.validator.errors if "mutually exclusive" in e.lower() or "Cannot specify both" in e + ] + assert len(mutual_errors) == 0 + + def test_validate_operation_section_mutually_exclusive_env_mapped_and_direct_conflict(self): + """Test that env-mapped value + direct value conflicts when target env matches.""" + self.validator.environment = "dev" + section = { + "folder_exclude_regex": {"dev": "^/legacy"}, # env-mapped, applies to dev + "folder_path_to_include": ["/subfolder"], # direct, applies to all + } + + self.validator._validate_operation_section(section, "publish") + + error_messages = " ".join(self.validator.errors) + assert ( + constants.CONFIG_VALIDATION_MSGS["operation"]["mutually_exclusive_env"].format( + "publish.folder_exclude_regex", "publish.folder_path_to_include", ["dev"] + ) + in error_messages + ) + + def test_validate_operation_section_mutually_exclusive_only_one_field_present(self): + """Test that having only one of the mutually exclusive fields is valid.""" + section = {"folder_exclude_regex": "^/legacy"} + + self.validator._validate_operation_section(section, "publish") + + mutual_errors = [ + e for e in self.validator.errors if "mutually exclusive" in e.lower() or "Cannot specify both" in e + ] + assert len(mutual_errors) == 0 + class TestFeaturesSectionValidation: """Tests for features section validation.""" diff --git a/tests/test_publish.py b/tests/test_publish.py index 187b4677..1a630764 100644 --- a/tests/test_publish.py +++ b/tests/test_publish.py @@ -1102,15 +1102,13 @@ def test_folder_inclusion_with_folder_path_to_include(mock_endpoint): constants.FEATURE_FLAG.update(original_flags) -def test_folder_inclusion_and_exclusion_together(mock_endpoint, caplog): - """Test that both folder_path_to_include and folder_path_exclude_regex can be used together. - Exclusion is applied first, followed by inclusion filtering. A warning is logged.""" - import logging +def test_folder_inclusion_and_exclusion_together(mock_endpoint): + """Test that using both folder_path_to_include and folder_path_exclude_regex raises InputError.""" with tempfile.TemporaryDirectory() as temp_dir: temp_path = Path(temp_dir) - # Create item in /deploy (included, not excluded) + # Create a minimal item so the workspace can be initialized deploy_notebook_dir = temp_path / "deploy" / "DeployNotebook.Notebook" deploy_notebook_dir.mkdir(parents=True, exist_ok=True) @@ -1130,46 +1128,6 @@ def test_folder_inclusion_and_exclusion_together(mock_endpoint, caplog): with (deploy_notebook_dir / "dummy.txt").open("w", encoding="utf-8") as f: f.write("Dummy file content") - # Create item in /deploy/legacy (included but excluded by regex) - legacy_notebook_dir = temp_path / "deploy" / "legacy" / "LegacyNotebook.Notebook" - legacy_notebook_dir.mkdir(parents=True, exist_ok=True) - - legacy_platform = legacy_notebook_dir / ".platform" - legacy_metadata = { - "metadata": { - "type": "Notebook", - "displayName": "LegacyNotebook", - "description": "Notebook in excluded subfolder", - }, - "config": {"logicalId": "legacy-notebook-id"}, - } - - with legacy_platform.open("w", encoding="utf-8") as f: - json.dump(legacy_metadata, f) - - with (legacy_notebook_dir / "dummy.txt").open("w", encoding="utf-8") as f: - f.write("Dummy file content") - - # Create item in /archive (not included, not excluded) - archive_notebook_dir = temp_path / "archive" / "ArchiveNotebook.Notebook" - archive_notebook_dir.mkdir(parents=True, exist_ok=True) - - archive_platform = archive_notebook_dir / ".platform" - archive_metadata = { - "metadata": { - "type": "Notebook", - "displayName": "ArchiveNotebook", - "description": "Notebook in non-included folder", - }, - "config": {"logicalId": "archive-notebook-id"}, - } - - with archive_platform.open("w", encoding="utf-8") as f: - json.dump(archive_metadata, f) - - with (archive_notebook_dir / "dummy.txt").open("w", encoding="utf-8") as f: - f.write("Dummy file content") - with ( patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint), patch.object( @@ -1178,7 +1136,6 @@ def test_folder_inclusion_and_exclusion_together(mock_endpoint, caplog): patch.object( FabricWorkspace, "_refresh_deployed_folders", new=lambda self: setattr(self, "deployed_folders", {}) ), - caplog.at_level(logging.WARNING), ): original_flags = constants.FEATURE_FLAG.copy() constants.FEATURE_FLAG.add("enable_experimental_features") @@ -1192,23 +1149,15 @@ def test_folder_inclusion_and_exclusion_together(mock_endpoint, caplog): item_type_in_scope=["Notebook"], ) - publish.publish_all_items( - workspace, - folder_path_to_include=["/deploy", "/deploy/legacy"], - folder_path_exclude_regex=r"^/deploy/legacy", - ) - - # Verify warning was logged about both being defined - assert "Both folder_path_exclude_regex and folder_path_to_include are defined" in caplog.text - - # /deploy item is included and not excluded -> published - assert workspace.repository_items["Notebook"]["DeployNotebook"].skip_publish is False - - # /deploy/legacy item is excluded by regex (exclusion applied first) -> skipped - assert workspace.repository_items["Notebook"]["LegacyNotebook"].skip_publish is True - - # /archive item is not in the include list -> skipped - assert workspace.repository_items["Notebook"]["ArchiveNotebook"].skip_publish is True + with pytest.raises( + InputError, + match="Cannot use both 'folder_path_exclude_regex' and 'folder_path_to_include'", + ): + publish.publish_all_items( + workspace, + folder_path_to_include=["/deploy"], + folder_path_exclude_regex=r"^/deploy/legacy", + ) finally: constants.FEATURE_FLAG.clear() From 7fa3ae9e936e0c4be41cfca9a6fcbc88e878c195 Mon Sep 17 00:00:00 2001 From: Shira Sassoon Date: Tue, 17 Feb 2026 10:30:23 +0200 Subject: [PATCH 17/18] feedback --- docs/how_to/config_deployment.md | 2 +- src/fabric_cicd/constants.py | 2 +- src/fabric_cicd/fabric_workspace.py | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/how_to/config_deployment.md b/docs/how_to/config_deployment.md index 9bacfb4f..4d654113 100644 --- a/docs/how_to/config_deployment.md +++ b/docs/how_to/config_deployment.md @@ -133,7 +133,7 @@ publish: folder_path_to_include: - - - - + - # publish items found in nested folder - subfolder_3 # Optional - specific items to publish (requires feature flags) items_to_include: diff --git a/src/fabric_cicd/constants.py b/src/fabric_cicd/constants.py index a33933d8..48cf5dfd 100644 --- a/src/fabric_cicd/constants.py +++ b/src/fabric_cicd/constants.py @@ -381,7 +381,7 @@ class OperationType(str, Enum): # Field validation "field": { "string_or_dict": "'{}' must be either a string or environment mapping dictionary (e.g., {{dev: 'dev_value', prod: 'prod_value'}}), got type {}", - "list_or_dict": "'{}' must be a list or environment mapping dictionary (e.g., {{dev: ['dev_value1', 'dev_value2'], prod: ['prod_value']}}), got type {}", + "list_or_dict": "'{}' must be either a list or environment mapping dictionary (e.g., {{dev: ['dev_value1', 'dev_value2'], prod: ['prod_value']}}), got type {}", "empty_value": "'{}' cannot be empty", "empty_list": "'{}' cannot be empty if specified", "invalid_guid": "'{}' must be a valid GUID format: {}", diff --git a/src/fabric_cicd/fabric_workspace.py b/src/fabric_cicd/fabric_workspace.py index 4adadc69..50e9e276 100644 --- a/src/fabric_cicd/fabric_workspace.py +++ b/src/fabric_cicd/fabric_workspace.py @@ -602,15 +602,18 @@ def _publish_item( # Check if the folder path itself or any ancestor matches the exclusion regex path_to_check = folder_path while path_to_check: + # If the current path (or ancestor) matches the exclusion pattern, skip this item if regex_pattern.search(path_to_check): item.skip_publish = True logger.info( f"Skipping publishing of {item_type} '{item_name}' due to folder path exclusion regex." ) return + # Move one level up by stripping the last path segment (e.g., "/a/b/c" -> "/a/b") if "/" in path_to_check and path_to_check != "": path_to_check = path_to_check.rsplit("/", 1)[0] else: + # Reached the root level with no match; stop checking break if self.publish_folder_path_to_include and folder_path not in self.publish_folder_path_to_include: From e78a7a870fce57d69ccd29e06511c4b61408900d Mon Sep 17 00:00:00 2001 From: Shira Sassoon Date: Tue, 17 Feb 2026 11:01:45 +0200 Subject: [PATCH 18/18] fix comments --- src/fabric_cicd/fabric_workspace.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/fabric_cicd/fabric_workspace.py b/src/fabric_cicd/fabric_workspace.py index 50e9e276..b2dd930b 100644 --- a/src/fabric_cicd/fabric_workspace.py +++ b/src/fabric_cicd/fabric_workspace.py @@ -599,7 +599,13 @@ def _publish_item( if folder_path: if self.publish_folder_path_exclude_regex: regex_pattern = check_regex(self.publish_folder_path_exclude_regex) - # Check if the folder path itself or any ancestor matches the exclusion regex + # Walk up the folder hierarchy checking each level against the exclusion regex. + # Cases handled: + # 1. Direct match — item's folder matches the regex (e.g., item in /A/B, regex matches /A/B) + # 2. Ancestor match — item's ancestor folder matches (e.g., item in /A/B/C, regex matches /A) + # 3. No match at any level — no exclusion applied, continue to next checks + # Note: Root-level items (empty folder_path) bypass this block entirely via the guard above. + # This ensures excluding a parent folder cascades to all descendants. path_to_check = folder_path while path_to_check: # If the current path (or ancestor) matches the exclusion pattern, skip this item @@ -616,6 +622,12 @@ def _publish_item( # Reached the root level with no match; stop checking break + # If the item's folder is not in the explicit include list, skip item publish (even though folder has been created). + # Note: unlike exclusion, this does NOT walk ancestors — only exact folder match is checked. + + # Skip if the item's folder is not explicitly in the include list. + # Unlike exclusion, this checks exact path only — ancestors are not considered. + # (e.g., including /A does NOT include items in /A/B, or includiing /A/B does NOT include items in /A, but the folder /A will still exist). if self.publish_folder_path_to_include and folder_path not in self.publish_folder_path_to_include: item.skip_publish = True logger.info( @@ -874,8 +886,8 @@ def _publish_folders(self) -> None: continue logger.debug(f"Folder path '{folder_path}' does not match the exclusion regex pattern.") # Skip folders not in the include list - # Ancestor folders must be published to preserve the correct hierarchy - # (e.g., if /A/B is included, /A must also be published). + # Ancestor folders must be published to preserve the correct hierarchy. + # Even though they may not be explicitly included, (e.g., if /A/B is included, /A must also be published). if self.publish_folder_path_to_include: is_included = folder_path in self.publish_folder_path_to_include is_ancestor_of_included = any(