diff --git a/.changes/unreleased/added-20260210-151850.yaml b/.changes/unreleased/added-20260210-151850.yaml new file mode 100644 index 00000000..6b00d3cd --- /dev/null +++ b/.changes/unreleased/added-20260210-151850.yaml @@ -0,0 +1,8 @@ +kind: added +body: Support selective folder deployment using inclusion list +time: 2026-02-10T15:18:50.3316666+02:00 +custom: + Author: shirasassoon + AuthorLink: https://github.com/shirasassoon + Issue: "757" + IssueLink: https://github.com/microsoft/fabric-cicd/issues/757 diff --git a/docs/how_to/config_deployment.md b/docs/how_to/config_deployment.md index 9cf55523..4d654113 100644 --- a/docs/how_to/config_deployment.md +++ b/docs/how_to/config_deployment.md @@ -33,7 +33,7 @@ Raise a [feature request](https://github.com/microsoft/fabric-cicd/issues/new?te The configuration file includes several sections with configurable settings for different aspects of the deployment process. -**Note**: Configuration values can be specified in two ways: as a single value (applied to any environment provided) or as an environment mapping. Both approaches can be used within the same configuration file - for example, using environment mappings for workspace IDs while keeping a single value for repository directory. +**Note**: Configuration values can be specified in two ways: as a single value (applied to any target environment provided) or as an environment mapping. Both approaches can be used within the same configuration file - for example, using environment mappings for workspace IDs while keeping a single value for repository directory. ### Core Settings @@ -115,19 +115,34 @@ core: `publish` is optional and can be used to control item publishing behavior. It includes various optional settings to enable/disable publishing operations or selectively publish items. +**Note:** Folder-level filtering only applies to items within a Fabric folder. Folder paths must start with `/` (e.g., `/folder_name` or `/folder_name/nested_folder`). `folder_exclude_regex` and `folder_path_to_include` are **mutually exclusive** — providing both for the same environment will result in a validation error. + +When using `folder_exclude_regex`, the pattern is matched using `search()` (substring match), so a pattern like `subfolder1` will match any folder path containing "subfolder1" (e.g., `/subfolder1`, `/subfolder1/subfolder2`, `/other/subfolder1`). To target a specific folder, use an anchored pattern with a leading `/` (e.g., `^/subfolder1$`) — this ensures only the exact folder path matches directly. Note that child folders like `/subfolder1/subfolder2` will also be excluded automatically since their parent folder was excluded, preserving a consistent folder hierarchy. + +When using `folder_path_to_include` with nested paths (e.g., `/subfolder1/subfolder2`), ancestor folders (e.g., `/subfolder1`) are automatically created to preserve the correct folder hierarchy, but items directly under the ancestor folder are **not** published unless the ancestor folder is also explicitly included in the list. + ```yaml publish: # Optional - pattern to exclude items from publishing exclude_regex: - # Optional - pattern to exclude items in specific folders from publishing + # Optional - pattern to exclude specific folder paths with items from publishing (requires feature flags) folder_exclude_regex: + # Optional - specific folder paths with items to publish (requires feature flags) + folder_path_to_include: + - + - + - # publish items found in nested folder - subfolder_3 + # Optional - specific items to publish (requires feature flags) items_to_include: - - + # Optional - pattern to exclude Lakehouse shortcuts from publishing (requires feature flags) + shortcut_exclude_regex: + # Optional - control publishing by environment skip: ``` @@ -141,15 +156,32 @@ publish: : : - # Optional - pattern to exclude items in specific folders from publishing + # Optional - pattern to exclude specific folder paths with items from publishing (requires feature flags) folder_exclude_regex: : : + # Optional - specific folder paths with items to publish (requires feature flags) + folder_path_to_include: + : + - + - + : + - + # Optional - specific items to publish (requires feature flags) items_to_include: - - - - + : + - + - + : + - + - + + # Optional - pattern to exclude Lakehouse Shortcuts from publishing (requires feature flags) + shortcut_exclude_regex: + : + : # Optional - control publishing by environment skip: @@ -188,7 +220,9 @@ unpublish: items_to_include: : - + - : + - - # Optional - control unpublishing by environment @@ -215,7 +249,9 @@ features: features: : - + - : + - - ``` @@ -262,6 +298,7 @@ Fields are categorized as **required** or **optional**, which affects how missin | `publish.exclude_regex` | ❌ | Debug logged, setting skipped | | `publish.folder_exclude_regex` | ❌ | Debug logged, setting skipped | | `publish.shortcut_exclude_regex` | ❌ | Debug logged, setting skipped | +| `publish.folder_path_to_include` | ❌ | Debug logged, setting skipped | | `publish.items_to_include` | ❌ | Debug logged, setting skipped | | `publish.skip` | ❌ | Defaults to `False` | | `unpublish.exclude_regex` | ❌ | Debug logged, setting skipped | @@ -285,7 +322,7 @@ core: publish: # Only exclude legacy folders in prod environment folder_exclude_regex: - prod: "^legacy_.*" + prod: "^/legacy_.*" # dev and test not specified - no folder exclusion applied # Skip publish in dev, run in test and prod @@ -298,7 +335,7 @@ In this example: - Deploying to `dev`: No folder exclusion applied, `skip` = `true` - Deploying to `test`: No folder exclusion applied, `skip` = `false` -- Deploying to `prod`: `folder_exclude_regex` = `"^legacy_.*"`, `skip` = `false` +- Deploying to `prod`: `folder_exclude_regex` = `"^/legacy_.*"`, `skip` = `false` ### Logging Behavior @@ -347,12 +384,22 @@ publish: # Don't publish items matching this pattern exclude_regex: "^DONT_DEPLOY.*" - folder_exclude_regex: "^DONT_DEPLOY_FOLDER/" + # Use folder_exclude_regex OR folder_path_to_include, not both for the same environment + folder_exclude_regex: + dev: "^/DONT_DEPLOY_FOLDER" + + folder_path_to_include: + prod: + - "/DEPLOY_FOLDER" + - "/DEPLOY_FOLDER/DEPLOY_NESTED_FOLDER" items_to_include: - "Hello World.Notebook" - "Run Hello World.DataPipeline" + shortcut_exclude_regex: + test: "^temp_.*" + skip: dev: true test: false @@ -371,6 +418,9 @@ features: - enable_shortcut_publish - enable_experimental_features - enable_items_to_include + - enable_exclude_folder + - enable_include_folder + - enable_shortcut_exclude constants: DEFAULT_API_ROOT_URL: "https://api.fabric.microsoft.com" diff --git a/docs/how_to/optional_feature.md b/docs/how_to/optional_feature.md index 41137ae4..42885d56 100644 --- a/docs/how_to/optional_feature.md +++ b/docs/how_to/optional_feature.md @@ -20,6 +20,7 @@ For scenarios that aren't supported by default, fabric-cicd offers `feature-flag | `enable_experimental_features` | Set to enable experimental features, such as selective deployments | | | `enable_items_to_include` | Set to enable selective publishing/unpublishing of items | ☑️ | | `enable_exclude_folder` | Set to enable folder-based exclusion during publish operations | ☑️ | +| `enable_include_folder` | Set to enable folder-based inclusion during publish operations | ☑️ | | `enable_shortcut_exclude` | Set to enable selective publishing of shortcuts in a Lakehouse | ☑️ | | `enable_response_collection` | Set to enable collection of API responses during publish operations | | | `continue_on_shortcut_failure` | Set to allow deployment to continue even when shortcuts fail to publish | | diff --git a/sample/workspace/config.yml b/sample/workspace/config.yml index c1988fe4..0f4a8adf 100644 --- a/sample/workspace/config.yml +++ b/sample/workspace/config.yml @@ -27,11 +27,17 @@ core: # Core configurations publish: # Publish configuration (optional) exclude_regex: "^DONT_DEPLOY.*" # Regex pattern to exclude items from publishing - # folder_exclude_regex: "^DONT_DEPLOY_FOLDER/" # Regex pattern to exclude folder with items from publishing (requires feature flags) + # folder_exclude_regex: "^/DONT_DEPLOY_FOLDER" # Regex pattern to exclude folder paths with items from publishing (requires feature flags) + + # folder_path_to_include: # Optional list of specific folder paths with items to publish (requires feature flags) + # - "/subfolderA" + # - "/subfolderA/subfolderB" # items_to_include: # Optional list of specific items to publish (requires feature flags) # - "Hello World.Notebook" # - "Run Hello World.DataPipeline" + + # shortcut_exclude_regex: "^DONT_DEPLOY_SHORTCUT.*" # Regex pattern to exclude Lakehouse shortcuts from publishing (requires feature flags) skip: # Skip publishing for specific environments dev: true # Skip publishing in dev environment diff --git a/src/fabric_cicd/_common/_config_utils.py b/src/fabric_cicd/_common/_config_utils.py index 555fbbc5..e517efac 100644 --- a/src/fabric_cicd/_common/_config_utils.py +++ b/src/fabric_cicd/_common/_config_utils.py @@ -113,6 +113,7 @@ def extract_publish_settings(config: dict, environment: str) -> dict: settings_to_update = [ "exclude_regex", "folder_exclude_regex", + "folder_path_to_include", "items_to_include", "shortcut_exclude_regex", ] diff --git a/src/fabric_cicd/_common/_config_validator.py b/src/fabric_cicd/_common/_config_validator.py index 449b89c3..c705415a 100644 --- a/src/fabric_cicd/_common/_config_validator.py +++ b/src/fabric_cicd/_common/_config_validator.py @@ -690,13 +690,16 @@ def _validate_operation_section(self, section: dict[str, Any], section_name: str # Validate exclude_regex if present if "exclude_regex" in section: exclude_regex = section["exclude_regex"] + if isinstance(exclude_regex, str): if not exclude_regex.strip(): self.errors.append( - constants.CONFIG_VALIDATION_MSGS["field"]["empty_value"].format(f"{section_name}.exclude_regex") + constants.CONFIG_VALIDATION_MSGS["operation"]["empty_string"].format( + f"{section_name}.exclude_regex" + ) ) else: - self._validate_regex(exclude_regex, section_name) + self._validate_regex(exclude_regex, f"{section_name}.exclude_regex") elif isinstance(exclude_regex, dict): # Validate environment mapping @@ -705,15 +708,8 @@ def _validate_operation_section(self, section: dict[str, Any], section_name: str # Validate each environment's regex pattern for env, regex_pattern in exclude_regex.items(): - if not regex_pattern.strip(): - self.errors.append( - constants.CONFIG_VALIDATION_MSGS["field"]["empty_value"].format( - f"{section_name}.exclude_regex.{env}" - ) - ) - continue - self._validate_regex(regex_pattern, f"{section_name}.exclude_regex.{env}") + else: self.errors.append( constants.CONFIG_VALIDATION_MSGS["field"]["string_or_dict"].format( @@ -728,7 +724,7 @@ def _validate_operation_section(self, section: dict[str, Any], section_name: str if isinstance(items, list): if not items: self.errors.append( - constants.CONFIG_VALIDATION_MSGS["field"]["empty_list"].format( + constants.CONFIG_VALIDATION_MSGS["operation"]["empty_list"].format( f"{section_name}.items_to_include" ) ) @@ -742,18 +738,13 @@ def _validate_operation_section(self, section: dict[str, Any], section_name: str # Validate each environment's items list for env, items_list in items.items(): - if not items_list: - self.errors.append( - constants.CONFIG_VALIDATION_MSGS["field"]["empty_list"].format( - f"{section_name}.items_to_include.{env}" - ) - ) - continue self._validate_items_list(items_list, f"{section_name}.items_to_include.{env}") else: self.errors.append( - constants.CONFIG_VALIDATION_MSGS["field"]["item_types_list_or_dict"].format(type(items).__name__) + constants.CONFIG_VALIDATION_MSGS["field"]["list_or_dict"].format( + f"{section_name}.items_to_include", type(items).__name__ + ) ) # Validate folder_exclude_regex if present (publish only) @@ -769,7 +760,7 @@ def _validate_operation_section(self, section: dict[str, Any], section_name: str if isinstance(folder_exclude_regex, str): if not folder_exclude_regex.strip(): self.errors.append( - constants.CONFIG_VALIDATION_MSGS["field"]["empty_value"].format( + constants.CONFIG_VALIDATION_MSGS["operation"]["empty_string"].format( f"{section_name}.folder_exclude_regex" ) ) @@ -785,15 +776,8 @@ def _validate_operation_section(self, section: dict[str, Any], section_name: str # Validate each environment's regex pattern for env, regex_pattern in folder_exclude_regex.items(): - if not regex_pattern.strip(): - self.errors.append( - constants.CONFIG_VALIDATION_MSGS["field"]["empty_value"].format( - f"{section_name}.folder_exclude_regex.{env}" - ) - ) - continue - self._validate_regex(regex_pattern, f"{section_name}.folder_exclude_regex.{env}") + else: self.errors.append( constants.CONFIG_VALIDATION_MSGS["field"]["string_or_dict"].format( @@ -801,6 +785,42 @@ def _validate_operation_section(self, section: dict[str, Any], section_name: str ) ) + # Validate folder_path_to_include if present (publish only) + if "folder_path_to_include" in section: + if section_name != "publish": + self.errors.append( + constants.CONFIG_VALIDATION_MSGS["operation"]["unsupported_field"].format( + "folder_path_to_include", section_name + ) + ) + + folders = section["folder_path_to_include"] + if isinstance(folders, list): + if not folders: + self.errors.append( + constants.CONFIG_VALIDATION_MSGS["operation"]["empty_list"].format( + f"{section_name}.folder_path_to_include" + ) + ) + else: + self._validate_folders_list(folders, f"{section_name}.folder_path_to_include") + + elif isinstance(folders, dict): + # Validate environment mapping + if not self._validate_environment_mapping(folders, f"{section_name}.folder_path_to_include", list): + return + + # Validate each environment's folders list + for env, folders_list in folders.items(): + self._validate_folders_list(folders_list, f"{section_name}.folder_path_to_include.{env}") + + else: + self.errors.append( + constants.CONFIG_VALIDATION_MSGS["field"]["list_or_dict"].format( + f"{section_name}.folder_path_to_include", type(folders).__name__ + ) + ) + # Validate shortcut_exclude_regex if present (publish only) if "shortcut_exclude_regex" in section: if section_name != "publish": @@ -812,7 +832,7 @@ def _validate_operation_section(self, section: dict[str, Any], section_name: str if isinstance(shortcut_exclude_regex, str): if not shortcut_exclude_regex.strip(): self.errors.append( - constants.CONFIG_VALIDATION_MSGS["field"]["empty_value"].format( + constants.CONFIG_VALIDATION_MSGS["operation"]["empty_string"].format( f"{section_name}.shortcut_exclude_regex" ) ) @@ -828,15 +848,8 @@ def _validate_operation_section(self, section: dict[str, Any], section_name: str # Validate each environment's regex pattern for env, regex_pattern in shortcut_exclude_regex.items(): - if not regex_pattern.strip(): - self.errors.append( - constants.CONFIG_VALIDATION_MSGS["field"]["empty_value"].format( - f"{section_name}.shortcut_exclude_regex.{env}" - ) - ) - continue - self._validate_regex(regex_pattern, f"{section_name}.shortcut_exclude_regex.{env}") + else: self.errors.append( constants.CONFIG_VALIDATION_MSGS["field"]["string_or_dict"].format( @@ -864,6 +877,11 @@ def _validate_operation_section(self, section: dict[str, Any], section_name: str .replace("a string", "a boolean") ) + # Validate mutual exclusivity of folder filtering options + self._validate_mutually_exclusive_fields( + section, "folder_exclude_regex", "folder_path_to_include", section_name + ) + def _validate_regex(self, regex: str, section_name: str) -> None: """Validate regex value.""" try: @@ -878,12 +896,66 @@ def _validate_items_list(self, items_list: list, context: str) -> None: for i, item in enumerate(items_list): if not isinstance(item, str): self.errors.append( - constants.CONFIG_VALIDATION_MSGS["operation"]["items_list_type"].format( + constants.CONFIG_VALIDATION_MSGS["operation"]["list_entry_type"].format( context, i, type(item).__name__ ) ) elif not item.strip(): - self.errors.append(constants.CONFIG_VALIDATION_MSGS["operation"]["items_list_empty"].format(context, i)) + self.errors.append(constants.CONFIG_VALIDATION_MSGS["operation"]["list_entry_empty"].format(context, i)) + + def _validate_folders_list(self, folders_list: list, context: str) -> None: + """Validate a list of folder paths with proper context for error messages.""" + for i, folder in enumerate(folders_list): + if not isinstance(folder, str): + self.errors.append( + constants.CONFIG_VALIDATION_MSGS["operation"]["list_entry_type"].format( + context, i, type(folder).__name__ + ) + ) + elif not folder.strip(): + self.errors.append(constants.CONFIG_VALIDATION_MSGS["operation"]["list_entry_empty"].format(context, i)) + elif not folder.startswith("/"): + self.errors.append( + constants.CONFIG_VALIDATION_MSGS["operation"]["folders_list_prefix"].format(context, i, folder) + ) + + def _validate_mutually_exclusive_fields(self, section: dict, field1: str, field2: str, section_name: str) -> None: + """Validate that two fields are not both specified for the same environment.""" + if field1 not in section or field2 not in section: + return + + value1 = section[field1] + value2 = section[field2] + + # Both are direct values (not environment-specific), throw error + if not isinstance(value1, dict) and not isinstance(value2, dict): + self.errors.append( + constants.CONFIG_VALIDATION_MSGS["operation"]["mutually_exclusive"].format( + f"{section_name}.{field1}", f"{section_name}.{field2}" + ) + ) + return + + # Determine which environments each field contains (if they are environment mappings) + value1_envs = set(value1.keys()) if isinstance(value1, dict) else set() + value2_envs = set(value2.keys()) if isinstance(value2, dict) else set() + + # Determine if it is a direct value + value1_is_direct = not isinstance(value1, dict) + value2_is_direct = not isinstance(value2, dict) + + # Check if both fields would resolve for the target environment + value1_applies = value1_is_direct or self.environment in value1_envs + value2_applies = value2_is_direct or self.environment in value2_envs + + if value1_applies and value2_applies: + self.errors.append( + constants.CONFIG_VALIDATION_MSGS["operation"]["mutually_exclusive_env"].format( + f"{section_name}.{field1}", + f"{section_name}.{field2}", + [self.environment], + ) + ) def _validate_features_section(self, features: any) -> None: """Validate features section.""" @@ -919,12 +991,12 @@ def _validate_features_list(self, features_list: list, context: str) -> None: for i, feature in enumerate(features_list): if not isinstance(feature, str): self.errors.append( - constants.CONFIG_VALIDATION_MSGS["operation"]["items_list_type"].format( + constants.CONFIG_VALIDATION_MSGS["operation"]["list_entry_type"].format( context, i, type(feature).__name__ ) ) elif not feature.strip(): - self.errors.append(constants.CONFIG_VALIDATION_MSGS["operation"]["items_list_empty"].format(context, i)) + self.errors.append(constants.CONFIG_VALIDATION_MSGS["operation"]["list_entry_empty"].format(context, i)) def _validate_constants_section(self, constants_section: any) -> None: """Validate constants section.""" @@ -989,6 +1061,7 @@ def _get_config_fields(config: dict) -> list[tuple[dict, str, str, bool, bool]]: # Publish section fields - optional (debug if missing) (config.get("publish", {}), "exclude_regex", "publish.exclude_regex", False, False), (config.get("publish", {}), "folder_exclude_regex", "publish.folder_exclude_regex", False, False), + (config.get("publish", {}), "folder_path_to_include", "publish.folder_path_to_include", False, False), (config.get("publish", {}), "shortcut_exclude_regex", "publish.shortcut_exclude_regex", False, False), (config.get("publish", {}), "items_to_include", "publish.items_to_include", False, False), (config.get("publish", {}), "skip", "publish.skip", False, False), diff --git a/src/fabric_cicd/_common/_item.py b/src/fabric_cicd/_common/_item.py index f4caf427..9806d7a6 100644 --- a/src/fabric_cicd/_common/_item.py +++ b/src/fabric_cicd/_common/_item.py @@ -23,6 +23,7 @@ class Item: path: Path = field(default_factory=Path) item_files: list = field(default_factory=list) folder_id: str = field(default="") + folder_path: str = field(default="") IMMUTABLE_FIELDS: ClassVar[set] = {"type", "name", "description"} skip_publish: bool = field(default=False) diff --git a/src/fabric_cicd/_common/_validate_input.py b/src/fabric_cicd/_common/_validate_input.py index cc9b411e..ae5d68a3 100644 --- a/src/fabric_cicd/_common/_validate_input.py +++ b/src/fabric_cicd/_common/_validate_input.py @@ -177,8 +177,6 @@ def validate_experimental_param( Raises: InputError: If required feature flags are not enabled. """ - from fabric_cicd.constants import FeatureFlag - if param_value is None: return @@ -204,8 +202,6 @@ def validate_items_to_include(items_to_include: Optional[list[str]], operation: Raises: InputError: If required feature flags are not enabled. """ - from fabric_cicd.constants import FeatureFlag - validate_experimental_param( param_value=items_to_include, required_flag=FeatureFlag.ENABLE_ITEMS_TO_INCLUDE, @@ -224,8 +220,6 @@ def validate_folder_path_exclude_regex(folder_path_exclude_regex: Optional[str]) Raises: InputError: If required feature flags are not enabled. """ - from fabric_cicd.constants import FeatureFlag - validate_experimental_param( param_value=folder_path_exclude_regex, required_flag=FeatureFlag.ENABLE_EXCLUDE_FOLDER, @@ -233,6 +227,40 @@ def validate_folder_path_exclude_regex(folder_path_exclude_regex: Optional[str]) risk_warning="Using folder_path_exclude_regex is risky as it can prevent needed dependencies from being deployed. Use at your own risk.", ) + if not isinstance(folder_path_exclude_regex, str): + msg = "folder_path_exclude_regex must be a string." + raise InputError(msg, logger) + + if folder_path_exclude_regex == "": + msg = "folder_path_exclude_regex must not be an empty string. Provide a valid regex pattern or omit the parameter." + raise InputError(msg, logger) + + +def validate_folder_path_to_include(folder_path_to_include: Optional[list[str]]) -> None: + """ + Validate folder_path_to_include parameter and check required feature flags. + + Args: + folder_path_to_include: List of folder paths with format ["/folder1", "/folder2", ...], or None. + + Raises: + InputError: If required feature flags are not enabled. + """ + validate_experimental_param( + param_value=folder_path_to_include, + required_flag=FeatureFlag.ENABLE_INCLUDE_FOLDER, + warning_message="Folder path inclusion is enabled.", + risk_warning="Using folder_path_to_include is risky as it can prevent needed dependencies from being deployed. Use at your own risk.", + ) + + if not isinstance(folder_path_to_include, list): + msg = "folder_path_to_include must be a list of folder paths." + raise InputError(msg, logger) + + if not folder_path_to_include: + msg = "folder_path_to_include must not be an empty list. Provide folder paths or omit the parameter." + raise InputError(msg, logger) + def validate_shortcut_exclude_regex(shortcut_exclude_regex: Optional[str]) -> None: """ @@ -244,8 +272,6 @@ def validate_shortcut_exclude_regex(shortcut_exclude_regex: Optional[str]) -> No Raises: InputError: If required feature flags are not enabled. """ - from fabric_cicd.constants import FeatureFlag - validate_experimental_param( param_value=shortcut_exclude_regex, required_flag=FeatureFlag.ENABLE_SHORTCUT_EXCLUDE, diff --git a/src/fabric_cicd/constants.py b/src/fabric_cicd/constants.py index a94b63d6..48cf5dfd 100644 --- a/src/fabric_cicd/constants.py +++ b/src/fabric_cicd/constants.py @@ -123,6 +123,8 @@ class FeatureFlag(str, Enum): """Set to enable selective publishing/unpublishing of items.""" ENABLE_EXCLUDE_FOLDER = "enable_exclude_folder" """Set to enable folder-based exclusion during publish operations.""" + ENABLE_INCLUDE_FOLDER = "enable_include_folder" + """Set to enable folder-based inclusion during publish operations.""" ENABLE_SHORTCUT_EXCLUDE = "enable_shortcut_exclude" """Set to enable selective publishing of shortcuts in a Lakehouse.""" ENABLE_RESPONSE_COLLECTION = "enable_response_collection" @@ -322,7 +324,14 @@ class OperationType(str, Enum): }, "publish": { "type": dict, - "settings": ["exclude_regex", "folder_exclude_regex", "items_to_include", "shortcut_exclude_regex", "skip"], + "settings": [ + "exclude_regex", + "folder_exclude_regex", + "folder_path_to_include", + "items_to_include", + "shortcut_exclude_regex", + "skip", + ], }, "unpublish": {"type": dict, "settings": ["exclude_regex", "items_to_include", "skip"]}, "features": {"type": (list, dict), "settings": []}, @@ -372,6 +381,7 @@ class OperationType(str, Enum): # Field validation "field": { "string_or_dict": "'{}' must be either a string or environment mapping dictionary (e.g., {{dev: 'dev_value', prod: 'prod_value'}}), got type {}", + "list_or_dict": "'{}' must be either a list or environment mapping dictionary (e.g., {{dev: ['dev_value1', 'dev_value2'], prod: ['prod_value']}}), got type {}", "empty_value": "'{}' cannot be empty", "empty_list": "'{}' cannot be empty if specified", "invalid_guid": "'{}' must be a valid GUID format: {}", @@ -393,15 +403,21 @@ class OperationType(str, Enum): }, # Operation section validation "operation": { + "unsupported_field": "'{}' field is not supported in '{}' section", "not_dict": "'{}' section must be a dictionary, got {}", "invalid_regex": "'{}' in {} is not a valid regex pattern: {}", - "items_list_type": "'{}[{}]' must be a string, got {}", - "items_list_empty": "'{}[{}]' cannot be empty", + "empty_string": "'{}' cannot be an empty string", + "empty_list": "'{}' cannot be an empty list", + "list_entry_type": "'{}[{}]' must be a string, got {}", + "list_entry_empty": "'{}[{}]' cannot be an empty string", "features_type": "'features' section must be either a list or environment mapping dictionary, got {}", "empty_section": "'{}' section cannot be empty if specified", "empty_section_env": "'{}.{}' cannot be empty if specified", "invalid_constant_key": "Constant key in '{}' must be a non-empty string, got: {}", "unknown_constant": "Unknown constant '{}' in '{}' - this constant does not exist in fabric_cicd.constants", + "folders_list_prefix": "'{}[{}]' entry must start with '/' (got '{}')", + "mutually_exclusive": "Cannot specify both '{}' and '{}'. Choose one filtering strategy.", + "mutually_exclusive_env": "Cannot specify both '{}' and '{}' for the same environment(s): {}. Choose one filtering strategy per environment.", }, # Log messages "log": { diff --git a/src/fabric_cicd/fabric_workspace.py b/src/fabric_cicd/fabric_workspace.py index b3b22b6b..b2dd930b 100644 --- a/src/fabric_cicd/fabric_workspace.py +++ b/src/fabric_cicd/fabric_workspace.py @@ -129,6 +129,7 @@ def __init__( self.environment = validate_environment(environment) self.publish_item_name_exclude_regex = None self.publish_folder_path_exclude_regex = None + self.publish_folder_path_to_include = None self.shortcut_exclude_regex = None self.items_to_include = None self.responses = None @@ -342,6 +343,7 @@ def _refresh_repository_items(self) -> None: logical_id=item_logical_id, path=item_path, folder_id=item_folder_id, + folder_path=relative_parent_path, ) self.repository_items[item_type][item_name].collect_item_files() @@ -591,15 +593,47 @@ def _publish_item( logger.info(f"Skipping publishing of {item_type} '{item_name}' due to exclusion regex.") return - # Skip publishing if the item's folder path is excluded by the regex - if self.publish_folder_path_exclude_regex: - regex_pattern = check_regex(self.publish_folder_path_exclude_regex) - relative_path = item.path.relative_to(Path(self.repository_directory)) - relative_path_str = relative_path.as_posix() - if regex_pattern.search(relative_path_str): - item.skip_publish = True - logger.info(f"Skipping publishing of {item_type} '{item_name}' due to folder path exclusion regex.") - return + # Skip publishing if the folder path is excluded by the regex or not in the include list + if self.publish_folder_path_exclude_regex or self.publish_folder_path_to_include: + folder_path = item.folder_path or "" + if folder_path: + if self.publish_folder_path_exclude_regex: + regex_pattern = check_regex(self.publish_folder_path_exclude_regex) + # Walk up the folder hierarchy checking each level against the exclusion regex. + # Cases handled: + # 1. Direct match — item's folder matches the regex (e.g., item in /A/B, regex matches /A/B) + # 2. Ancestor match — item's ancestor folder matches (e.g., item in /A/B/C, regex matches /A) + # 3. No match at any level — no exclusion applied, continue to next checks + # Note: Root-level items (empty folder_path) bypass this block entirely via the guard above. + # This ensures excluding a parent folder cascades to all descendants. + path_to_check = folder_path + while path_to_check: + # If the current path (or ancestor) matches the exclusion pattern, skip this item + if regex_pattern.search(path_to_check): + item.skip_publish = True + logger.info( + f"Skipping publishing of {item_type} '{item_name}' due to folder path exclusion regex." + ) + return + # Move one level up by stripping the last path segment (e.g., "/a/b/c" -> "/a/b") + if "/" in path_to_check and path_to_check != "": + path_to_check = path_to_check.rsplit("/", 1)[0] + else: + # Reached the root level with no match; stop checking + break + + # If the item's folder is not in the explicit include list, skip item publish (even though folder has been created). + # Note: unlike exclusion, this does NOT walk ancestors — only exact folder match is checked. + + # Skip if the item's folder is not explicitly in the include list. + # Unlike exclusion, this checks exact path only — ancestors are not considered. + # (e.g., including /A does NOT include items in /A/B, or includiing /A/B does NOT include items in /A, but the folder /A will still exist). + if self.publish_folder_path_to_include and folder_path not in self.publish_folder_path_to_include: + item.skip_publish = True + logger.info( + f"Skipping publishing of {item_type} '{item_name}' under {folder_path} as it is not in the include list." + ) + return # Skip publishing if the item is not in the include list if self.items_to_include: @@ -830,6 +864,38 @@ def _publish_folders(self) -> None: log_header(logger, "Publishing Workspace Folders") logger.info("Publishing Workspace Folders") for folder_path in sorted_folders: + # Skip folders matching the exclusion regex + if self.publish_folder_path_exclude_regex: + regex_pattern = check_regex(self.publish_folder_path_exclude_regex) + if regex_pattern.search(folder_path): + logger.info(f"Skipping publishing of folder '{folder_path}' due to folder path exclusion regex.") + continue + # If any ancestor folder was excluded by the regex, skip this + # descendant folder too to preserve a consistent hierarchy + ancestor_path = folder_path + ancestor_excluded = False + while "/" in ancestor_path and ancestor_path != "": + ancestor_path = ancestor_path.rsplit("/", 1)[0] + if ancestor_path and regex_pattern.search(ancestor_path): + ancestor_excluded = True + break + if ancestor_excluded: + logger.info( + f"Skipping publishing of folder '{folder_path}' because ancestor folder was excluded by regex." + ) + continue + logger.debug(f"Folder path '{folder_path}' does not match the exclusion regex pattern.") + # Skip folders not in the include list + # Ancestor folders must be published to preserve the correct hierarchy. + # Even though they may not be explicitly included, (e.g., if /A/B is included, /A must also be published). + if self.publish_folder_path_to_include: + is_included = folder_path in self.publish_folder_path_to_include + is_ancestor_of_included = any( + included.startswith(folder_path + "/") for included in self.publish_folder_path_to_include + ) + if not is_included and not is_ancestor_of_included: + logger.info(f"Skipping publishing of folder '{folder_path}' as it is not in the include list.") + continue if folder_path in self.deployed_folders: # Folder already deployed, update local hierarchy self.repository_folders[folder_path] = self.deployed_folders[folder_path] diff --git a/src/fabric_cicd/publish.py b/src/fabric_cicd/publish.py index 14449b72..83a64ec2 100644 --- a/src/fabric_cicd/publish.py +++ b/src/fabric_cicd/publish.py @@ -18,12 +18,13 @@ extract_workspace_settings, load_config_file, ) -from fabric_cicd._common._exceptions import FailedPublishedItemStatusError +from fabric_cicd._common._exceptions import FailedPublishedItemStatusError, InputError from fabric_cicd._common._logging import log_header from fabric_cicd._common._validate_input import ( validate_environment, validate_fabric_workspace_obj, validate_folder_path_exclude_regex, + validate_folder_path_to_include, validate_items_to_include, validate_shortcut_exclude_regex, ) @@ -37,6 +38,7 @@ def publish_all_items( fabric_workspace_obj: FabricWorkspace, item_name_exclude_regex: Optional[str] = None, folder_path_exclude_regex: Optional[str] = None, + folder_path_to_include: Optional[list[str]] = None, items_to_include: Optional[list[str]] = None, shortcut_exclude_regex: Optional[str] = None, ) -> Optional[dict]: @@ -46,7 +48,8 @@ def publish_all_items( Args: fabric_workspace_obj: The FabricWorkspace object containing the items to be published. item_name_exclude_regex: Regex pattern to exclude specific items from being published. - folder_path_exclude_regex: Regex pattern to exclude items based on their folder path. + folder_path_exclude_regex: Regex pattern matched against folder paths (e.g., "/folder_name") to exclude folders and their contents from being published. + folder_path_to_include: List of folder paths in the format "/folder_name"; only the specified folders and their contents will be published. items_to_include: List of items in the format "item_name.item_type" that should be published. shortcut_exclude_regex: Regex pattern to exclude specific shortcuts from being published in lakehouses. @@ -55,8 +58,15 @@ def publish_all_items( folder_path_exclude_regex: This is an experimental feature in fabric-cicd. Use at your own risk as selective deployments are - not recommended due to item dependencies. To enable this feature, see How To -> Optional Features - for information on which flags to enable. + not recommended due to item dependencies. Cannot be used together with ``folder_path_to_include`` + for the same environment. To enable this feature, see How To -> Optional Features for information + on which flags to enable. + + folder_path_to_include: + This is an experimental feature in fabric-cicd. Use at your own risk as selective deployments are + not recommended due to item dependencies. Cannot be used together with ``folder_path_exclude_regex`` + for the same environment. To enable this feature, see How To -> Optional Features for information + on which flags to enable. items_to_include: This is an experimental feature in fabric-cicd. Use at your own risk as selective deployments are @@ -97,9 +107,21 @@ def publish_all_items( ... repository_directory="/path/to/repo", ... item_type_in_scope=["Environment", "Notebook", "DataPipeline"] ... ) - >>> folder_exclude_regex = "^legacy/" + >>> folder_exclude_regex = "^/legacy" >>> publish_all_items(workspace, folder_path_exclude_regex=folder_exclude_regex) + With folder inclusion + >>> from fabric_cicd import FabricWorkspace, publish_all_items, append_feature_flag + >>> append_feature_flag("enable_experimental_features") + >>> append_feature_flag("enable_include_folder") + >>> workspace = FabricWorkspace( + ... workspace_id="your-workspace-id", + ... repository_directory="/path/to/repo", + ... item_type_in_scope=["Environment", "Notebook", "DataPipeline"] + ... ) + >>> folder_path_to_include = ["/subfolder"] + >>> publish_all_items(workspace, folder_path_to_include=folder_path_to_include) + With items to include >>> from fabric_cicd import FabricWorkspace, publish_all_items, append_feature_flag >>> append_feature_flag("enable_experimental_features") @@ -161,6 +183,18 @@ def publish_all_items( raise FailedPublishedItemStatusError(msg, logger) if FeatureFlag.DISABLE_WORKSPACE_FOLDER_PUBLISH.value not in constants.FEATURE_FLAG: + if folder_path_exclude_regex is not None and folder_path_to_include is not None: + msg = "Cannot use both 'folder_path_exclude_regex' and 'folder_path_to_include' simultaneously. Choose one filtering strategy." + raise InputError(msg, logger) + + if folder_path_exclude_regex is not None: + validate_folder_path_exclude_regex(folder_path_exclude_regex) + fabric_workspace_obj.publish_folder_path_exclude_regex = folder_path_exclude_regex + + if folder_path_to_include is not None: + validate_folder_path_to_include(folder_path_to_include) + fabric_workspace_obj.publish_folder_path_to_include = folder_path_to_include + fabric_workspace_obj._refresh_deployed_folders() fabric_workspace_obj._refresh_repository_folders() fabric_workspace_obj._publish_folders() @@ -174,10 +208,6 @@ def publish_all_items( ) fabric_workspace_obj.publish_item_name_exclude_regex = item_name_exclude_regex - if folder_path_exclude_regex: - validate_folder_path_exclude_regex(folder_path_exclude_regex) - fabric_workspace_obj.publish_folder_path_exclude_regex = folder_path_exclude_regex - if items_to_include: validate_items_to_include(items_to_include, operation=constants.OperationType.PUBLISH) fabric_workspace_obj.items_to_include = items_to_include @@ -386,6 +416,7 @@ def deploy_with_config( workspace, item_name_exclude_regex=publish_settings.get("exclude_regex"), folder_path_exclude_regex=publish_settings.get("folder_exclude_regex"), + folder_path_to_include=publish_settings.get("folder_path_to_include"), items_to_include=publish_settings.get("items_to_include"), shortcut_exclude_regex=publish_settings.get("shortcut_exclude_regex"), ) diff --git a/tests/test_config_validator.py b/tests/test_config_validator.py index 0192d3ee..cd2d20d2 100644 --- a/tests/test_config_validator.py +++ b/tests/test_config_validator.py @@ -431,7 +431,10 @@ def test_validate_items_list_empty_item(self): self.validator._validate_items_list(items_list, "test_context") assert len(self.validator.errors) == 1 - assert "'test_context[1]' cannot be empty" in self.validator.errors[0] + assert ( + constants.CONFIG_VALIDATION_MSGS["operation"]["list_entry_empty"].format("test_context", 1) + in self.validator.errors[0] + ) def test_validate_features_list_valid(self): """Test _validate_features_list with valid features.""" @@ -457,7 +460,10 @@ def test_validate_features_list_empty_feature(self): self.validator._validate_features_list(features_list, "test_context") assert len(self.validator.errors) == 1 - assert "'test_context[1]' cannot be empty" in self.validator.errors[0] + assert ( + constants.CONFIG_VALIDATION_MSGS["operation"]["list_entry_empty"].format("test_context", 1) + in self.validator.errors[0] + ) def test_validate_constants_dict_valid(self): """Test _validate_constants_dict with valid constants.""" @@ -1375,7 +1381,8 @@ def test_get_config_fields_complete_config(self): }, "publish": { "exclude_regex": ".*_test", - "folder_exclude_regex": "^temp/", + "folder_exclude_regex": "^/temp", + "folder_path_to_include": ["/subfolder"], "shortcut_exclude_regex": "^shortcut_temp/", "items_to_include": ["item1"], "skip": False, @@ -1388,7 +1395,7 @@ def test_get_config_fields_complete_config(self): fields = _get_config_fields(config) # Should return all fields from all sections - assert len(fields) == 15 # Updated count with folder_exclude_regex and shortcut_exclude_regex + assert len(fields) == 16 # Updated count with folder_exclude_regex and shortcut_exclude_regex # Check some specific fields field_names = [field[1] for field in fields] @@ -1642,7 +1649,10 @@ def test_validate_operation_section_empty_exclude_regex(self): self.validator._validate_operation_section(section, "publish") assert len(self.validator.errors) == 1 - assert "'publish.exclude_regex' cannot be empty" in self.validator.errors[0] + assert ( + constants.CONFIG_VALIDATION_MSGS["operation"]["empty_string"].format("publish.exclude_regex") + in self.validator.errors[0] + ) def test_validate_operation_section_invalid_regex(self): """Test _validate_operation_section with invalid regex.""" @@ -1677,7 +1687,10 @@ def test_validate_operation_section_empty_items_to_include(self): self.validator._validate_operation_section(section, "publish") assert len(self.validator.errors) == 1 - assert "'publish.items_to_include' cannot be empty if specified" in self.validator.errors[0] + assert ( + constants.CONFIG_VALIDATION_MSGS["operation"]["empty_list"].format("publish.items_to_include") + in self.validator.errors[0] + ) def test_validate_operation_section_items_to_include_environment_mapping(self): """Test _validate_operation_section with items_to_include environment mapping.""" @@ -1694,7 +1707,10 @@ def test_validate_operation_section_items_to_include_invalid_type(self): self.validator._validate_operation_section(section, "publish") assert len(self.validator.errors) == 1 - assert "must be either a list or environment mapping dictionary" in self.validator.errors[0] + assert ( + constants.CONFIG_VALIDATION_MSGS["field"]["list_or_dict"].format("publish.items_to_include", "str") + in self.validator.errors[0] + ) def test_validate_operation_section_skip_boolean(self): """Test _validate_operation_section with skip as boolean.""" @@ -1728,7 +1744,7 @@ def test_validate_operation_section_skip_invalid_type(self): def test_validate_operation_section_with_folder_exclude_regex(self): """Test _validate_operation_section with folder_exclude_regex.""" - section = {"folder_exclude_regex": "^DONT_DEPLOY_FOLDER/"} + section = {"folder_exclude_regex": "^/DONT_DEPLOY_FOLDER"} self.validator._validate_operation_section(section, "publish") @@ -1745,7 +1761,7 @@ def test_validate_operation_section_with_invalid_folder_exclude_regex(self): def test_validate_operation_section_with_folder_exclude_regex_environment_mapping(self): """Test _validate_operation_section with folder_exclude_regex environment mapping.""" - section = {"folder_exclude_regex": {"dev": "^DEV_FOLDER/", "prod": "^PROD_FOLDER/"}} + section = {"folder_exclude_regex": {"dev": "^/DEV_FOLDER", "prod": "^/PROD_FOLDER"}} self.validator._validate_operation_section(section, "publish") @@ -1757,7 +1773,7 @@ def test_folder_exclude_regex_restricted_to_publish_section(self): # This test doesn't directly test the implementation or error message # Test that it's allowed in publish - section_publish = {"folder_exclude_regex": "^DONT_DEPLOY_FOLDER/"} + section_publish = {"folder_exclude_regex": "^/DONT_DEPLOY_FOLDER"} self.validator.errors = [] # Reset errors self.validator._validate_operation_section(section_publish, "publish") assert len(self.validator.errors) == 0 # Should be valid in publish @@ -1765,6 +1781,152 @@ def test_folder_exclude_regex_restricted_to_publish_section(self): # We can't test the negative case (unpublish) directly due to missing error message key # So we'll just document that the feature should be restricted to publish section + def test_validate_operation_section_folder_path_to_include_valid_list(self): + """Test _validate_operation_section with valid folder_path_to_include list.""" + section = {"folder_path_to_include": ["/FolderA", "/FolderB"]} + + self.validator._validate_operation_section(section, "publish") + + assert len(self.validator.errors) == 0 + + def test_validate_operation_section_folder_path_to_include_valid_env_mapping(self): + """Test _validate_operation_section with valid folder_path_to_include environment mapping.""" + section = {"folder_path_to_include": {"dev": ["/FolderA"], "prod": ["/FolderA", "/FolderB"]}} + + self.validator._validate_operation_section(section, "publish") + + assert len(self.validator.errors) == 0 + + def test_validate_operation_section_folder_path_to_include_empty_list(self): + """Test _validate_operation_section with empty folder_path_to_include list.""" + section = {"folder_path_to_include": []} + + self.validator._validate_operation_section(section, "publish") + + assert len(self.validator.errors) == 1 + assert ( + constants.CONFIG_VALIDATION_MSGS["operation"]["empty_list"].format("publish.folder_path_to_include") + in self.validator.errors[0] + ) + + def test_validate_operation_section_folder_path_to_include_invalid_type(self): + """Test _validate_operation_section with folder_path_to_include invalid type.""" + section = {"folder_path_to_include": "not a list or dict"} + + self.validator._validate_operation_section(section, "publish") + + assert len(self.validator.errors) == 1 + assert ( + constants.CONFIG_VALIDATION_MSGS["field"]["list_or_dict"].format("publish.folder_path_to_include", "str") + in self.validator.errors[0] + ) + + def test_validate_operation_section_folder_path_to_include_unsupported_in_unpublish(self): + """Test _validate_operation_section with folder_path_to_include in unpublish section.""" + section = {"folder_path_to_include": ["/FolderA"]} + + self.validator._validate_operation_section(section, "unpublish") + + assert len(self.validator.errors) == 1 + assert ( + constants.CONFIG_VALIDATION_MSGS["operation"]["unsupported_field"].format( + "folder_path_to_include", "unpublish" + ) + in self.validator.errors[0] + ) + + def test_validate_operation_section_folder_path_to_include_entry_not_string(self): + """Test _validate_operation_section with non-string entry in folder_path_to_include.""" + section = {"folder_path_to_include": ["/FolderA", 123, "/FolderB"]} + + self.validator._validate_operation_section(section, "publish") + + assert len(self.validator.errors) == 1 + assert ( + constants.CONFIG_VALIDATION_MSGS["operation"]["list_entry_type"].format( + "publish.folder_path_to_include", 1, "int" + ) + in self.validator.errors[0] + ) + + def test_validate_operation_section_folder_path_to_include_empty_string_entry(self): + """Test _validate_operation_section with empty string entry in folder_path_to_include.""" + section = {"folder_path_to_include": ["/FolderA", "", "/FolderB"]} + + self.validator._validate_operation_section(section, "publish") + + assert len(self.validator.errors) == 1 + assert ( + constants.CONFIG_VALIDATION_MSGS["operation"]["list_entry_empty"].format( + "publish.folder_path_to_include", 1 + ) + in self.validator.errors[0] + ) + + def test_validate_operation_section_folder_path_to_include_missing_prefix(self): + """Test _validate_operation_section with folder entry missing leading slash.""" + section = {"folder_path_to_include": ["/FolderA", "FolderB"]} + + self.validator._validate_operation_section(section, "publish") + + assert len(self.validator.errors) == 1 + assert ( + constants.CONFIG_VALIDATION_MSGS["operation"]["folders_list_prefix"].format( + "publish.folder_path_to_include", 1, "FolderB" + ) + in self.validator.errors[0] + ) + + def test_validate_operation_section_folder_path_to_include_env_mapping_empty_list(self): + """Test _validate_operation_section with empty list in environment mapping for folder_path_to_include.""" + section = {"folder_path_to_include": {"dev": ["/FolderA"], "prod": []}} + + self.validator._validate_operation_section(section, "publish") + + assert len(self.validator.errors) == 1 + assert ( + constants.CONFIG_VALIDATION_MSGS["environment"]["empty_env_value"].format( + "publish.folder_path_to_include", "prod" + ) + in self.validator.errors[0] + ) + + def test_validate_operation_section_folder_path_to_include_env_mapping_invalid_entry(self): + """Test _validate_operation_section with invalid entry in environment mapping for folder_path_to_include.""" + section = {"folder_path_to_include": {"dev": ["/FolderA", "NoSlash"]}} + + self.validator._validate_operation_section(section, "publish") + + assert len(self.validator.errors) == 1 + assert ( + constants.CONFIG_VALIDATION_MSGS["operation"]["folders_list_prefix"].format( + "publish.folder_path_to_include.dev", 1, "NoSlash" + ) + in self.validator.errors[0] + ) + + def test_validate_operation_section_folder_path_to_include_nested_path(self): + """Test _validate_operation_section with nested folder paths in folder_path_to_include.""" + section = {"folder_path_to_include": ["/FolderA/SubFolder", "/FolderB/Sub1/Sub2"]} + + self.validator._validate_operation_section(section, "publish") + + assert len(self.validator.errors) == 0 + + def test_validate_operation_section_folder_path_to_include_whitespace_entry(self): + """Test _validate_operation_section with whitespace-only entry in folder_path_to_include.""" + section = {"folder_path_to_include": ["/FolderA", " "]} + + self.validator._validate_operation_section(section, "publish") + + assert len(self.validator.errors) == 1 + assert ( + constants.CONFIG_VALIDATION_MSGS["operation"]["list_entry_empty"].format( + "publish.folder_path_to_include", 1 + ) + in self.validator.errors[0] + ) + def test_validate_operation_section_with_shortcut_exclude_regex(self): """Test _validate_operation_section with shortcut_exclude_regex.""" section = {"shortcut_exclude_regex": "^temp_.*"} @@ -1790,6 +1952,117 @@ def test_validate_operation_section_with_shortcut_exclude_regex_environment_mapp assert self.validator.errors == [] + def test_validate_operation_section_mutually_exclusive_both_direct_values(self): + """Test that both folder_exclude_regex and folder_path_to_include as direct values raises error.""" + section = {"folder_exclude_regex": "^/legacy", "folder_path_to_include": ["/subfolder"]} + + self.validator._validate_operation_section(section, "publish") + + error_messages = " ".join(self.validator.errors) + assert ( + constants.CONFIG_VALIDATION_MSGS["operation"]["mutually_exclusive"].format( + "publish.folder_exclude_regex", "publish.folder_path_to_include" + ) + in error_messages + ) + + def test_validate_operation_section_mutually_exclusive_both_env_mapped_overlapping(self): + """Test that both fields with overlapping environment mappings raises error.""" + self.validator.environment = "dev" + section = { + "folder_exclude_regex": {"dev": "^/legacy"}, + "folder_path_to_include": {"dev": ["/subfolder"]}, + } + + self.validator._validate_operation_section(section, "publish") + + error_messages = " ".join(self.validator.errors) + assert ( + constants.CONFIG_VALIDATION_MSGS["operation"]["mutually_exclusive_env"].format( + "publish.folder_exclude_regex", "publish.folder_path_to_include", ["dev"] + ) + in error_messages + ) + + def test_validate_operation_section_mutually_exclusive_both_env_mapped_no_overlap(self): + """Test that both fields with non-overlapping environment mappings is valid.""" + self.validator.environment = "dev" + section = { + "folder_exclude_regex": {"dev": "^/legacy"}, + "folder_path_to_include": {"prod": ["/subfolder"]}, + } + + self.validator._validate_operation_section(section, "publish") + + # Filter errors to only mutual exclusivity errors + mutual_errors = [ + e for e in self.validator.errors if "mutually exclusive" in e.lower() or "Cannot specify both" in e + ] + assert len(mutual_errors) == 0 + + def test_validate_operation_section_mutually_exclusive_direct_and_env_mapped_conflict(self): + """Test that direct value + env-mapped value conflicts when target env matches.""" + self.validator.environment = "dev" + section = { + "folder_exclude_regex": "^/legacy", # direct, applies to all + "folder_path_to_include": {"dev": ["/subfolder"]}, # env-mapped, applies to dev + } + + self.validator._validate_operation_section(section, "publish") + + error_messages = " ".join(self.validator.errors) + assert ( + constants.CONFIG_VALIDATION_MSGS["operation"]["mutually_exclusive_env"].format( + "publish.folder_exclude_regex", "publish.folder_path_to_include", ["dev"] + ) + in error_messages + ) + + def test_validate_operation_section_mutually_exclusive_direct_and_env_mapped_no_conflict(self): + """Test that direct value + env-mapped value is valid when target env doesn't match.""" + self.validator.environment = "prod" + section = { + "folder_exclude_regex": "^/legacy", # direct, applies to all + "folder_path_to_include": {"dev": ["/subfolder"]}, # env-mapped, only dev + } + + self.validator._validate_operation_section(section, "publish") + + # The direct value applies to prod, but folder_path_to_include doesn't apply to prod + mutual_errors = [ + e for e in self.validator.errors if "mutually exclusive" in e.lower() or "Cannot specify both" in e + ] + assert len(mutual_errors) == 0 + + def test_validate_operation_section_mutually_exclusive_env_mapped_and_direct_conflict(self): + """Test that env-mapped value + direct value conflicts when target env matches.""" + self.validator.environment = "dev" + section = { + "folder_exclude_regex": {"dev": "^/legacy"}, # env-mapped, applies to dev + "folder_path_to_include": ["/subfolder"], # direct, applies to all + } + + self.validator._validate_operation_section(section, "publish") + + error_messages = " ".join(self.validator.errors) + assert ( + constants.CONFIG_VALIDATION_MSGS["operation"]["mutually_exclusive_env"].format( + "publish.folder_exclude_regex", "publish.folder_path_to_include", ["dev"] + ) + in error_messages + ) + + def test_validate_operation_section_mutually_exclusive_only_one_field_present(self): + """Test that having only one of the mutually exclusive fields is valid.""" + section = {"folder_exclude_regex": "^/legacy"} + + self.validator._validate_operation_section(section, "publish") + + mutual_errors = [ + e for e in self.validator.errors if "mutually exclusive" in e.lower() or "Cannot specify both" in e + ] + assert len(mutual_errors) == 0 + class TestFeaturesSectionValidation: """Tests for features section validation.""" diff --git a/tests/test_deploy_with_config.py b/tests/test_deploy_with_config.py index 3de917d6..86068841 100644 --- a/tests/test_deploy_with_config.py +++ b/tests/test_deploy_with_config.py @@ -458,6 +458,7 @@ def test_deploy_with_config_full_deployment(self, mock_unpublish, mock_publish, mock_workspace_instance, item_name_exclude_regex="^DONT_DEPLOY.*", folder_path_exclude_regex=None, + folder_path_to_include=None, items_to_include=None, shortcut_exclude_regex=None, ) @@ -637,12 +638,94 @@ def test_deploy_with_config_shortcut_exclude_regex(self, mock_unpublish, mock_pu mock_workspace_instance, item_name_exclude_regex=None, folder_path_exclude_regex=None, + folder_path_to_include=None, items_to_include=None, shortcut_exclude_regex="^temp_.*", ) # Verify unpublish was also called (but without shortcut_exclude_regex since it's publish-only) mock_unpublish.assert_called_once() + @patch("fabric_cicd.publish.FabricWorkspace") + @patch("fabric_cicd.publish.publish_all_items") + @patch("fabric_cicd.publish.unpublish_all_orphan_items") + def test_folder_path_to_include_passed_to_publish(self, _mock_unpublish, mock_publish, mock_workspace, tmp_path): # noqa: PT019 + """Test that folder_path_to_include from config is passed to publish_all_items.""" + test_repo_dir = tmp_path / "repo" + test_repo_dir.mkdir(parents=True) + + config_data = { + "core": { + "workspace_id": "11111111-1111-1111-1111-111111111111", + "repository_directory": str(test_repo_dir), + }, + "publish": { + "folder_path_to_include": ["/my/folder/path"], + }, + } + config_file = tmp_path / "config.yml" + with Path.open(config_file, "w") as f: + yaml.dump(config_data, f) + + mock_workspace.return_value = MagicMock() + deploy_with_config(str(config_file), "dev") + + call_args = mock_publish.call_args[1] + assert call_args["folder_path_to_include"] == ["/my/folder/path"] + + @patch("fabric_cicd.publish.FabricWorkspace") + @patch("fabric_cicd.publish.publish_all_items") + @patch("fabric_cicd.publish.unpublish_all_orphan_items") + def test_folder_path_to_include_defaults_to_none(self, _mock_unpublish, mock_publish, mock_workspace, tmp_path): # noqa: PT019 + """Test that folder_path_to_include defaults to None when not specified.""" + test_repo_dir = tmp_path / "repo" + test_repo_dir.mkdir(parents=True) + + config_data = { + "core": { + "workspace_id": "11111111-1111-1111-1111-111111111111", + "repository_directory": str(test_repo_dir), + }, + } + config_file = tmp_path / "config.yml" + with Path.open(config_file, "w") as f: + yaml.dump(config_data, f) + + mock_workspace.return_value = MagicMock() + deploy_with_config(str(config_file), "dev") + + call_args = mock_publish.call_args[1] + assert call_args["folder_path_to_include"] is None + + @patch("fabric_cicd.publish.FabricWorkspace") + @patch("fabric_cicd.publish.publish_all_items") + @patch("fabric_cicd.publish.unpublish_all_orphan_items") + def test_folder_path_to_include_environment_specific(self, _mock_unpublish, mock_publish, mock_workspace, tmp_path): # noqa: PT019 + """Test that folder_path_to_include resolves environment-specific values.""" + test_repo_dir = tmp_path / "repo" + test_repo_dir.mkdir(parents=True) + + config_data = { + "core": { + "workspace_id": { + "dev": "11111111-1111-1111-1111-111111111111", + "prod": "22222222-2222-2222-2222-222222222222", + }, + "repository_directory": str(test_repo_dir), + }, + "publish": { + "folder_path_to_include": {"dev": ["/dev/folder"], "prod": ["/prod/folder"]}, + }, + } + config_file = tmp_path / "config.yml" + with Path.open(config_file, "w") as f: + yaml.dump(config_data, f) + + mock_workspace.return_value = MagicMock() + deploy_with_config(str(config_file), "dev") + + call_args = mock_publish.call_args[1] + assert call_args["folder_path_to_include"] == ["/dev/folder"] + class TestConfigIntegration: """Integration tests for config functionality.""" @@ -739,33 +822,33 @@ def test_extract_publish_settings_with_folder_exclude_regex(self): """Test extracting publish settings with folder_exclude_regex.""" config = { "publish": { - "folder_exclude_regex": "^DONT_DEPLOY_FOLDER/", + "folder_exclude_regex": "^/DONT_DEPLOY_FOLDER", } } settings = extract_publish_settings(config, "dev") - assert settings["folder_exclude_regex"] == "^DONT_DEPLOY_FOLDER/" + assert settings["folder_exclude_regex"] == "^/DONT_DEPLOY_FOLDER" def test_extract_publish_settings_with_environment_specific_folder_exclude_regex(self): """Test extracting publish settings with environment-specific folder_exclude_regex.""" config = { "publish": { - "folder_exclude_regex": {"dev": "^DEV_FOLDER/", "prod": "^PROD_FOLDER/"}, + "folder_exclude_regex": {"dev": "^/DEV_FOLDER", "prod": "^/PROD_FOLDER"}, } } settings = extract_publish_settings(config, "dev") - assert settings["folder_exclude_regex"] == "^DEV_FOLDER/" + assert settings["folder_exclude_regex"] == "^/DEV_FOLDER" settings = extract_publish_settings(config, "prod") - assert settings["folder_exclude_regex"] == "^PROD_FOLDER/" + assert settings["folder_exclude_regex"] == "^/PROD_FOLDER" def test_extract_publish_settings_missing_environment_skips_setting(self): """Test that missing environment in optional publish settings skips the setting.""" config = { "publish": { "exclude_regex": {"dev": "^DEV.*"}, # Only dev defined - "folder_exclude_regex": {"dev": "^DEV_FOLDER/"}, # Only dev defined + "folder_exclude_regex": {"dev": "^/DEV_FOLDER"}, # Only dev defined } } @@ -855,6 +938,42 @@ def test_extract_publish_settings_items_to_include_missing_environment(self): settings = extract_publish_settings(config, "prod") assert "items_to_include" not in settings + def test_extract_publish_settings_folder_path_to_include_list(self): + """Test extract_publish_settings returns folder_path_to_include as a list.""" + config = { + "publish": { + "folder_path_to_include": ["/my/folder/path"], + }, + } + result = extract_publish_settings(config, "dev") + assert result["folder_path_to_include"] == ["/my/folder/path"] + + def test_extract_publish_settings_folder_path_to_include_env_specific(self): + """Test extract_publish_settings resolves folder_path_to_include per environment.""" + config = { + "publish": { + "folder_path_to_include": {"dev": ["/dev/folder"], "prod": ["/prod/folder"]}, + }, + } + result = extract_publish_settings(config, "dev") + assert result["folder_path_to_include"] == ["/dev/folder"] + + def test_extract_publish_settings_folder_path_to_include_missing(self): + """Test extract_publish_settings defaults folder_path_to_include to None.""" + config = { + "publish": { + "exclude_regex": "^SKIP.*", + }, + } + settings = extract_publish_settings(config, "dev") + assert "folder_path_to_include" not in settings + + def test_extract_publish_settings_no_publish_section_folder_path_to_include(self): + """Test extract_publish_settings defaults folder_path_to_include to None when no publish section.""" + config = {} + settings = extract_publish_settings(config, "dev") + assert "folder_path_to_include" not in settings + class TestGetConfigValue: """Test the get_config_value utility function.""" diff --git a/tests/test_publish.py b/tests/test_publish.py index 3133b088..1a630764 100644 --- a/tests/test_publish.py +++ b/tests/test_publish.py @@ -583,6 +583,113 @@ def test_folder_exclusion_with_regex(mock_endpoint): constants.FEATURE_FLAG.update(original_flags) +def test_folder_exclusion_with_anchored_regex(mock_endpoint): + """Test that excluding a parent folder with an anchored regex also excludes + items in child folders, preserving consistent hierarchy behavior.""" + + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + + # Create item directly under /legacy (should be excluded - direct match) + legacy_notebook_dir = temp_path / "legacy" / "LegacyNotebook.Notebook" + legacy_notebook_dir.mkdir(parents=True, exist_ok=True) + + legacy_notebook_platform = legacy_notebook_dir / ".platform" + legacy_notebook_metadata = { + "metadata": { + "type": "Notebook", + "displayName": "LegacyNotebook", + "description": "Legacy notebook in excluded parent folder", + }, + "config": {"logicalId": "legacy-notebook-id"}, + } + + with legacy_notebook_platform.open("w", encoding="utf-8") as f: + json.dump(legacy_notebook_metadata, f) + + with (legacy_notebook_dir / "dummy.txt").open("w", encoding="utf-8") as f: + f.write("Dummy file content") + + # Create item in /legacy/archived (should also be excluded - ancestor excluded) + archived_notebook_dir = temp_path / "legacy" / "archived" / "ArchivedNotebook.Notebook" + archived_notebook_dir.mkdir(parents=True, exist_ok=True) + + archived_notebook_platform = archived_notebook_dir / ".platform" + archived_notebook_metadata = { + "metadata": { + "type": "Notebook", + "displayName": "ArchivedNotebook", + "description": "Notebook in child folder of excluded parent - should also be excluded", + }, + "config": {"logicalId": "archived-notebook-id"}, + } + + with archived_notebook_platform.open("w", encoding="utf-8") as f: + json.dump(archived_notebook_metadata, f) + + with (archived_notebook_dir / "dummy.txt").open("w", encoding="utf-8") as f: + f.write("Dummy file content") + + # Create item in /current (should NOT be excluded) + current_notebook_dir = temp_path / "current" / "CurrentNotebook.Notebook" + current_notebook_dir.mkdir(parents=True, exist_ok=True) + + current_notebook_platform = current_notebook_dir / ".platform" + current_notebook_metadata = { + "metadata": { + "type": "Notebook", + "displayName": "CurrentNotebook", + "description": "Current notebook to be included", + }, + "config": {"logicalId": "current-notebook-id"}, + } + + with current_notebook_platform.open("w", encoding="utf-8") as f: + json.dump(current_notebook_metadata, f) + + with (current_notebook_dir / "dummy.txt").open("w", encoding="utf-8") as f: + f.write("Dummy file content") + + with ( + patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint), + patch.object( + FabricWorkspace, "_refresh_deployed_items", new=lambda self: setattr(self, "deployed_items", {}) + ), + patch.object( + FabricWorkspace, "_refresh_deployed_folders", new=lambda self: setattr(self, "deployed_folders", {}) + ), + ): + original_flags = constants.FEATURE_FLAG.copy() + constants.FEATURE_FLAG.add("enable_experimental_features") + constants.FEATURE_FLAG.add("enable_exclude_folder") + + try: + workspace = FabricWorkspace( + workspace_id="12345678-1234-5678-abcd-1234567890ab", + repository_directory=str(temp_path), + item_type_in_scope=["Notebook"], + ) + + # Use an anchored regex that only matches /legacy exactly, + # NOT /legacy/archived directly. The ancestor walk logic should + # still exclude /legacy/archived because its parent /legacy is excluded. + exclude_regex = r"^/legacy$" + publish.publish_all_items(workspace, folder_path_exclude_regex=exclude_regex) + + # Direct match: /legacy is excluded + assert workspace.repository_items["Notebook"]["LegacyNotebook"].skip_publish is True + + # Ancestor excluded: /legacy/archived is excluded because /legacy matches + assert workspace.repository_items["Notebook"]["ArchivedNotebook"].skip_publish is True + + # Unrelated folder: /current is NOT excluded + assert workspace.repository_items["Notebook"]["CurrentNotebook"].skip_publish is False + + finally: + constants.FEATURE_FLAG.clear() + constants.FEATURE_FLAG.update(original_flags) + + def test_item_name_exclusion_still_works(mock_endpoint): """Test that existing item name exclusion still works with the new folder exclusion feature.""" @@ -743,7 +850,7 @@ def test_legacy_folder_exclusion_example(mock_endpoint): ) # Test: Exclude all items in 'legacy' folder using the folder path regex pattern - exclude_regex = r"^legacy/" # Match items that start with 'legacy/' + exclude_regex = r"^/legacy" # Match items that start with '/legacy' publish.publish_all_items(workspace, folder_path_exclude_regex=exclude_regex) # Verify that legacy items were excluded @@ -757,3 +864,335 @@ def test_legacy_folder_exclusion_example(mock_endpoint): # Restore original feature flags constants.FEATURE_FLAG.clear() constants.FEATURE_FLAG.update(original_flags) + + +def test_folder_inclusion_with_folder_path_to_include(mock_endpoint): + """Test that folder_path_to_include only filters items found within a Fabric folder. + Root-level items (not located within any subfolder) are always published, + as folder inclusion can only apply to items that reside inside a folder. + Items in ancestor folders of included paths are excluded even though the + ancestor folder itself is created to preserve hierarchy. + When an ancestor folder is explicitly included, its items are also published.""" + + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + + # Create items in 'active' folder (should be included via folder_path_to_include) + active_notebook_dir = temp_path / "active" / "ActiveNotebook.Notebook" + active_notebook_dir.mkdir(parents=True, exist_ok=True) + + active_notebook_platform = active_notebook_dir / ".platform" + active_notebook_metadata = { + "metadata": { + "type": "Notebook", + "displayName": "ActiveNotebook", + "description": "Active notebook to be included", + }, + "config": {"logicalId": "active-notebook-id"}, + } + + with active_notebook_platform.open("w", encoding="utf-8") as f: + json.dump(active_notebook_metadata, f) + + with (active_notebook_dir / "dummy.txt").open("w", encoding="utf-8") as f: + f.write("Dummy file content") + + active_model_dir = temp_path / "active" / "ActiveModel.SemanticModel" + active_model_dir.mkdir(parents=True, exist_ok=True) + + active_model_platform = active_model_dir / ".platform" + active_model_metadata = { + "metadata": { + "type": "SemanticModel", + "displayName": "ActiveModel", + "description": "Active semantic model to be included", + }, + "config": {"logicalId": "active-model-id"}, + } + + with active_model_platform.open("w", encoding="utf-8") as f: + json.dump(active_model_metadata, f) + + with (active_model_dir / "dummy.txt").open("w", encoding="utf-8") as f: + f.write("Dummy file content") + + # Create items in 'archive' folder (should be excluded - folder not in inclusion list) + archive_notebook_dir = temp_path / "archive" / "ArchivedNotebook.Notebook" + archive_notebook_dir.mkdir(parents=True, exist_ok=True) + + archive_notebook_platform = archive_notebook_dir / ".platform" + archive_notebook_metadata = { + "metadata": { + "type": "Notebook", + "displayName": "ArchivedNotebook", + "description": "Archived notebook to be excluded", + }, + "config": {"logicalId": "archived-notebook-id"}, + } + + with archive_notebook_platform.open("w", encoding="utf-8") as f: + json.dump(archive_notebook_metadata, f) + + with (archive_notebook_dir / "dummy.txt").open("w", encoding="utf-8") as f: + f.write("Dummy file content") + + # Create root-level item (not inside any folder - always published) + root_notebook_dir = temp_path / "RootNotebook.Notebook" + root_notebook_dir.mkdir(parents=True, exist_ok=True) + + root_notebook_platform = root_notebook_dir / ".platform" + root_notebook_metadata = { + "metadata": { + "type": "Notebook", + "displayName": "RootNotebook", + "description": "Root level notebook - always published regardless of folder inclusion", + }, + "config": {"logicalId": "root-notebook-id"}, + } + + with root_notebook_platform.open("w", encoding="utf-8") as f: + json.dump(root_notebook_metadata, f) + + with (root_notebook_dir / "dummy.txt").open("w", encoding="utf-8") as f: + f.write("Dummy file content") + + # Create nested folder structure: /projects/team1/NestedNotebook.Notebook + # Only /projects/team1 is in the include list (not /projects), so: + # - /projects folder is created (ancestor) but items directly under it are excluded + # - /projects/team1 items are included + projects_notebook_dir = temp_path / "projects" / "ProjectNotebook.Notebook" + projects_notebook_dir.mkdir(parents=True, exist_ok=True) + + projects_notebook_platform = projects_notebook_dir / ".platform" + projects_notebook_metadata = { + "metadata": { + "type": "Notebook", + "displayName": "ProjectNotebook", + "description": "Item directly under ancestor folder - should be excluded", + }, + "config": {"logicalId": "projects-notebook-id"}, + } + + with projects_notebook_platform.open("w", encoding="utf-8") as f: + json.dump(projects_notebook_metadata, f) + + with (projects_notebook_dir / "dummy.txt").open("w", encoding="utf-8") as f: + f.write("Dummy file content") + + nested_notebook_dir = temp_path / "projects" / "team1" / "NestedNotebook.Notebook" + nested_notebook_dir.mkdir(parents=True, exist_ok=True) + + nested_notebook_platform = nested_notebook_dir / ".platform" + nested_notebook_metadata = { + "metadata": { + "type": "Notebook", + "displayName": "NestedNotebook", + "description": "Notebook in nested included folder", + }, + "config": {"logicalId": "nested-notebook-id"}, + } + + with nested_notebook_platform.open("w", encoding="utf-8") as f: + json.dump(nested_notebook_metadata, f) + + with (nested_notebook_dir / "dummy.txt").open("w", encoding="utf-8") as f: + f.write("Dummy file content") + + # Create structure where ancestor IS in the include list: /dept/eng/EngNotebook.Notebook + # Both /dept and /dept/eng are in the include list, so items under both should be published + dept_notebook_dir = temp_path / "dept" / "DeptNotebook.Notebook" + dept_notebook_dir.mkdir(parents=True, exist_ok=True) + + dept_notebook_platform = dept_notebook_dir / ".platform" + dept_notebook_metadata = { + "metadata": { + "type": "Notebook", + "displayName": "DeptNotebook", + "description": "Item under explicitly included ancestor folder - should be published", + }, + "config": {"logicalId": "dept-notebook-id"}, + } + + with dept_notebook_platform.open("w", encoding="utf-8") as f: + json.dump(dept_notebook_metadata, f) + + with (dept_notebook_dir / "dummy.txt").open("w", encoding="utf-8") as f: + f.write("Dummy file content") + + eng_notebook_dir = temp_path / "dept" / "eng" / "EngNotebook.Notebook" + eng_notebook_dir.mkdir(parents=True, exist_ok=True) + + eng_notebook_platform = eng_notebook_dir / ".platform" + eng_notebook_metadata = { + "metadata": { + "type": "Notebook", + "displayName": "EngNotebook", + "description": "Item in nested folder under explicitly included ancestor", + }, + "config": {"logicalId": "eng-notebook-id"}, + } + + with eng_notebook_platform.open("w", encoding="utf-8") as f: + json.dump(eng_notebook_metadata, f) + + with (eng_notebook_dir / "dummy.txt").open("w", encoding="utf-8") as f: + f.write("Dummy file content") + + with ( + patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint), + patch.object( + FabricWorkspace, "_refresh_deployed_items", new=lambda self: setattr(self, "deployed_items", {}) + ), + patch.object( + FabricWorkspace, "_refresh_deployed_folders", new=lambda self: setattr(self, "deployed_folders", {}) + ), + ): + # Enable experimental feature flags for folder inclusion + original_flags = constants.FEATURE_FLAG.copy() + constants.FEATURE_FLAG.add("enable_experimental_features") + constants.FEATURE_FLAG.add("enable_include_folder") + + try: + workspace = FabricWorkspace( + workspace_id="12345678-1234-5678-abcd-1234567890ab", + repository_directory=str(temp_path), + item_type_in_scope=["Notebook", "SemanticModel"], + ) + + # Test: Include items in 'active', 'projects/team1', 'dept', and 'dept/eng' folders + publish.publish_all_items( + workspace, + folder_path_to_include=["/active", "/projects/team1", "/dept", "/dept/eng"], + ) + + # Verify that repository_items are populated correctly + assert "Notebook" in workspace.repository_items + assert "SemanticModel" in workspace.repository_items + + # Check that items in the included folder are published (skip_publish = False) + assert workspace.repository_items["Notebook"]["ActiveNotebook"].skip_publish is False + assert workspace.repository_items["SemanticModel"]["ActiveModel"].skip_publish is False + + # Check that items in a non-included folder are excluded (skip_publish = True) + assert workspace.repository_items["Notebook"]["ArchivedNotebook"].skip_publish is True + + # Root-level items are not located within any Fabric folder, so + # folder_path_to_include does not apply to them. They are always + # published regardless of the folder inclusion filter. + assert workspace.repository_items["Notebook"]["RootNotebook"].skip_publish is False + + # Nested folder: items in the included nested path are published + assert workspace.repository_items["Notebook"]["NestedNotebook"].skip_publish is False + + # Ancestor folder: /projects is NOT in the include list (only /projects/team1 is), + # so items directly under /projects are excluded + assert workspace.repository_items["Notebook"]["ProjectNotebook"].skip_publish is True + + # Explicitly included ancestor: /dept IS in the include list, + # so items directly under /dept are published + assert workspace.repository_items["Notebook"]["DeptNotebook"].skip_publish is False + + # Nested folder under explicitly included ancestor: /dept/eng is also + # in the include list, so its items are published too + assert workspace.repository_items["Notebook"]["EngNotebook"].skip_publish is False + + finally: + # Restore original feature flags + constants.FEATURE_FLAG.clear() + constants.FEATURE_FLAG.update(original_flags) + + +def test_folder_inclusion_and_exclusion_together(mock_endpoint): + """Test that using both folder_path_to_include and folder_path_exclude_regex raises InputError.""" + + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + + # Create a minimal item so the workspace can be initialized + deploy_notebook_dir = temp_path / "deploy" / "DeployNotebook.Notebook" + deploy_notebook_dir.mkdir(parents=True, exist_ok=True) + + deploy_platform = deploy_notebook_dir / ".platform" + deploy_metadata = { + "metadata": { + "type": "Notebook", + "displayName": "DeployNotebook", + "description": "Notebook in included folder", + }, + "config": {"logicalId": "deploy-notebook-id"}, + } + + with deploy_platform.open("w", encoding="utf-8") as f: + json.dump(deploy_metadata, f) + + with (deploy_notebook_dir / "dummy.txt").open("w", encoding="utf-8") as f: + f.write("Dummy file content") + + with ( + patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint), + patch.object( + FabricWorkspace, "_refresh_deployed_items", new=lambda self: setattr(self, "deployed_items", {}) + ), + patch.object( + FabricWorkspace, "_refresh_deployed_folders", new=lambda self: setattr(self, "deployed_folders", {}) + ), + ): + original_flags = constants.FEATURE_FLAG.copy() + constants.FEATURE_FLAG.add("enable_experimental_features") + constants.FEATURE_FLAG.add("enable_include_folder") + constants.FEATURE_FLAG.add("enable_exclude_folder") + + try: + workspace = FabricWorkspace( + workspace_id="12345678-1234-5678-abcd-1234567890ab", + repository_directory=str(temp_path), + item_type_in_scope=["Notebook"], + ) + + with pytest.raises( + InputError, + match="Cannot use both 'folder_path_exclude_regex' and 'folder_path_to_include'", + ): + publish.publish_all_items( + workspace, + folder_path_to_include=["/deploy"], + folder_path_exclude_regex=r"^/deploy/legacy", + ) + + finally: + constants.FEATURE_FLAG.clear() + constants.FEATURE_FLAG.update(original_flags) + + +def test_empty_folder_path_to_include_raises_error(mock_endpoint): + """Test that passing an empty list for folder_path_to_include raises an InputError.""" + + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + + with ( + patch("fabric_cicd.fabric_workspace.FabricEndpoint", return_value=mock_endpoint), + patch.object( + FabricWorkspace, "_refresh_deployed_items", new=lambda self: setattr(self, "deployed_items", {}) + ), + patch.object( + FabricWorkspace, "_refresh_deployed_folders", new=lambda self: setattr(self, "deployed_folders", {}) + ), + ): + original_flags = constants.FEATURE_FLAG.copy() + constants.FEATURE_FLAG.add("enable_experimental_features") + constants.FEATURE_FLAG.add("enable_include_folder") + + try: + workspace = FabricWorkspace( + workspace_id="12345678-1234-5678-abcd-1234567890ab", + repository_directory=str(temp_path), + item_type_in_scope=["Notebook"], + ) + + with pytest.raises(InputError, match="folder_path_to_include must not be an empty list"): + publish.publish_all_items(workspace, folder_path_to_include=[]) + + finally: + constants.FEATURE_FLAG.clear() + constants.FEATURE_FLAG.update(original_flags)