diff --git a/src/openjd/model/_range_expr.py b/src/openjd/model/_range_expr.py index f2edf28..f74edac 100644 --- a/src/openjd/model/_range_expr.py +++ b/src/openjd/model/_range_expr.py @@ -54,7 +54,7 @@ def from_str(range_str: str) -> IntRangeExpr: return Parser().parse(range_str) @staticmethod - def from_list(values: list[int | str]) -> IntRangeExpr: + def from_list(values: list[int] | list[str] | list[int | str]) -> IntRangeExpr: """Creates a range expression object from a list of integers/strings containing integers.""" if len(values) == 0: return IntRangeExpr([]) @@ -66,7 +66,7 @@ def from_list(values: list[int | str]) -> IntRangeExpr: values_as_int: list[int] = sorted({int(i) for i in values}) # Find all the ranges, and concatenate them ranges = [] - start = values_as_int[0] + start = end = values_as_int[0] step = None for value in values_as_int[1:]: @@ -78,7 +78,7 @@ def from_list(values: list[int | str]) -> IntRangeExpr: end = value else: ranges.append(IntRange(start, end, step)) - start = value + start = end = value step = None ranges.append(IntRange(start, end, step or 1)) return IntRangeExpr(ranges) @@ -183,8 +183,11 @@ def __init__(self, start: int, end: int, step: int = 1): self._validate() def __str__(self) -> str: - if len(self) == 1: + len_self = len(self) + if len_self == 1: return str(self._start) + elif len_self == 2: + return f"{self._start},{self._end}" elif self.step == 1: return f"{self._start}-{self._end}" else: diff --git a/src/openjd/model/_step_param_space_iter.py b/src/openjd/model/_step_param_space_iter.py index cf4e840..04806da 100644 --- a/src/openjd/model/_step_param_space_iter.py +++ b/src/openjd/model/_step_param_space_iter.py @@ -7,7 +7,7 @@ from dataclasses import dataclass, field from functools import reduce from operator import mul -from typing import AbstractSet, Optional, Union +from typing import AbstractSet, Any, Optional, Union from ._internal import ( CombinationExpressionAssociationNode, @@ -20,8 +20,10 @@ from ._types import ParameterValue, ParameterValueType, StepParameterSpace, TaskParameterSet from .v2023_09 import ( RangeExpressionTaskParameterDefinition as RangeExpressionTaskParameterDefinition_2023_09, + RangeListTaskParameterDefinition as RangeListTaskParameterDefinition_2023_09, + TaskParameterType as TaskParameterType_2023_09, + TaskChunksRangeConstraint as TaskChunksRangeConstraint_2023_09, ) -from .v2023_09 import RangeListTaskParameterDefinition as RangeListTaskParameterDefinition_2023_09 __all__ = ["StepParameterSpaceIterator"] @@ -33,12 +35,26 @@ class StepParameterSpaceIterator(Iterable[TaskParameterSet], Sized): - """The multidimensional space formed by all possible task parameter values. + """An iterator for the multidimensional space of all possible task parameter values, + returning an individual task or a chunk of tasks in sequence. - The iteration order is dictated by the order of each parameter's values, and + Note: The ordering of inputs generated by this iterator is **NOT** + guaranteed to be invariant across versions of this library. + + In most cases, the iteration order is dictated by the order of each parameter's values, and the order of the expressions within the combination expression of the StepParameterSpace. Ordering is row-major (right-most moves fastest) for products ('*') in the expression. + An exception to this rule occurs with adaptive chunked iteration. The iteration order is different + when the parameter space includes a CHUNK[INT] parameter, and its `chunks.targetRuntimeSeconds` + value is non-zero. No matter where the CHUNK[INT] parameter is place in the combination expression, + it becomes the innermost iteration dimension in order to keep the implementation simple and + memory-efficient. + + In the adaptive chunked iteration case, the iterator's `chunks_default_task_count` value is mutable + so that the code using it can collect task runtime statistics and adapt the size of the chunk to more + closely match the target runtime. + For example, given: A = [1,2,3] B = [1,2] @@ -65,15 +81,23 @@ class StepParameterSpaceIterator(Iterable[TaskParameterSet], Sized): """ _parameters: dict[str, TaskParameter] - _expr_tree: Union[Node, list] + _expr_tree: Node + _iter: NodeIterator _parsedtree: CombinationExpressionNode + _chunks_adaptive: bool + _chunks_default_task_count: Optional[int] + def __init__(self, *, space: Optional[StepParameterSpace]): + self._chunks_adaptive = False + self._chunks_default_task_count = None + # Special case the zero-dimensional space with one element if space is None: self._parameters = {} self._parsetree = None - self._expr_tree = [{}] + self._expr_tree = ZeroDimSpaceNode() + self._iter = self._expr_tree.iter() else: if space.combination is None: # space.taskParameterDefinitions is a dict[str,TaskParameter] @@ -82,41 +106,78 @@ def __init__(self, *, space: Optional[StepParameterSpace]): combination = space.combination self._parameters = dict(space.taskParameterDefinitions) + # Determine whether and what kind of chunking to do + for name, param in self._parameters.items(): + if param.type == TaskParameterType_2023_09.CHUNK_INT: + if param.chunks is None: + raise ValueError( + f"CHUNK[INT] parameter '{name}' must have a chunks definition." + ) + self._chunks_adaptive = ( + param.chunks.targetRuntimeSeconds is not None + and int(param.chunks.targetRuntimeSeconds) > 0 + ) + self._chunks_default_task_count = int(param.chunks.defaultTaskCount) + break + # Raises: TokenError, ExpressionError self._parsetree = CombinationExpressionParser().parse(combination) - self._expr_tree = self._create_expr_tree(self._parsetree) + self._expr_tree = self._create_expr_tree( + self._parsetree, + chunks_adaptive=self._chunks_adaptive, + chunks_default_task_count=self._chunks_default_task_count, + ) + self._iter = self._expr_tree.iter() @property def names(self) -> AbstractSet[str]: """Get the names of all parameters in the parameter space.""" return self._parameters.keys() - def __iter__(self) -> Iterator[TaskParameterSet]: - """Obtain an iterator that will iterate over every task parameter set - in this parameter space. + @property + def chunks_adaptive(self) -> bool: + """True if the parameter space includes a CHUNK[INT] parameter with a non-zero target runtime.""" + return self._chunks_adaptive + + @property + def chunks_default_task_count(self) -> Optional[int]: + """The default task count for the CHUNK[INT] parameter, if any. - Note: The ordering of inputs generated by this iterator is **NOT** - guaranteed to be invariant across versions of this library. + Returns: + Optional[int]: The default task count, or None if the parameter space does not include a CHUNK[INT] parameter. """ + return self._chunks_default_task_count + + @chunks_default_task_count.setter + def chunks_default_task_count(self, value: int): + """Set the default task count for the CHUNK[INT] parameter. - class Iter: - _root: NodeIterator + Args: + value (int): The new default task count. - def __init__(self, root: Node): - self._root = root.iter() + Raises: + ValueError: If the parameter space does not use adaptive chunking, i.e. a CHUNK[INT] parameter with a target runtime. + """ + if not self._chunks_adaptive: + raise ValueError( + "The parameter space does not use adaptive chunking, so cannot modify chunks_default_task_count." + ) + if not (isinstance(value, int) and value > 0): + raise ValueError("chunks_default_task_count must be a positive integer.") - def __iter__(self): # pragma: no cover - return self + self._expr_tree.set_chunks_default_task_count(value) + self._chunks_default_task_count = value - def __next__(self) -> TaskParameterSet: - result: TaskParameterSet = TaskParameterSet() - self._root.next(result) - return result + def reset_iter(self) -> None: + self._iter.reset_iter() - if isinstance(self._expr_tree, Node): - return Iter(self._expr_tree) - else: - return iter(self._expr_tree) + def __iter__(self) -> StepParameterSpaceIterator: + return self + + def __next__(self) -> TaskParameterSet: + result: TaskParameterSet = TaskParameterSet() + self._iter.next(result) + return result def __len__(self) -> int: """The number of task parameter sets that are defined by this parameter space""" @@ -124,6 +185,7 @@ def __len__(self) -> int: def __getitem__(self, index: int) -> TaskParameterSet: """Get a specific task parameter set given an index. + This cannot be used when adaptive chunking is active. Note: The ordering of inputs is **NOT** guaranteed to be invariant across versions of this library. @@ -134,17 +196,78 @@ def __getitem__(self, index: int) -> TaskParameterSet: Returns: dict[str, Union[int, float, str]]: Values of every task parameter. Dictionary key is the parameter name. + + Raises: + RuntimeError: When getitem is called but adaptive chunking is active. """ return self._expr_tree[index] - def _create_expr_tree(self, root: CombinationExpressionNode) -> Node: + def _create_expr_tree( + self, + root: CombinationExpressionNode, + chunks_adaptive: bool, + chunks_default_task_count: Optional[int], + ) -> Node: """Recursively make a copy of the given Parser-generated expression tree using the Node types defined in this file. """ if isinstance(root, CombinationExpressionIdentifierNode): name = root.parameter parameter = self._parameters[name] - if isinstance(parameter.range, list): + if parameter.type == TaskParameterType_2023_09.CHUNK_INT: + if parameter.chunks is None: + raise ValueError( + f"CHUNK[INT] parameter '{name}' must have a chunks definition." + ) + if chunks_default_task_count is None: + raise ValueError( + f"CHUNK[INT] parameter '{name}' must have a default task count." + ) + # Expand the range to a list + if isinstance(parameter.range, list): + parameter_range: list[int] = [int(v) for v in parameter.range] + else: + parameter_range = list[int](IntRangeExpr.from_str(parameter.range)) + + if chunks_adaptive: + if ( + parameter.chunks.rangeConstraint + == TaskChunksRangeConstraint_2023_09.CONTIGUOUS + ): + return AdaptiveContiguousChunkIdentifierNode( + name=name, + type=ParameterValueType(parameter.type), + range=parameter_range, + chunks_default_task_count=chunks_default_task_count, + ) + else: + return AdaptiveNoncontiguousChunkIdentifierNode( + name=name, + type=ParameterValueType(parameter.type), + range=parameter_range, + chunks_default_task_count=chunks_default_task_count, + ) + else: + # Divide the range into contiguous or noncontiguous chunks + if ( + parameter.chunks.rangeConstraint + == TaskChunksRangeConstraint_2023_09.CONTIGUOUS + ): + chunk_list = divide_int_list_into_contiguous_chunks( + chunks_default_task_count, parameter_range + ) + else: + chunk_list = divide_int_list_into_noncontiguous_chunks( + chunks_default_task_count, parameter_range + ) + + # With the chunks determined, we can use the range list node for iteration + return RangeListIdentifierNode( + name, + ParameterValueType(parameter.type), + chunk_list, + ) + elif isinstance(parameter.range, list): return RangeListIdentifierNode( name, ParameterValueType(parameter.type), @@ -157,10 +280,38 @@ def _create_expr_tree(self, root: CombinationExpressionNode) -> Node: parameter.range, ) elif isinstance(root, CombinationExpressionAssociationNode): - return AssociationNode(tuple(self._create_expr_tree(child) for child in root.children)) + return AssociationNode( + tuple( + self._create_expr_tree(child, chunks_adaptive, chunks_default_task_count) + for child in root.children + ), + ) else: assert isinstance(root, CombinationExpressionProductNode) - return ProductNode(tuple(self._create_expr_tree(child) for child in root.children)) + # For adaptive chunking, shift the chunked parameter to the innermost dimension + if chunks_adaptive: + children: Any = [] + chunked_child = None + for child in root.children: + if ( + isinstance(child, CombinationExpressionIdentifierNode) + and self._parameters[child.parameter].type + == TaskParameterType_2023_09.CHUNK_INT + ): + chunked_child = child + else: + children.append(child) + if chunked_child is not None: + children.append(chunked_child) + else: + children = root.children + + return ProductNode( + tuple( + self._create_expr_tree(child, chunks_adaptive, chunks_default_task_count) + for child in children + ), + ) def __eq__(self, other: object) -> bool: # For assisting unit testing @@ -171,6 +322,84 @@ def __eq__(self, other: object) -> bool: ) +def divide_chunk_sizes(chunks_default_task_count: int, task_count: int) -> list[int]: + """Determines evenly distributed chunk lengths for a specified task count divided + into the requested number of chunks.""" + if chunks_default_task_count <= 0: + chunk_count = 1 + else: + chunk_count = (task_count + chunks_default_task_count - 1) // chunks_default_task_count + + if chunk_count >= task_count: + return [1] * task_count + elif chunk_count <= 1: + return [task_count] + + # Each chunk will be one of two sizes, small_chunk_size or small_chunk_size + 1 + small_chunk_size, leftovers = divmod(task_count, chunk_count) + # Create a list of chunk sizes starting from smaller_chunk_size + chunk_sizes = [small_chunk_size] * chunk_count + # Distribute the leftovers evenly across the chunks + for i in range(leftovers): + chunk_sizes[(i * chunk_count) // leftovers] += 1 + return chunk_sizes + + +def divide_int_interval_into_chunks( + chunks_default_task_count: int, first_value: int, last_value: int +) -> list[str]: + """Divide the provided value interval into CONTIGUOUS chunks, given the default chunk size to use. Every + chunk is in the form of "-" even if the first and last values are equal. + """ + task_count = last_value - first_value + 1 + chunk_sizes = divide_chunk_sizes(chunks_default_task_count, task_count) + chunks = [] + for chunk_size in chunk_sizes: + chunks.append(f"{first_value}-{first_value + chunk_size - 1}") + first_value += chunk_size + return chunks + + +def divide_int_list_into_contiguous_chunks( + chunks_default_task_count: int, values: list[int] +) -> list[str]: + """Divide the provided list of values into CONTIGUOUS chunks, given the default chunk size to use. Every + chunk is in the form of "-" even if the first and last values are equal. + """ + if len(values) == 0: + return [] + + interval_starts = [0] + interval_starts.extend( + i + 1 for i, (v0, v1) in enumerate(zip(values, values[1:])) if v1 != v0 + 1 + ) + interval_starts.append(len(values)) + chunks = [] + for interval_index in range(len(interval_starts) - 1): + index_start, index_end = ( + interval_starts[interval_index], + interval_starts[interval_index + 1] - 1, + ) + value_start, value_end = (values[index_start], values[index_end]) + chunks.extend( + divide_int_interval_into_chunks(chunks_default_task_count, value_start, value_end) + ) + return chunks + + +def divide_int_list_into_noncontiguous_chunks( + chunks_default_task_count: int, values: list[int] +) -> list[str]: + """Divide the provided list of values into NONCONTIGUOUS chunks, given the default chunk size to use.""" + chunk_sizes = divide_chunk_sizes(chunks_default_task_count, len(values)) + chunks = [] + index_start = 0 + for chunk_size in chunk_sizes: + chunks.append(str(IntRangeExpr.from_list(values[index_start : index_start + chunk_size]))) + index_start += chunk_size + return chunks + + # ------- # Mirror classes for the parsed combination expression tree. # We create these to separate out the functionality required by this class @@ -186,6 +415,11 @@ class NodeIterator(ABC): appropriate key/values in it. With the standard iterator we would have __next__ returning a dict and end up with a tonne of intermediary dicts created as we ran __next__ on the entire tree. + + When iterating with adaptive chunking, the `chunks_default_task_count` + is not None, and can be modified. It's the responsibility of the caller + to perform any statistics and heuristics to determine updates to the chunk + size. """ @abstractmethod @@ -206,6 +440,53 @@ def __getitem__(self, index: int) -> TaskParameterSet: def iter(self) -> NodeIterator: raise NotImplementedError("Base class") # pragma: no cover + def set_chunks_default_task_count(self, value: int) -> None: + raise ValueError( + "The parameter space does not use adaptive chunking, so cannot modify chunks_default_task_count." + ) + + +class ZeroDimSpaceIter(NodeIterator): + """Iterator for a zero-dimensional space + + Attributes: + _first_value: True if and only if we have not yet evaluated the first value of the iterator. + """ + + _first_value: bool + + def __init__(self): + self._first_value = True + + def reset_iter(self) -> None: + self._first_value = True + + def next(self, result: TaskParameterSet) -> None: + if self._first_value: + self._first_value = False + result.clear() + else: + raise StopIteration + + +class ZeroDimSpaceNode(Node): + """A zero-dimensional space""" + + def __init__(self): + pass + + def __len__(self) -> int: + return 1 + + def __getitem__(self, index: int) -> TaskParameterSet: + if -1 <= index <= 0: + return TaskParameterSet() + else: + raise IndexError(f"index {index} is out of range") + + def iter(self) -> NodeIterator: + return ZeroDimSpaceIter() + class ProductNodeIter(NodeIterator): """Iterator for a ProductNode @@ -247,7 +528,7 @@ def next(self, result: TaskParameterSet) -> None: # A = [1,2]; B = [3,4]; C = [5,6]; and # expr="A * B * C" # then the parameters on the right change more rapidly - # than the parmeters on the left. In this case our traversal + # than the parameters on the left. In this case our traversal # order is: # A | B | C # --------------- @@ -329,6 +610,12 @@ def __getitem__(self, index: int) -> TaskParameterSet: def iter(self) -> ProductNodeIter: return ProductNodeIter(self.children) + def set_chunks_default_task_count(self, value: int) -> None: + # If chunks_default_task_count is settable, the last child is + # an AdaptiveChunkIdentifierNode and will accept the value, + # otherwise it will raise an exception. + self.children[-1].set_chunks_default_task_count(value) + class AssociationNodeIter(NodeIterator): """Iterator for an AssociationNode @@ -377,7 +664,7 @@ class RangeListIdentifierNodeIterator(NodeIterator): Attributes: _it: Iterator for the corresponding task parameter - _parameter: The RangeListIdentifierNode this is iterating over. + _node: The RangeListIdentifierNode this is iterating over. """ _it: Iterator[str] @@ -421,7 +708,7 @@ class RangeExpressionIdentifierNodeIterator(NodeIterator): Attributes: _it: Iterator for the corresponding task parameter - _parameter: The RangeExpressionIdentifierNode this is iterating over. + _node: The RangeExpressionIdentifierNode this is iterating over. """ _it: Iterator[int] @@ -460,3 +747,128 @@ def __getitem__(self, index: int) -> TaskParameterSet: def iter(self) -> RangeExpressionIdentifierNodeIterator: return RangeExpressionIdentifierNodeIterator(self) + + +class AdaptiveContiguousChunkIdentifierNodeIterator(NodeIterator): + """Iterator for an AdaptiveChunkIdentifierNode + + Attributes: + _index: The index into _node.range for the start of the next chunk. + _node: The AdaptiveChunkIdentifierNode this is iterating over. + """ + + _index: int + _node: AdaptiveContiguousChunkIdentifierNode + + def __init__(self, node: AdaptiveContiguousChunkIdentifierNode): + self._node = node + self.reset_iter() + + def reset_iter(self) -> None: + self._index = 0 + + def next(self, result: TaskParameterSet) -> None: + if self._index >= len(self._node.range): + raise StopIteration() + + chunks_default_task_count = self._node.chunks_default_task_count + parameter_range = self._node.range + range_len = len(parameter_range) + first_index = self._index + first_v = last_v = parameter_range[first_index] + + # Step through the range values until reaching the chunk size or a non-contiguous value + index = first_index + 1 + while ( + (index - first_index) < chunks_default_task_count + and index < range_len + and parameter_range[index] == last_v + 1 + ): + last_v += 1 + index += 1 + + self._index = index + result[self._node.name] = ParameterValue(type=self._node.type, value=f"{first_v}-{last_v}") + + +@dataclass +class AdaptiveContiguousChunkIdentifierNode(Node): + name: str + type: ParameterValueType + range: list[int] + chunks_default_task_count: int + """This value can be modified by the caller to adapt the chunk size.""" + + def __len__(self) -> int: + raise ValueError( + "Length is not available because the parameter space uses adaptive chunking." + ) + + def __getitem__(self, index: int) -> TaskParameterSet: + raise LookupError( + "Items cannot be retrieved by index because the parameter space uses adaptive chunking." + ) + + def iter(self) -> AdaptiveContiguousChunkIdentifierNodeIterator: + return AdaptiveContiguousChunkIdentifierNodeIterator(self) + + def set_chunks_default_task_count(self, value: int) -> None: + self.chunks_default_task_count = value + + +class AdaptiveNoncontiguousChunkIdentifierNodeIterator(NodeIterator): + """Iterator for an AdaptiveNoncontiguousChunkIdentifierNode + + Attributes: + _index: The index into _node.range for the start of the next chunk. + _node: The AdaptiveNoncontiguousChunkIdentifierNode this is iterating over. + """ + + _index: int + _node: AdaptiveNoncontiguousChunkIdentifierNode + + def __init__(self, node: AdaptiveNoncontiguousChunkIdentifierNode): + self._node = node + self.reset_iter() + + def reset_iter(self) -> None: + self._index = 0 + + def next(self, result: TaskParameterSet) -> None: + if self._index >= len(self._node.range): + raise StopIteration() + + # Form a range expression of the default chunk size or until the end of the range + first_index = self._index + next_first_index = min( + first_index + self._node.chunks_default_task_count, len(self._node.range) + ) + v = str(IntRangeExpr.from_list(self._node.range[first_index:next_first_index])) + + self._index = next_first_index + result[self._node.name] = ParameterValue(type=self._node.type, value=v) + + +@dataclass +class AdaptiveNoncontiguousChunkIdentifierNode(Node): + name: str + type: ParameterValueType + range: list[int] + chunks_default_task_count: int + """This value can be modified by the caller to adapt the chunk size.""" + + def __len__(self) -> int: + raise ValueError( + "Length is not available because the parameter space uses adaptive chunking." + ) + + def __getitem__(self, index: int) -> TaskParameterSet: + raise LookupError( + "Items cannot be retrieved by index because the parameter space uses adaptive chunking." + ) + + def iter(self) -> AdaptiveNoncontiguousChunkIdentifierNodeIterator: + return AdaptiveNoncontiguousChunkIdentifierNodeIterator(self) + + def set_chunks_default_task_count(self, value: int) -> None: + self.chunks_default_task_count = value diff --git a/src/openjd/model/_types.py b/src/openjd/model/_types.py index 7742953..f7b4195 100644 --- a/src/openjd/model/_types.py +++ b/src/openjd/model/_types.py @@ -75,6 +75,8 @@ class ParameterValueType(str, Enum): INT = "INT" FLOAT = "FLOAT" PATH = "PATH" + # This type is only used for task parameters, not job parameters + CHUNK_INT = "CHUNK[INT]" @dataclass(frozen=True, **dataclass_kwargs) diff --git a/src/openjd/model/v2023_09/__init__.py b/src/openjd/model/v2023_09/__init__.py index 704e525..7971bda 100644 --- a/src/openjd/model/v2023_09/__init__.py +++ b/src/openjd/model/v2023_09/__init__.py @@ -75,6 +75,7 @@ StringRangeList, StringTaskParameterDefinition, TaskChunksDefinition, + TaskChunksRangeConstraint, TaskParameterList, TaskParameterStringValue, TaskParameterStringValueAsJob, @@ -157,6 +158,7 @@ "StringRangeList", "StringTaskParameterDefinition", "TaskChunksDefinition", + "TaskChunksRangeConstraint", "TaskParameterList", "TaskParameterStringValue", "TaskParameterStringValueAsJob", diff --git a/src/openjd/model/v2023_09/_model.py b/src/openjd/model/v2023_09/_model.py index 8de45dd..57dd9ab 100644 --- a/src/openjd/model/v2023_09/_model.py +++ b/src/openjd/model/v2023_09/_model.py @@ -584,6 +584,8 @@ def _validate_default_task_count(cls, value: Any) -> Any: @field_validator("targetRuntimeSeconds", mode="before") @classmethod def _validate_target_runtime_seconds(cls, value: Any) -> Any: + if value is None: + return value return validate_int_fmtstring_field(value, ge=0) @@ -909,6 +911,9 @@ class StepParameterSpaceDefinition(OpenJDModel_v2023_09): @field_validator("taskParameterDefinitions") @classmethod def _validate_parameters(cls, v: TaskParameterList) -> TaskParameterList: + # Only one CHUNK[INT] parameter is permitted + if len([param for param in v if param.type == TaskParameterType.CHUNK_INT]) > 1: + raise ValueError("Only one CHUNK[INT] task parameter is permitted") # Must have a unique name for each Task parameter return validate_unique_elements(v, item_value=lambda v: v.name, property="name") diff --git a/test/openjd/model/_internal/test_range_expr.py b/test/openjd/model/_internal/test_range_expr.py index 33b12a3..7d385eb 100644 --- a/test/openjd/model/_internal/test_range_expr.py +++ b/test/openjd/model/_internal/test_range_expr.py @@ -128,7 +128,8 @@ def test_parse_one_positive_range_no_step( "range_expr,start,end,total_range,range_str", [ pytest.param("1-100,101-200", 1, 200, 200, "1-200"), - pytest.param("0-1,3-4,7-9,10", 0, 10, 8, "0-1,3-4,7-10"), + pytest.param("0-1,3-4,7-9,10", 0, 10, 8, "0,1,3,4,7-10"), + pytest.param("0-3:3,5-10:5,12,13,14,15", 0, 15, 8, "0,3,5,10,12-15"), pytest.param("20-29,0-9,10-19", 0, 29, 30, "0-29"), ], ) @@ -208,6 +209,7 @@ def test_range_expr_from_str(self, range_input_str: str, range_str: str): "range_list,range_str", [ pytest.param([5], "5", id="one int"), + pytest.param([1, 2, 3, 4, 5, 7], "1-5,7", id="two ranges"), pytest.param(["7"], "7", id="one int as a str"), pytest.param([9, 0, 3, 2, 8, 10, 1, 4, 7, 6, 5], "0-10", id="values 0-10 out of order"), pytest.param( diff --git a/test/openjd/model/test_step_param_space_iter.py b/test/openjd/model/test_step_param_space_iter.py index be06bc1..3b997a2 100644 --- a/test/openjd/model/test_step_param_space_iter.py +++ b/test/openjd/model/test_step_param_space_iter.py @@ -12,14 +12,10 @@ parse_model, ) -from openjd.model.v2023_09 import JobTemplate as JobTemplate_2023_09 from openjd.model.v2023_09 import ( + JobTemplate as JobTemplate_2023_09, RangeExpressionTaskParameterDefinition as RangeExpressionTaskParameterDefinition_2023_09, -) -from openjd.model.v2023_09 import ( RangeListTaskParameterDefinition as RangeListTaskParameterDefinition_2023_09, -) -from openjd.model.v2023_09 import ( StepParameterSpace as StepParameterSpace_2023_09, ) @@ -68,13 +64,14 @@ def test_no_param_iteration(self): job = create_job(job_template=job_template, job_parameter_values=dict()) space = job.steps[0].parameterSpace - iterator = StepParameterSpaceIterator(space=space) # WHEN - result = list(iterator) + it = StepParameterSpaceIterator(space=space) # THEN - assert result == expected + assert list(it) == expected + it.reset_iter() + assert list(it) == expected def test_no_param_getelem(self): # GIVEN @@ -90,16 +87,16 @@ def test_no_param_getelem(self): space = job.steps[0].parameterSpace # WHEN - result = StepParameterSpaceIterator(space=space) + it = StepParameterSpaceIterator(space=space) # THEN with pytest.raises(IndexError): - result[1] + it[1] with pytest.raises(IndexError): - result[-2] + it[-2] expected = {} - assert result[0] == expected - assert result[-1] == expected + assert it[0] == expected + assert it[-1] == expected @pytest.mark.parametrize( "range_int_param", @@ -130,6 +127,9 @@ def test_single_param_iteration(self, range_int_param): } == next(it), f"i = {i}" with pytest.raises(StopIteration): next(it) + # The chunks parameter is only relevant when the parameter space is chunked + with pytest.raises(ValueError): + it.chunks_default_task_count = 1 @pytest.mark.parametrize("param_range", [["10"], ["10", "11", "12", "13", "14", "15"]]) def test_single_param_getelem(self, param_range): diff --git a/test/openjd/model/test_step_param_space_iter_with_chunks.py b/test/openjd/model/test_step_param_space_iter_with_chunks.py new file mode 100644 index 0000000..9bd16f4 --- /dev/null +++ b/test/openjd/model/test_step_param_space_iter_with_chunks.py @@ -0,0 +1,556 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +import pytest +from typing import Any + +from openjd.model import ( + ParameterValue, + ParameterValueType, + StepParameterSpaceIterator, +) + +from openjd.model.v2023_09 import ( + RangeExpressionTaskParameterDefinition as RangeExpressionTaskParameterDefinition_2023_09, + RangeListTaskParameterDefinition as RangeListTaskParameterDefinition_2023_09, + StepParameterSpace as StepParameterSpace_2023_09, + TaskChunksDefinition as TaskChunksDefinition_2023_09, + TaskChunksRangeConstraint as TaskChunksRangeConstraint_2023_09, +) + +from openjd.model._step_param_space_iter import ( + divide_chunk_sizes, + divide_int_interval_into_chunks, + divide_int_list_into_contiguous_chunks, + divide_int_list_into_noncontiguous_chunks, +) + + +PARAMETRIZE_CASES: tuple = ( + pytest.param( + RangeListTaskParameterDefinition_2023_09( + type=ParameterValueType.CHUNK_INT, + range=["1", "2"], + chunks=TaskChunksDefinition_2023_09( + defaultTaskCount=1, rangeConstraint=TaskChunksRangeConstraint_2023_09.CONTIGUOUS + ), + ), + ["1-1", "2-2"], + False, + id="contig chunks, chunksize 1, range is short list", + ), + pytest.param( + RangeListTaskParameterDefinition_2023_09( + type=ParameterValueType.CHUNK_INT, + range=["1", "2"], + chunks=TaskChunksDefinition_2023_09( + defaultTaskCount=2, rangeConstraint=TaskChunksRangeConstraint_2023_09.CONTIGUOUS + ), + ), + ["1-2"], + False, + id="contig chunks, chunksize 2, range is short list", + ), + pytest.param( + RangeExpressionTaskParameterDefinition_2023_09( + type=ParameterValueType.CHUNK_INT, + range="1-2", + chunks=TaskChunksDefinition_2023_09( + defaultTaskCount=1, rangeConstraint=TaskChunksRangeConstraint_2023_09.CONTIGUOUS + ), + ), + ["1-1", "2-2"], + False, + id="contig chunks, chunksize 1, range is short range expr", + ), + pytest.param( + RangeExpressionTaskParameterDefinition_2023_09( + type=ParameterValueType.CHUNK_INT, + range="1-2", + chunks=TaskChunksDefinition_2023_09( + defaultTaskCount=2, rangeConstraint=TaskChunksRangeConstraint_2023_09.CONTIGUOUS + ), + ), + ["1-2"], + False, + id="contig chunks, chunksize 2, range is short range expr", + ), + pytest.param( + RangeListTaskParameterDefinition_2023_09( + type=ParameterValueType.CHUNK_INT, + range=["1", "2"], + chunks=TaskChunksDefinition_2023_09( + defaultTaskCount=1, rangeConstraint=TaskChunksRangeConstraint_2023_09.NONCONTIGUOUS + ), + ), + ["1", "2"], + False, + id="noncontig chunks, chunksize 1, range is short list", + ), + pytest.param( + RangeListTaskParameterDefinition_2023_09( + type=ParameterValueType.CHUNK_INT, + range=["1", "2"], + chunks=TaskChunksDefinition_2023_09( + defaultTaskCount=2, rangeConstraint=TaskChunksRangeConstraint_2023_09.NONCONTIGUOUS + ), + ), + ["1,2"], + False, + id="noncontig chunks, chunksize 2, range is short list", + ), + pytest.param( + RangeExpressionTaskParameterDefinition_2023_09( + type=ParameterValueType.CHUNK_INT, + range="1-2", + chunks=TaskChunksDefinition_2023_09( + defaultTaskCount=1, rangeConstraint=TaskChunksRangeConstraint_2023_09.NONCONTIGUOUS + ), + ), + ["1", "2"], + False, + id="noncontig chunks, chunksize 1, range is short range expr", + ), + pytest.param( + RangeExpressionTaskParameterDefinition_2023_09( + type=ParameterValueType.CHUNK_INT, + range="1-2", + chunks=TaskChunksDefinition_2023_09( + defaultTaskCount=2, rangeConstraint=TaskChunksRangeConstraint_2023_09.NONCONTIGUOUS + ), + ), + ["1,2"], + False, + id="noncontig chunks, chunksize 2, range is short range expr", + ), + pytest.param( + RangeExpressionTaskParameterDefinition_2023_09( + type=ParameterValueType.CHUNK_INT, + range="1,3,5", + chunks=TaskChunksDefinition_2023_09( + defaultTaskCount=100, rangeConstraint=TaskChunksRangeConstraint_2023_09.CONTIGUOUS + ), + ), + ["1-1", "3-3", "5-5"], + False, + id="contig chunks, chunksize 100, range is noncontig", + ), + pytest.param( + RangeExpressionTaskParameterDefinition_2023_09( + type=ParameterValueType.CHUNK_INT, + range="1,3,5", + chunks=TaskChunksDefinition_2023_09( + defaultTaskCount=100, + rangeConstraint=TaskChunksRangeConstraint_2023_09.NONCONTIGUOUS, + ), + ), + ["1-5:2"], + False, + id="noncontig chunks, chunksize 100, range is noncontig", + ), + pytest.param( + RangeExpressionTaskParameterDefinition_2023_09( + type=ParameterValueType.CHUNK_INT, + range="1-35", + chunks=TaskChunksDefinition_2023_09( + defaultTaskCount=10, rangeConstraint=TaskChunksRangeConstraint_2023_09.CONTIGUOUS + ), + ), + # Non-adaptive spreads out the chunks evenly + ["1-9", "10-18", "19-27", "28-35"], + False, + id="contig chunks, chunksize 10, range 1-35, non-adaptive", + ), + pytest.param( + RangeExpressionTaskParameterDefinition_2023_09( + type=ParameterValueType.CHUNK_INT, + range="1-35", + chunks=TaskChunksDefinition_2023_09( + defaultTaskCount=10, + targetRuntimeSeconds=20, + rangeConstraint=TaskChunksRangeConstraint_2023_09.NONCONTIGUOUS, + ), + ), + # Adaptive makes chunks as big as possible, so the last chunk ends up smaller + ["1-10", "11-20", "21-30", "31-35"], + True, + id="noncontig chunks, chunksize 10, range 1-35, adaptive", + ), + pytest.param( + RangeExpressionTaskParameterDefinition_2023_09( + type=ParameterValueType.CHUNK_INT, + range="-20--5", + chunks=TaskChunksDefinition_2023_09( + defaultTaskCount=5, rangeConstraint=TaskChunksRangeConstraint_2023_09.CONTIGUOUS + ), + ), + # Non-adaptive spreads out the chunks evenly + ["-20--17", "-16--13", "-12--9", "-8--5"], + False, + id="contig chunks, chunksize 5, negative frames, non-adaptive", + ), + pytest.param( + RangeExpressionTaskParameterDefinition_2023_09( + type=ParameterValueType.CHUNK_INT, + range="-20--5", + chunks=TaskChunksDefinition_2023_09( + defaultTaskCount=5, + targetRuntimeSeconds=20, + rangeConstraint=TaskChunksRangeConstraint_2023_09.CONTIGUOUS, + ), + ), + # Adaptive makes chunks as big as possible, so the last chunk ends up smaller + ["-20--16", "-15--11", "-10--6", "-5--5"], + True, + id="contig chunks, chunksize 5, negative frames, adaptive", + ), + pytest.param( + RangeExpressionTaskParameterDefinition_2023_09( + type=ParameterValueType.CHUNK_INT, + range="-20--5", + chunks=TaskChunksDefinition_2023_09( + defaultTaskCount=5, + targetRuntimeSeconds=20, + rangeConstraint=TaskChunksRangeConstraint_2023_09.NONCONTIGUOUS, + ), + ), + # Adaptive makes chunks as big as possible, so the last chunk ends up smaller + ["-20--16", "-15--11", "-10--6", "-5"], + True, + id="noncontig chunks, chunksize 5, negative frames, adaptive", + ), +) + + +@pytest.mark.parametrize("range_int_param,expected,chunks_adaptive", PARAMETRIZE_CASES) +def test_single_param_chunked_iteration(range_int_param, expected, chunks_adaptive): + # GIVEN + space = StepParameterSpace_2023_09( + taskParameterDefinitions={ + "Param1": range_int_param, + } + ) + + # WHEN + it = StepParameterSpaceIterator(space=space) + + # THEN + assert it.chunks_adaptive == chunks_adaptive + assert it.chunks_default_task_count == range_int_param.chunks.defaultTaskCount + # Check that full iteration over the range gives the expected result + assert [v for v in it] == [ + {"Param1": ParameterValue(type=ParameterValueType.CHUNK_INT, value=v)} for v in expected + ] + # Check that resetting the iterator and re-iterating produces the same again + it.reset_iter() + assert [v for v in it] == [ + {"Param1": ParameterValue(type=ParameterValueType.CHUNK_INT, value=v)} for v in expected + ] + # Check that the length and indexing work as expected + it.reset_iter() + if chunks_adaptive: + # With adaptive chunking, it can't retrieve the length or use the indexing operator + with pytest.raises(ValueError): + len(it) + with pytest.raises(LookupError): + it[0] + else: + assert len(it) == len(expected) + for i, v in enumerate(expected): + assert it[i] == {"Param1": ParameterValue(type=ParameterValueType.CHUNK_INT, value=v)} + assert it[i - len(expected)] == { + "Param1": ParameterValue(type=ParameterValueType.CHUNK_INT, value=v) + } + # Without adaptive chunking, the chunk size is fixed + with pytest.raises(ValueError): + it.chunks_default_task_count = 1 + + +PARAMETRIZE_CASES = ( + pytest.param( + { + "Param1": RangeListTaskParameterDefinition_2023_09( + type=ParameterValueType.CHUNK_INT, + range=["1", "2"], + chunks=TaskChunksDefinition_2023_09( + defaultTaskCount=1, rangeConstraint=TaskChunksRangeConstraint_2023_09.CONTIGUOUS + ), + ), + "Param2": RangeListTaskParameterDefinition_2023_09( + type=ParameterValueType.STRING, + range=["A", "B"], + ), + }, + [("1-1", "A"), ("1-1", "B"), ("2-2", "A"), ("2-2", "B")], + False, + id="2 dim, chunked outer, chunksize 1, non-adaptive", + ), + pytest.param( + { + "Param1": RangeListTaskParameterDefinition_2023_09( + type=ParameterValueType.CHUNK_INT, + range=["1", "2"], + chunks=TaskChunksDefinition_2023_09( + defaultTaskCount=2, rangeConstraint=TaskChunksRangeConstraint_2023_09.CONTIGUOUS + ), + ), + "Param2": RangeListTaskParameterDefinition_2023_09( + type=ParameterValueType.STRING, + range=["A", "B"], + ), + }, + [("1-2", "A"), ("1-2", "B")], + False, + id="2 dim, chunked outer, chunksize 2, non-adaptive", + ), + pytest.param( + { + "Param1": RangeListTaskParameterDefinition_2023_09( + type=ParameterValueType.CHUNK_INT, + range=["1", "2"], + chunks=TaskChunksDefinition_2023_09( + defaultTaskCount=1, + targetRuntimeSeconds=20, + rangeConstraint=TaskChunksRangeConstraint_2023_09.CONTIGUOUS, + ), + ), + "Param2": RangeListTaskParameterDefinition_2023_09( + type=ParameterValueType.STRING, + range=["A", "B"], + ), + }, + # The order is different from the equivalent non-adaptive, because the chunked dimension + # is moved to the inside + [("1-1", "A"), ("2-2", "A"), ("1-1", "B"), ("2-2", "B")], + True, + id="2 dim, chunked outer, chunksize 1, adaptive", + ), + pytest.param( + { + "Param1": RangeListTaskParameterDefinition_2023_09( + type=ParameterValueType.CHUNK_INT, + range=["1", "2"], + chunks=TaskChunksDefinition_2023_09( + defaultTaskCount=2, + targetRuntimeSeconds=20, + rangeConstraint=TaskChunksRangeConstraint_2023_09.CONTIGUOUS, + ), + ), + "Param2": RangeListTaskParameterDefinition_2023_09( + type=ParameterValueType.STRING, + range=["A", "B"], + ), + }, + [("1-2", "A"), ("1-2", "B")], + True, + id="2 dim, chunked outer, chunksize 2, adaptive", + ), +) + + +@pytest.mark.parametrize("param_defs,expected,chunks_adaptive", PARAMETRIZE_CASES) +def test_multi_param_chunked_iteration(param_defs: dict[str, Any], expected, chunks_adaptive): + # GIVEN + space = StepParameterSpace_2023_09(taskParameterDefinitions=param_defs) + + # WHEN + it = StepParameterSpaceIterator(space=space) + + # THEN + assert it.chunks_adaptive == chunks_adaptive + # Check that full iteration over the range gives the expected result + assert [v for v in it] == [ + { + n: ParameterValue(type=ParameterValueType(param.type), value=v) + for (n, param), v in zip(param_defs.items(), item) + } + for item in expected + ] + # Check that resetting the iterator and re-iterating produces the same again + it.reset_iter() + assert [v for v in it] == [ + { + n: ParameterValue(type=ParameterValueType(param.type), value=v) + for (n, param), v in zip(param_defs.items(), item) + } + for item in expected + ] + + +def test_adaptive_contiguous_chunked_iteration(): + # GIVEN + param_defs = { + "P1": RangeExpressionTaskParameterDefinition_2023_09( + type=ParameterValueType.CHUNK_INT, + range="1-20", + chunks=TaskChunksDefinition_2023_09( + defaultTaskCount=2, + targetRuntimeSeconds=20, + rangeConstraint=TaskChunksRangeConstraint_2023_09.CONTIGUOUS, + ), + ), + "P2": RangeListTaskParameterDefinition_2023_09( + type=ParameterValueType.STRING, + range=["A", "B"], + ), + } + space = StepParameterSpace_2023_09(taskParameterDefinitions=param_defs) + + # WHEN + it = StepParameterSpaceIterator(space=space) + + def make_item(v0, v1): + return { + "P1": ParameterValue(type=ParameterValueType.CHUNK_INT, value=v0), + "P2": ParameterValue(type=ParameterValueType.STRING, value=v1), + } + + # THEN + # Starting with chunk size 2, then change the chunk size periodically + assert next(it) == make_item("1-2", "A") + assert next(it) == make_item("3-4", "A") + it.chunks_default_task_count = 10 + assert next(it) == make_item("5-14", "A") + assert next(it) == make_item("15-20", "A") + assert next(it) == make_item("1-10", "B") + it.chunks_default_task_count = 4 + assert next(it) == make_item("11-14", "B") + assert next(it) == make_item("15-18", "B") + it.chunks_default_task_count = 1 + assert next(it) == make_item("19-19", "B") + assert next(it) == make_item("20-20", "B") + + with pytest.raises(StopIteration): + next(it) + + +def test_adaptive_noncontiguous_chunked_iteration(): + # GIVEN + param_defs = { + "P1": RangeExpressionTaskParameterDefinition_2023_09( + type=ParameterValueType.CHUNK_INT, + range="1-10,12,15,18,20-23,1000", + chunks=TaskChunksDefinition_2023_09( + defaultTaskCount=2, + targetRuntimeSeconds=20, + rangeConstraint=TaskChunksRangeConstraint_2023_09.NONCONTIGUOUS, + ), + ), + "P2": RangeListTaskParameterDefinition_2023_09( + type=ParameterValueType.STRING, + range=["A", "B"], + ), + } + space = StepParameterSpace_2023_09(taskParameterDefinitions=param_defs) + + # WHEN + it = StepParameterSpaceIterator(space=space) + + def make_item(v0, v1): + return { + "P1": ParameterValue(type=ParameterValueType.CHUNK_INT, value=v0), + "P2": ParameterValue(type=ParameterValueType.STRING, value=v1), + } + + # THEN + # Starting with chunk size 2, then change the chunk size periodically + assert next(it) == make_item("1,2", "A") + assert next(it) == make_item("3,4", "A") + it.chunks_default_task_count = 10 + assert next(it) == make_item("5-10,12-18:3,20", "A") + assert next(it) == make_item("21-23,1000", "A") + assert next(it) == make_item("1-10", "B") + it.chunks_default_task_count = 4 + assert next(it) == make_item("12-18:3,20", "B") + assert next(it) == make_item("21-23,1000", "B") + + with pytest.raises(StopIteration): + next(it) + + +def test_divide_chunk_sizes(): + # Some edge cases + assert divide_chunk_sizes(1, 0) == [] + assert divide_chunk_sizes(1, 1) == [1] + assert divide_chunk_sizes(10000, 10000) == [10000] + assert divide_chunk_sizes(1, 100) == [1] * 100 + # Splitting in two + assert divide_chunk_sizes(25, 49) == [25, 24] + assert divide_chunk_sizes(25, 50) == [25, 25] + assert divide_chunk_sizes(26, 51) == [26, 25] + # Check that chunks are evenly divided across a variety of chunk counts for a prime task count + assert divide_chunk_sizes(37, 37) == [37] + assert divide_chunk_sizes(36, 37) == [19, 18] + assert divide_chunk_sizes(17, 37) == [13, 12, 12] + assert divide_chunk_sizes(9, 37) == [8, 7, 8, 7, 7] + assert divide_chunk_sizes(4, 37) == [4, 4, 4, 3, 4, 4, 3, 4, 4, 3] + assert divide_chunk_sizes(3, 37) == [3, 3, 3, 3, 3, 3, 2, 3, 3, 3, 3, 3, 2] + assert divide_chunk_sizes(2, 37) == [2] * 18 + [1] + assert divide_chunk_sizes(1, 37) == [1] * 37 + + +def test_divide_interval_into_chunks(): + # Some edge cases + assert divide_int_interval_into_chunks(1, 0, 0) == ["0-0"] + assert divide_int_interval_into_chunks(10, -10, -1) == ["-10--1"] + assert divide_int_interval_into_chunks(5, -10, -1) == ["-10--6", "-5--1"] + assert divide_int_interval_into_chunks(100000, 0, 10000) == ["0-10000"] + assert divide_int_interval_into_chunks(10000, 0, 10000) == ["0-5000", "5001-10000"] + assert divide_int_interval_into_chunks(1, 1, 100) == [f"{i}-{i}" for i in range(1, 101)] + # Where the boundaries of splitting in 1, 2, or 3 live + assert divide_int_interval_into_chunks(24, 1, 50) == ["1-17", "18-34", "35-50"] + assert divide_int_interval_into_chunks(25, 1, 50) == ["1-25", "26-50"] + assert divide_int_interval_into_chunks(49, 1, 50) == ["1-25", "26-50"] + assert divide_int_interval_into_chunks(50, 1, 50) == ["1-50"] + # Check that chunks are evenly divided across a variety of chunk counts for a prime task count + assert divide_int_interval_into_chunks(1, 22, 40) == [f"{i}-{i}" for i in range(22, 41)] + assert divide_int_interval_into_chunks(2, 22, 40) == [ + f"{i}-{min(i + 1, 40)}" for i in range(22, 41, 2) + ] + assert divide_int_interval_into_chunks(3, 22, 40) == [ + "22-24", + "25-27", + "28-30", + "31-32", + "33-35", + "36-38", + "39-40", + ] + assert divide_int_interval_into_chunks(5, 22, 40) == ["22-26", "27-31", "32-36", "37-40"] + assert divide_int_interval_into_chunks(15, 22, 40) == ["22-31", "32-40"] + assert divide_int_interval_into_chunks(37, 22, 40) == ["22-40"] + + +def test_divide_int_list_into_contiguous_chunks(): + # Cases of dividing an integer list into contiguous chunks + assert divide_int_list_into_contiguous_chunks(1, []) == [] + assert divide_int_list_into_contiguous_chunks(1, [1, 2, 3, 5, 7]) == [ + "1-1", + "2-2", + "3-3", + "5-5", + "7-7", + ] + assert divide_int_list_into_contiguous_chunks(100, [1, 2, 3, 5, 7]) == ["1-3", "5-5", "7-7"] + assert divide_int_list_into_contiguous_chunks(100, [1, 2, 3, 7, 4, 5]) == ["1-3", "7-7", "4-5"] + assert divide_int_list_into_contiguous_chunks(2, [1, 2, 3, 7, 4, 5]) == [ + "1-2", + "3-3", + "7-7", + "4-5", + ] + + +def test_divide_int_list_into_noncontiguous_chunks(): + # Cases of dividing an integer list into noncontiguous chunks + assert divide_int_list_into_noncontiguous_chunks(1, []) == [] + assert divide_int_list_into_noncontiguous_chunks(1, [1, 2, 3, 5, 7]) == [ + "1", + "2", + "3", + "5", + "7", + ] + assert divide_int_list_into_noncontiguous_chunks(100, [1, 2, 3, 5, 7]) == ["1-3,5,7"] + assert divide_int_list_into_noncontiguous_chunks(100, [1, 2, 3, 7, 4, 5]) == ["1-5,7"] + assert divide_int_list_into_noncontiguous_chunks(2, [1, 2, 3, 7, 4, 5]) == ["1,2", "3,7", "4,5"] + assert divide_int_list_into_noncontiguous_chunks(3, [1, 2, 3, 7, 4, 5]) == ["1-3", "4,5,7"] diff --git a/test/openjd/model/v2023_09/test_chunk_int_task_parameter_type.py b/test/openjd/model/v2023_09/test_chunk_int_task_parameter_type.py index 3b5ab67..d3d8815 100644 --- a/test/openjd/model/v2023_09/test_chunk_int_task_parameter_type.py +++ b/test/openjd/model/v2023_09/test_chunk_int_task_parameter_type.py @@ -752,3 +752,35 @@ def test_param_space_with_chunk_int_parse_fails( excinfo.value ) assert excinfo.value.error_count() == 1 + + +def test_only_one_chunk_parameter(): + data = { + "taskParameterDefinitions": [ + { + "name": "oof", + "type": "CHUNK[INT]", + "range": "1-10", + "chunks": {"defaultTaskCount": 1, "rangeConstraint": "CONTIGUOUS"}, + }, + {"name": "foo", "type": "INT", "range": [1]}, + {"name": "bar", "type": "INT", "range": [1]}, + { + "name": "baz", + "type": "CHUNK[INT]", + "range": "1-10", + "chunks": {"defaultTaskCount": 1, "rangeConstraint": "CONTIGUOUS"}, + }, + ], + } + + with pytest.raises(ValidationError) as excinfo: + _parse_model( + model=StepParameterSpaceDefinition, + obj=data, + context=ModelParsingContext(supported_extensions=["TASK_CHUNKING"]), + ) + + # THEN + assert "Only one CHUNK[INT] task parameter is permitted" in str(excinfo.value) + assert len(excinfo.value.errors()) == 1, str(excinfo.value)