diff --git a/src/openjd/model/_errors.py b/src/openjd/model/_errors.py index 8908b0a..934e93f 100644 --- a/src/openjd/model/_errors.py +++ b/src/openjd/model/_errors.py @@ -10,21 +10,7 @@ ] -class _BaseMessageError(Exception): - """A base class for exceptions that have an error message""" - - msg: str - """The error message""" - - def __init__(self, msg: str) -> None: - self.msg = msg - super(_BaseMessageError, self).__init__(msg) - - def __str__(self) -> str: - return self.msg - - -class UnsupportedSchema(_BaseMessageError): +class UnsupportedSchema(ValueError): """Error raised when an attempt is made to decode a template with an unknown or otherwise nonvalid schema identification. """ @@ -36,29 +22,23 @@ def __init__(self, version: str): super().__init__(f"Unsupported schema version: {self._version}") -class DecodeValidationError(_BaseMessageError): +class DecodeValidationError(ValueError): """Error raised when an decoding error is encountered while decoding a template. """ - pass - -class ModelValidationError(_BaseMessageError): +class ModelValidationError(ValueError): """Error raised when a validation error is encountered while validating a model. """ - pass - -class ExpressionError(_BaseMessageError): +class ExpressionError(ValueError): """Error raised when there is an error in the form of an expression that is being parsed. """ - pass - class TokenError(ExpressionError): """Error raised when performing lexical analysis on an expression for parsing.""" @@ -68,9 +48,7 @@ def __init__(self, expression: str, token_value: str, position: int): super().__init__(msg) -class CompatibilityError(_BaseMessageError): +class CompatibilityError(ValueError): """Error raised when a check that two, or more, models are compatible determines that there are non-compatibilities between the models. """ - - pass diff --git a/src/openjd/model/_range_expr.py b/src/openjd/model/_range_expr.py index f74edac..7c78994 100644 --- a/src/openjd/model/_range_expr.py +++ b/src/openjd/model/_range_expr.py @@ -2,11 +2,14 @@ from __future__ import annotations -from bisect import bisect +from bisect import bisect, bisect_left from collections.abc import Iterator, Sized -from functools import total_ordering from itertools import chain -from typing import Tuple +from typing import Any, Tuple + +from pydantic import GetCoreSchemaHandler, GetJsonSchemaHandler +from pydantic.json_schema import JsonSchemaValue +from pydantic_core import core_schema from ._errors import ExpressionError, TokenError from ._tokenstream import Token, TokenStream, TokenType @@ -15,15 +18,17 @@ class IntRangeExpr(Sized): """An Int Range Expression is a set of integer values represented as a sorted list of IntRange objects.""" - _start: int - _end: int + _starts: list[int] + _ends: list[int] _ranges: list[IntRange] _length: int _range_length_indicies: list[int] def __init__(self, ranges: list[IntRange]): + if len(ranges) <= 0: + raise ExpressionError("Range expression cannot be empty") # Sort the ranges, then combine them where possible - sorted_ranges = sorted(ranges) + sorted_ranges = sorted(ranges, key=lambda v: (v._start, v._end, v._step)) self._ranges = [sorted_ranges[0]] for range in sorted_ranges[1:]: if ( @@ -33,8 +38,8 @@ def __init__(self, ranges: list[IntRange]): self._ranges[-1] = IntRange(self._ranges[-1].start, range.end, range.step) else: self._ranges.append(range) - self._start = self.ranges[0].start - self._end = self.ranges[-1].end + self._starts = [v.start for v in self.ranges] + self._ends = [v.end for v in self.ranges] # used to binary search ranges for __getitem__ # ie. [32, 100, 132] @@ -95,7 +100,7 @@ def __str__(self) -> str: return ",".join(str(range) for range in self.ranges) def __repr__(self) -> str: - return f"{type(self).__name__}({self.ranges})" + return f"{type(self).__name__}.from_str({str(self)})" def __iter__(self) -> Iterator[int]: return chain(*self.ranges) @@ -120,15 +125,23 @@ def __getitem__(self, index: int) -> int: actual_index = index - self._range_length_indicies[range_index - 1] return self.ranges[range_index][actual_index] + def __contains__(self, value: object) -> bool: + if not isinstance(value, int): + return False + range_index = bisect_left(self._ends, value) + if range_index >= len(self._ends): + return False + return value in self.ranges[range_index]._range + @property def start(self) -> int: """The smallest value in the range expression.""" - return self._start + return self._starts[0] @property def end(self) -> int: """The largest value in the range expression""" - return self._end + return self._ends[-1] @property def ranges(self) -> list[IntRange]: @@ -137,10 +150,6 @@ def ranges(self) -> list[IntRange]: def _validate(self) -> None: """raises: ValueError - if not valid""" - - if len(self) <= 0: - raise ValueError("range expression cannot be empty") - # Validate that the ranges are not overlapping prev_range: IntRange | None = None for range_ in self.ranges: @@ -156,10 +165,34 @@ def _validate(self) -> None: ) prev_range = range_ + @classmethod + def _pydantic_validate(cls, value: Any) -> Any: + if isinstance(value, IntRangeExpr): + return value + elif isinstance(value, str): + return IntRangeExpr.from_str(value) + elif isinstance(value, list): + return IntRangeExpr.from_list(value) + else: + raise ValueError("Value must be an integer range expression or a list of integers.") + + @classmethod + def __get_pydantic_core_schema__( + cls, source_type: type[Any], handler: GetCoreSchemaHandler + ) -> core_schema.CoreSchema: + return core_schema.no_info_plain_validator_function(cls._pydantic_validate) + + @classmethod + def __get_pydantic_json_schema__( + cls, core_schema: core_schema.CoreSchema, handler: GetJsonSchemaHandler + ) -> JsonSchemaValue: + return {"type": "string"} + -@total_ordering class IntRange(Sized): - """Inclusive on the start and end value""" + """A linear sequence of integers. + + Both _start and _end are always included in the set of values, and _step is always positive.""" _start: int _end: int @@ -167,20 +200,23 @@ class IntRange(Sized): _range: range def __init__(self, start: int, end: int, step: int = 1): - self._start = start - self._end = end - self._step = step - - # makes the range inclusive on end value - offset = 0 - if self._step > 0: - offset = 1 - elif self._step < 0: - offset = -1 - - self._range = range(self._start, self._end + offset, self._step) - - self._validate() + if step > 0: + if start > end: + raise ValueError("Range: a descending range must have a negative step") + self._range = range(start, end + 1, step) + self._start = start + self._end = self._range[-1] + self._step = step + elif step < 0: + if start < end: + raise ValueError("Range: an ascending range must have a positive step") + # Reverse the range if the step is negative + self._range = range(start, end - 1, step) + self._start = self._range[-1] + self._end = start + self._step = -step + else: + raise ValueError("Range: step must not be zero") def __str__(self) -> str: len_self = len(self) @@ -204,11 +240,6 @@ def __eq__(self, other: object) -> bool: raise NotImplementedError return (self.start, self.end, self.step) == (other.start, other.end, other.step) - def __lt__(self, other: object) -> bool: - if not isinstance(other, IntRange): - raise NotImplementedError - return (self.start, self.end, self.step) < (other.start, other.end, other.step) - def __iter__(self) -> Iterator[int]: return iter(self._range) @@ -232,21 +263,6 @@ def step(self) -> int: """read-only property""" return self._step - def _validate(self) -> None: - """raises: ValueError - if not valid""" - - if self._step == 0: - raise ValueError("Range: step must not be zero") - - if self._start < self._end and self._step < 0: - raise ValueError("Range: an ascending range must have a positive step") - - if self._start > self._end and self._step > 0: - raise ValueError("Range: a descending range must have a negative step") - - if len(self) <= 0: - raise ValueError("Range: cannot be empty") - class PosIntToken(Token): """A positive integer""" diff --git a/src/openjd/model/_step_param_space_iter.py b/src/openjd/model/_step_param_space_iter.py index 04806da..106ca1b 100644 --- a/src/openjd/model/_step_param_space_iter.py +++ b/src/openjd/model/_step_param_space_iter.py @@ -7,6 +7,7 @@ from dataclasses import dataclass, field from functools import reduce from operator import mul +import re from typing import AbstractSet, Any, Optional, Union from ._internal import ( @@ -76,8 +77,21 @@ class StepParameterSpaceIterator(Iterable[TaskParameterSet], Sized): would be the order: (A=3,B=1,C=10), (A=3,B=2,C=11), (A=2,B=1,C=10), (A=2,B=2,C=11), (A=1,B=1,C=10), (A=1,B=2,C=11) + Args: + space (Optional[StepParameterSpace]): The parameter space to iterate over. + chunks_task_count_override (Optional[int]): If provided, turns off adaptive chunking + and overrides the default task count from the parameter space with the provided value. + For example, to iterate over all the individual tasks, set this value to 1. + Attributes: - None + names (AbstractSet[str]): The set of all task parameter names. + chunks_adaptive (bool): If True, the iterator is configured for adaptive chunking. This means + the caller can modify the chunks_default_task_count property when it determines a different + chunk size is appropriate to better match the targetRuntimeSeconds of the CHUNK parameter. + chunks_default_task_count (Optional[int]): If chunking is disabled, is None. If chunking is + enabled, is the default chunk size to use for grouping tasks into chunks. If adaptive + chunking is enabled, the caller can modify this value and subsequent iteration will reflect + the new chunk size. """ _parameters: dict[str, TaskParameter] @@ -86,10 +100,17 @@ class StepParameterSpaceIterator(Iterable[TaskParameterSet], Sized): _parsedtree: CombinationExpressionNode _chunks_adaptive: bool + _chunks_parameter_name: Optional[str] _chunks_default_task_count: Optional[int] - def __init__(self, *, space: Optional[StepParameterSpace]): + def __init__( + self, + *, + space: Optional[StepParameterSpace], + chunks_task_count_override: Optional[int] = None, + ): self._chunks_adaptive = False + self._chunks_parameter_name = None self._chunks_default_task_count = None # Special case the zero-dimensional space with one element @@ -117,9 +138,18 @@ def __init__(self, *, space: Optional[StepParameterSpace]): param.chunks.targetRuntimeSeconds is not None and int(param.chunks.targetRuntimeSeconds) > 0 ) + self._chunks_parameter_name = name self._chunks_default_task_count = int(param.chunks.defaultTaskCount) break + # If the chunks task count is overridden, and the parameter space uses chunking + if ( + chunks_task_count_override is not None + and self._chunks_default_task_count is not None + ): + self._chunks_adaptive = False + self._chunks_default_task_count = chunks_task_count_override + # Raises: TokenError, ExpressionError self._parsetree = CombinationExpressionParser().parse(combination) self._expr_tree = self._create_expr_tree( @@ -139,6 +169,11 @@ def chunks_adaptive(self) -> bool: """True if the parameter space includes a CHUNK[INT] parameter with a non-zero target runtime.""" return self._chunks_adaptive + @property + def chunks_parameter_name(self) -> Optional[str]: + """The name of the CHUNK[INT] parameter, if any.""" + return self._chunks_parameter_name + @property def chunks_default_task_count(self) -> Optional[int]: """The default task count for the CHUNK[INT] parameter, if any. @@ -165,6 +200,7 @@ def chunks_default_task_count(self, value: int): if not (isinstance(value, int) and value > 0): raise ValueError("chunks_default_task_count must be a positive integer.") + # The expr tree will raise if the parameter space does not use adaptive chunking self._expr_tree.set_chunks_default_task_count(value) self._chunks_default_task_count = value @@ -202,6 +238,33 @@ def __getitem__(self, index: int) -> TaskParameterSet: """ return self._expr_tree[index] + def __contains__(self, params: TaskParameterSet) -> bool: + """Check if a specific task parameter set is included in this parameter space. + + In the case of chunking, the chunked dimension is treated via a subset operator + instead of whether it equals one of the values along the chunked dimension.""" + try: + self.validate_containment(params) + return True + except ValueError: + return False + + def validate_containment(self, params: TaskParameterSet): + """Check if a specific task parameter set is included in this parameter space, + raising a ValueError if it is not. + + In the case of chunking, the chunked dimension is treated via a subset operator + instead of whether it equals one of the values along the chunked dimension.""" + # The task parameter names must match + params_keys = sorted(params.keys()) + space_keys = sorted(self._parameters.keys()) + if params_keys != space_keys: + raise ValueError( + f"Task parameter names {params_keys} do not match the parameter space names {space_keys}." + ) + # Check containment against the expr tree nodes + self._expr_tree.validate_containment(params) + def _create_expr_tree( self, root: CombinationExpressionNode, @@ -227,7 +290,7 @@ def _create_expr_tree( if isinstance(parameter.range, list): parameter_range: list[int] = [int(v) for v in parameter.range] else: - parameter_range = list[int](IntRangeExpr.from_str(parameter.range)) + parameter_range = list[int](parameter.range) if chunks_adaptive: if ( @@ -263,25 +326,28 @@ def _create_expr_tree( # With the chunks determined, we can use the range list node for iteration return RangeListIdentifierNode( - name, - ParameterValueType(parameter.type), - chunk_list, + name=name, + type=ParameterValueType(parameter.type), + range=chunk_list, + range_set=set(parameter_range), + range_constraint=parameter.chunks.rangeConstraint, ) elif isinstance(parameter.range, list): return RangeListIdentifierNode( - name, - ParameterValueType(parameter.type), - parameter.range, + name=name, + type=ParameterValueType(parameter.type), + range=parameter.range, + range_set=set(parameter.range), ) else: return RangeExpressionIdentifierNode( - name, - ParameterValueType(parameter.type), - parameter.range, + name=name, + type=ParameterValueType(parameter.type), + range=parameter.range, ) elif isinstance(root, CombinationExpressionAssociationNode): return AssociationNode( - tuple( + children=tuple( self._create_expr_tree(child, chunks_adaptive, chunks_default_task_count) for child in root.children ), @@ -307,7 +373,7 @@ def _create_expr_tree( children = root.children return ProductNode( - tuple( + children=tuple( self._create_expr_tree(child, chunks_adaptive, chunks_default_task_count) for child in children ), @@ -445,6 +511,10 @@ def set_chunks_default_task_count(self, value: int) -> None: "The parameter space does not use adaptive chunking, so cannot modify chunks_default_task_count." ) + def validate_containment(self, params: TaskParameterSet): + """Validates that the parameter space contains the provided parameters. Raises ValueError if not.""" + raise NotImplementedError("Base class") # pragma: no cover + class ZeroDimSpaceIter(NodeIterator): """Iterator for a zero-dimensional space @@ -487,6 +557,9 @@ def __getitem__(self, index: int) -> TaskParameterSet: def iter(self) -> NodeIterator: return ZeroDimSpaceIter() + def validate_containment(self, params: TaskParameterSet): + pass + class ProductNodeIter(NodeIterator): """Iterator for a ProductNode @@ -541,7 +614,7 @@ def next(self, result: TaskParameterSet) -> None: # 2 | 4 | 5 # 2 | 4 | 6 # - # To acheive, algorithmically, think about grade-school addition by + # To achieve, algorithmically, think about grade-school addition by # 1 with a carry. # We advance the right-most parameter by one. If that overflows (i.e. # hits the end of the iterator) then we reset its iterator and advance @@ -616,6 +689,11 @@ def set_chunks_default_task_count(self, value: int) -> None: # otherwise it will raise an exception. self.children[-1].set_chunks_default_task_count(value) + def validate_containment(self, params: TaskParameterSet): + """Checks if the params restricted to this node are part of the node's range.""" + for child in self.children: + child.validate_containment(params) + class AssociationNodeIter(NodeIterator): """Iterator for an AssociationNode @@ -658,6 +736,33 @@ def __getitem__(self, index: int) -> TaskParameterSet: def iter(self) -> AssociationNodeIter: return AssociationNodeIter(self.children) + def validate_containment(self, params: TaskParameterSet): + """Checks if the params restricted to this node are part of the node's range.""" + # To keep this code simple, we perform a linear search over the full subspace. + # TODO: Make a more efficient implementation. + it = self.iter() + # Use the first iteration value to take a subset of the keys from params + first_value: TaskParameterSet = TaskParameterSet() + it.next(first_value) + + # Restrict the params to the keys in this associative expression so that + # we can use equality checking in the linear search. + params = {key: value for key, value in params.items() if key in first_value} + if first_value == params: + return + # Perform the linear search + try: + while True: + value: TaskParameterSet = TaskParameterSet() + it.next(value) + if value == params: + return + except StopIteration: + pass + raise ValueError( + f"The values { {name: param.value for name, param in params.items()} }, of an association expression in the combination expression, do not appear in the parameter space." + ) + class RangeListIdentifierNodeIterator(NodeIterator): """Iterator for a RangeListIdentifierNode @@ -683,11 +788,16 @@ def next(self, result: TaskParameterSet) -> None: result[self._node.name] = ParameterValue(type=self._node.type, value=v) +INTERVAL_RE = re.compile(r"\s*(-?[0-9]+)\s*-\s*(-?[0-9]+)\s*") + + @dataclass class RangeListIdentifierNode(Node): name: str type: ParameterValueType range: list[str] + range_set: set + range_constraint: Optional[TaskChunksRangeConstraint_2023_09] = None _len: int = field(init=False, repr=False, compare=False) def __post_init__(self): @@ -699,6 +809,42 @@ def __len__(self) -> int: def __getitem__(self, index: int) -> TaskParameterSet: return {self.name: ParameterValue(type=self.type, value=self.range[index])} + def validate_containment(self, params: TaskParameterSet): + """Checks if the params restricted to this node are part of the node's range.""" + param = params[self.name] + if param.type != self.type: + raise ValueError( + f"Parameter {self.name} of type {param.type.value} does not match the parameter space type {self.type.name}." + ) + if self.type == ParameterValueType.CHUNK_INT: + # The chunk must be an interval if the range constraint is CONTIGUOUS + if ( + self.range_constraint == TaskChunksRangeConstraint_2023_09.CONTIGUOUS + and not INTERVAL_RE.match(param.value) + ): + raise ValueError( + f"Parameter {self.name} of type {param.type.value} value {param.value} is not a contiguous interval like '1-5' as required by the chunk range constraint." + ) + # The chunk must be a subset of the range + try: + range_expr = IntRangeExpr.from_str(param.value) + except ValueError: + # If the value is not a range expression + raise ValueError( + f"Parameter {self.name} of type {param.type.value} value {param.value} is not a valid range expression like '1-5,10' as required by the chunk range constraint." + ) + for v in range_expr: + if v not in self.range_set: + raise ValueError( + f"Parameter {self.name} of type {param.type.value} value {param.value} is not a subset of the range in the parameter space." + ) + else: + # The value must be an item in the range + if param.value not in self.range_set: + raise ValueError( + f"Parameter {self.name} of type {param.type.value} value {param.value} is not in the parameter space range." + ) + def iter(self) -> RangeListIdentifierNodeIterator: return RangeListIdentifierNodeIterator(self) @@ -719,7 +865,7 @@ def __init__(self, node: RangeExpressionIdentifierNode): self.reset_iter() def reset_iter(self) -> None: - self._it = iter(self._node.range_expression) + self._it = iter(self._node.range) def next(self, result: TaskParameterSet) -> None: # Raises: StopIteration @@ -731,23 +877,40 @@ def next(self, result: TaskParameterSet) -> None: class RangeExpressionIdentifierNode(Node): name: str type: ParameterValueType - range: str - range_expression: IntRangeExpr = field(init=False, repr=False, compare=False) + range: IntRangeExpr + range_constraint: Optional[TaskChunksRangeConstraint_2023_09] = None _len: int = field(init=False, repr=False, compare=False) def __post_init__(self): - self.range_expression = IntRangeExpr.from_str(self.range) - self._len = len(self.range_expression) + self._len = len(self.range) def __len__(self) -> int: return self._len def __getitem__(self, index: int) -> TaskParameterSet: - return {self.name: ParameterValue(type=self.type, value=str(self.range_expression[index]))} + return {self.name: ParameterValue(type=self.type, value=str(self.range[index]))} def iter(self) -> RangeExpressionIdentifierNodeIterator: return RangeExpressionIdentifierNodeIterator(self) + def validate_containment(self, params: TaskParameterSet): + """Checks if the params restricted to this node are part of the node's range.""" + param = params[self.name] + if param.type != self.type: + raise ValueError( + f"Parameter {self.name} of type {param.type.value} does not match the parameter space type {self.type.name}." + ) + # The value must be an item in the range + try: + if int(param.value) not in self.range: + raise ValueError( + f"Parameter {self.name} of type {param.type.value} value {param.value} is not in the parameter space range." + ) + except ValueError: + raise ValueError( + f"Parameter {self.name} of type {param.type.value} value {param.value} is not in the parameter space range." + ) + class AdaptiveContiguousChunkIdentifierNodeIterator(NodeIterator): """Iterator for an AdaptiveChunkIdentifierNode @@ -796,9 +959,13 @@ class AdaptiveContiguousChunkIdentifierNode(Node): name: str type: ParameterValueType range: list[int] + _range_set: set[int] = field(init=False, repr=False, compare=False) chunks_default_task_count: int """This value can be modified by the caller to adapt the chunk size.""" + def __post_init__(self): + self._range_set = set(self.range) + def __len__(self) -> int: raise ValueError( "Length is not available because the parameter space uses adaptive chunking." @@ -815,6 +982,25 @@ def iter(self) -> AdaptiveContiguousChunkIdentifierNodeIterator: def set_chunks_default_task_count(self, value: int) -> None: self.chunks_default_task_count = value + def validate_containment(self, params: TaskParameterSet): + """Checks if the params restricted to this node are part of the node's range.""" + param = params[self.name] + if param.type != self.type: + raise ValueError( + f"Parameter {self.name} of type {param.type.value} does not match the parameter space type {self.type.name}." + ) + # The chunk must be an interval if the range constraint is CONTIGUOUS + if not INTERVAL_RE.match(param.value): + raise ValueError( + f"Parameter {self.name} of type {param.type.value} value {param.value} is not a contiguous interval like '1-5' as required by the chunk range constraint." + ) + # The chunk must be a subset of the range + for v in IntRangeExpr.from_str(param.value): + if v not in self._range_set: + raise ValueError( + f"Parameter {self.name} of type {param.type.value} value {param.value} is not in the parameter space range." + ) + class AdaptiveNoncontiguousChunkIdentifierNodeIterator(NodeIterator): """Iterator for an AdaptiveNoncontiguousChunkIdentifierNode @@ -854,9 +1040,14 @@ class AdaptiveNoncontiguousChunkIdentifierNode(Node): name: str type: ParameterValueType range: list[int] + _range_set: set[int] = field(init=False, repr=False, compare=False) chunks_default_task_count: int """This value can be modified by the caller to adapt the chunk size.""" + def __post_init__(self): + self._range_set = set(self.range) + self._len = len(self.range) + def __len__(self) -> int: raise ValueError( "Length is not available because the parameter space uses adaptive chunking." @@ -872,3 +1063,23 @@ def iter(self) -> AdaptiveNoncontiguousChunkIdentifierNodeIterator: def set_chunks_default_task_count(self, value: int) -> None: self.chunks_default_task_count = value + + def validate_containment(self, params: TaskParameterSet): + """Checks if the params restricted to this node are part of the node's range.""" + param = params[self.name] + if param.type != self.type: + raise ValueError( + f"Parameter {self.name} of type {param.type.value} does not match the parameter space type {self.type.name}." + ) + # The chunk must be a subset of the range + try: + for v in IntRangeExpr.from_str(param.value): + if v not in self._range_set: + raise ValueError( + f"Parameter {self.name} of type {param.type.value} value {param.value} is not in the parameter space range." + ) + except ValueError: + # If the value is not a range expression + raise ValueError( + f"Parameter {self.name} of type {param.type.value} value {param.value} is not a valid range expression like '1-5,10' as required by the chunk range constraint." + ) diff --git a/src/openjd/model/v2023_09/_model.py b/src/openjd/model/v2023_09/_model.py index 57dd9ab..9b1e480 100644 --- a/src/openjd/model/v2023_09/_model.py +++ b/src/openjd/model/v2023_09/_model.py @@ -518,7 +518,6 @@ class RangeString(FormatString): TaskParameterStringValueAsJob = Annotated[str, StringConstraints(min_length=0, max_length=1024)] TaskRangeList = list[TaskParameterStringValueAsJob] -TaskRangeExpression = RangeString # Target model for task parameters when instantiating a job. @@ -545,22 +544,10 @@ def coerce(v: Any) -> Any: class RangeExpressionTaskParameterDefinition(OpenJDModel_v2023_09): # element type of items in the range type: TaskParameterType - range: TaskRangeExpression + range: IntRangeExpr # has a value when type is CHUNK[INT], which is only possible from the TASK_CHUNKING extension chunks: Optional[TaskChunksDefinition] = None - @field_validator("range") - @classmethod - def _validate_range_expression(cls, value: Any) -> Any: - """At this point, the format expressions have been resolved - and we can determine if it's a valid RangeExpression""" - try: - IntRangeExpr.from_str(value) - except Exception as e: - raise ValueError(str(e)) - - return value - class TaskChunksRangeConstraint(str, Enum): CONTIGUOUS = "CONTIGUOUS" @@ -874,14 +861,7 @@ def _validate_parameter_space(cls, v: str, info: ValidationInfo) -> str: param_defs = cast( dict[Identifier, TaskRangeParameter], info.data["taskParameterDefinitions"] ) - parameter_range_lengths = { - id: ( - len(param.range) - if isinstance(param.range, list) - else len(IntRangeExpr.from_str(param.range)) - ) - for id, param in param_defs.items() - } + parameter_range_lengths = {id: len(param.range) for id, param in param_defs.items()} try: validate_step_parameter_space_dimensions(parameter_range_lengths, v) except ExpressionError as e: diff --git a/test/openjd/model/_internal/test_range_expr.py b/test/openjd/model/_internal/test_range_expr.py index 7d385eb..f79d2ad 100644 --- a/test/openjd/model/_internal/test_range_expr.py +++ b/test/openjd/model/_internal/test_range_expr.py @@ -19,17 +19,10 @@ def test_propagates_error(self) -> None: with pytest.raises(TokenError): parser.parse("!") - def test_fails_empty(self) -> None: - # GIVEN - parser = RangeExpressionParser() - - # THEN - with pytest.raises(ExpressionError): - parser.parse("") - @pytest.mark.parametrize( "range_expr", [ + pytest.param("", id="empty str"), pytest.param("1-a", id="non-valid character at end"), pytest.param("b-1", id="non-valid character at start"), pytest.param("--12", id="missing start"), @@ -194,6 +187,14 @@ def test_range_sorting_and_merging(self): pytest.param(" 5 ", "5", id="one int"), pytest.param("9,0,3,2,8,10,1,4,7,6,5", "0-10", id="values 0-10 out of order"), pytest.param("3-5,0-2,8-12:2", "0-5,8-12:2", id="ranges out of order with steps"), + pytest.param( + "5-3:-1,12-8:-2,2-0:-1,6-7:2", + "0-5,6-12:2", + id="ranges out of order with pos + neg steps", + ), + pytest.param("1-5:3", "1,4", id="End value becomes actual last value (2 values)"), + pytest.param("1-7:3", "1-7:3", id="End value included"), + pytest.param("1-9:3", "1-7:3", id="End value becomes actual last value (3 values)"), ], ) def test_range_expr_from_str(self, range_input_str: str, range_str: str): @@ -228,35 +229,33 @@ def test_range_expr_from_list(self, range_list: list[Union[int, str]], range_str # THEN assert str(full_range) == range_str - def test_sorting_with_descending_ranges(self): + def test_range_expr_from_empty_list(self): + with pytest.raises(ExpressionError): + IntRangeExpr.from_list([]) + + def test_sorting_merging_with_descending_ranges(self): # GIVEN first = IntRange(start=-10, end=-19, step=-1) second = IntRange(start=-1, end=-9, step=-1) third = IntRange(start=10, end=0, step=-1) - sorted_ranges = [first, second, third] - # WHEN full_range = IntRangeExpr([second, third, first]) # THEN - for actual, expected in zip_longest(full_range.ranges, sorted_ranges): - assert actual == expected + assert full_range.ranges == [IntRange(-19, 10, 1)] - def test_sorting_mixed_ascending_and_descending_ranges(self): + def test_sorting_merging_mixed_ascending_and_descending_ranges(self): # GIVEN first = IntRange(start=-9, end=-7, step=1) second = IntRange(start=1, end=-6, step=-1) third = IntRange(start=2, end=9, step=1) - sorted_ranges = [first, second, third] - # WHEN full_range = IntRangeExpr([second, first, third]) # THEN - for actual, expected in zip_longest(full_range.ranges, sorted_ranges): - assert actual == expected + assert full_range.ranges == [IntRange(-9, 9, 1)] @pytest.mark.parametrize( "index", @@ -351,24 +350,20 @@ def test_read_only_properties(self) -> None: with pytest.raises(AttributeError): full_range.ranges = [] # type: ignore + def test_contains(self) -> None: + assert 0 in IntRangeExpr.from_str("0-10") + assert 10 in IntRangeExpr.from_str("0-10") + assert -1 not in IntRangeExpr.from_str("0-10") + assert 11 not in IntRangeExpr.from_str("0-10") + assert "X" not in IntRangeExpr.from_str("0-10") -class TestIntRange: - def test_comparisons(self): - # GIVEN / WHEN / THEN - # start, end, and step are the same, therefore equal and not less/greater than - assert IntRange(start=0, end=0, step=1) == IntRange(start=0, end=0) - assert not IntRange(start=0, end=0) < IntRange(start=0, end=0) - assert not IntRange(start=0, end=0) > IntRange(start=0, end=0) - - # start is different - assert IntRange(start=0, end=1) < IntRange(start=1, end=1) + assert -1 in IntRangeExpr.from_str("-1--2:-1") + assert -2 in IntRangeExpr.from_str("-1--2:-1") + assert -3 not in IntRangeExpr.from_str("-1--2:-1") + assert 0 not in IntRangeExpr.from_str("-1--2:-1") - # start is the same, other end is bigger - assert IntRange(start=0, end=0) < IntRange(start=0, end=1) - - # start and end are same, other step is bigger - assert IntRange(start=0, end=0, step=1) < IntRange(start=0, end=0, step=2) +class TestIntRange: def test_length(self): # GIVEN / WHEN / THEN # positive ascending diff --git a/test/openjd/model/test_step_param_space_iter.py b/test/openjd/model/test_step_param_space_iter.py index 3b997a2..e059dd0 100644 --- a/test/openjd/model/test_step_param_space_iter.py +++ b/test/openjd/model/test_step_param_space_iter.py @@ -5,6 +5,7 @@ import pytest from openjd.model import ( + IntRangeExpr, ParameterValue, ParameterValueType, StepParameterSpaceIterator, @@ -30,7 +31,7 @@ class TestStepParameterSpaceIterator_2023_09: # noqa: N801 [ RangeListTaskParameterDefinition_2023_09(type=ParameterValueType.INT, range=["1", "2"]), RangeExpressionTaskParameterDefinition_2023_09( - type=ParameterValueType.INT, range="1-2" + type=ParameterValueType.INT, range=IntRangeExpr.from_str("1-2") ), ], ) @@ -46,10 +47,10 @@ def test_names(self, range_int_param): ) # WHEN - result = StepParameterSpaceIterator(space=space) + it = StepParameterSpaceIterator(space=space) # THEN - assert result.names == set(("Param1", "Param2")) + assert it.names == set(("Param1", "Param2")) def test_no_param_iteration(self): # GIVEN @@ -98,12 +99,16 @@ def test_no_param_getelem(self): assert it[0] == expected assert it[-1] == expected + assert expected in it + assert {"Param": ParameterValue(type=ParameterValueType.INT, value="3")} not in it + assert {"Param": ParameterValue(type=ParameterValueType.FLOAT, value="3")} not in it + @pytest.mark.parametrize( "range_int_param", [ RangeListTaskParameterDefinition_2023_09(type=ParameterValueType.INT, range=["1", "2"]), RangeExpressionTaskParameterDefinition_2023_09( - type=ParameterValueType.INT, range="1-2" + type=ParameterValueType.INT, range=IntRangeExpr.from_str("1-2") ), ], ) @@ -115,10 +120,9 @@ def test_single_param_iteration(self, range_int_param): "Param1": range_int_param, } ) - iterator = StepParameterSpaceIterator(space=space) # WHEN - it = iter(iterator) + it = StepParameterSpaceIterator(space=space) # THEN for i in range(len(expected)): @@ -131,6 +135,10 @@ def test_single_param_iteration(self, range_int_param): with pytest.raises(ValueError): it.chunks_default_task_count = 1 + assert {"Param1": ParameterValue(type=ParameterValueType.INT, value="1")} in it + assert {"Param1": ParameterValue(type=ParameterValueType.INT, value="2")} in it + assert {"Param1": ParameterValue(type=ParameterValueType.INT, value="x")} not in it + @pytest.mark.parametrize("param_range", [["10"], ["10", "11", "12", "13", "14", "15"]]) def test_single_param_getelem(self, param_range): # GIVEN @@ -143,29 +151,40 @@ def test_single_param_getelem(self, param_range): ) # WHEN - result = StepParameterSpaceIterator(space=space) + it = StepParameterSpaceIterator(space=space) # THEN with pytest.raises(IndexError): - result[len(param_range)] + it[len(param_range)] with pytest.raises(IndexError): - result[-len(param_range) - 1] + it[-len(param_range) - 1] expected = [ {"Param1": ParameterValue(type=ParameterValueType.INT, value=str(v))} for v in param_range ] - assert [result[i] for i in range(0, len(param_range))] == expected + assert [it[i] for i in range(0, len(param_range))] == expected range_reversed = param_range.copy() range_reversed.reverse() expected.reverse() - assert [result[-i - 1] for i in range(0, len(param_range))] == expected + assert [it[-i - 1] for i in range(0, len(param_range))] == expected + + for i in range(len(param_range)): + assert expected[i] in it + assert {"Param1": ParameterValue(type=ParameterValueType.INT, value="9")} not in it + assert {"Param2": ParameterValue(type=ParameterValueType.INT, value="10")} not in it + assert { + "Param1": ParameterValue(type=ParameterValueType.INT, value="10"), + "Param2": ParameterValue(type=ParameterValueType.INT, value="10"), + } not in it + assert {} not in it + assert it.chunks_parameter_name is None @pytest.mark.parametrize( "given, expected", [ (["1", "2", "3"], 3), ("1-5", 5), - (["a", "b", "c", "d"], 4), + (["0", "10", "20", "40"], 4), ], ) def test_single_param_len(self, given, expected) -> None: @@ -177,7 +196,7 @@ def test_single_param_len(self, given, expected) -> None: ) elif isinstance(given, str): range_int_param = RangeExpressionTaskParameterDefinition_2023_09( - type=ParameterValueType.INT, range=given + type=ParameterValueType.INT, range=IntRangeExpr.from_str(given) ) space = StepParameterSpace_2023_09( @@ -199,7 +218,7 @@ def test_single_param_len(self, given, expected) -> None: [ RangeListTaskParameterDefinition_2023_09(type=ParameterValueType.INT, range=["1", "2"]), RangeExpressionTaskParameterDefinition_2023_09( - type=ParameterValueType.INT, range="1-2" + type=ParameterValueType.INT, range=IntRangeExpr.from_str("1-2") ), ], ) @@ -217,11 +236,11 @@ def test_defaults_product( ) # WHEN - result = StepParameterSpaceIterator(space=space) + it = StepParameterSpaceIterator(space=space) # THEN # The combination_expr should default to "Param1 * Param2" - assert len(result) == 2 * 2 + assert len(it) == 2 * 2 element: Callable[[int, str], dict[str, ParameterValue]] = lambda p1, p2: { "Param1": ParameterValue(type=ParameterValueType.INT, value=str(p1)), "Param2": ParameterValue(type=ParameterValueType.STRING, value=str(p2)), @@ -232,7 +251,19 @@ def test_defaults_product( element(2, "a"), element(2, "b"), ) - assert expected_values == tuple(v for v in result) + assert expected_values == tuple(v for v in it) + + for value in expected_values: + assert value in it + assert element(1, "A") not in it + assert element(1, "c") not in it + assert element(0, "a") not in it + assert element(3, "a") not in it + assert {} not in it + assert { + "Param1": ParameterValue(type=ParameterValueType.FLOAT, value="1"), + "Param2": ParameterValue(type=ParameterValueType.STRING, value="a"), + } not in it def test_product_iteration(self) -> None: # GIVEN @@ -245,14 +276,14 @@ def test_product_iteration(self) -> None: type=ParameterValueType.STRING, range=["a", "b", "c"] ), "Param3": RangeExpressionTaskParameterDefinition_2023_09( - type=ParameterValueType.INT, range="-1 - -2 : -1" + type=ParameterValueType.INT, range=IntRangeExpr.from_str("-1 - -2 : -1") ), }, combination="Param1 * Param2 * Param3", ) # WHEN - result = StepParameterSpaceIterator(space=space) + it = StepParameterSpaceIterator(space=space) # THEN element: Callable[[int, str, int], dict[str, ParameterValue]] = lambda p1, p2, p3: { @@ -274,14 +305,14 @@ def test_product_iteration(self) -> None: element(2, "c", -1), element(2, "c", -2), ] - assert expected_values == [v for v in result] + assert expected_values == [v for v in it] def test_product_len(self): # GIVEN space = StepParameterSpace_2023_09( taskParameterDefinitions={ "Param1": RangeExpressionTaskParameterDefinition_2023_09( - type=ParameterValueType.INT, range="1-2:1" + type=ParameterValueType.INT, range=IntRangeExpr.from_str("1-2:1") ), "Param2": RangeListTaskParameterDefinition_2023_09( type=ParameterValueType.STRING, range=["a", "b", "c"] @@ -312,14 +343,14 @@ def test_product_getitem(self) -> None: type=ParameterValueType.STRING, range=["a", "b", "c"] ), "Param3": RangeExpressionTaskParameterDefinition_2023_09( - type=ParameterValueType.INT, range="-1--2:-1" + type=ParameterValueType.INT, range=IntRangeExpr.from_str("-1--2:-1") ), }, combination="Param1 * Param2 * Param3", ) # WHEN - result = StepParameterSpaceIterator(space=space) + it = StepParameterSpaceIterator(space=space) # THEN element: Callable[[int, str, int], dict[str, ParameterValue]] = lambda p1, p2, p3: { @@ -342,20 +373,33 @@ def test_product_getitem(self) -> None: element(2, "c", -2), ] with pytest.raises(IndexError): - result[len(expected_values)] + it[len(expected_values)] with pytest.raises(IndexError): - result[-len(expected_values) - 1] - assert expected_values == [result[i] for i in range(0, len(expected_values))] + it[-len(expected_values) - 1] + assert expected_values == [it[i] for i in range(0, len(expected_values))] expected_reversed = expected_values.copy() expected_reversed.reverse() - assert expected_reversed == [result[-i - 1] for i in range(0, len(expected_values))] + assert expected_reversed == [it[-i - 1] for i in range(0, len(expected_values))] + + for value in expected_values: + assert value in it + assert element(1, "A", -1) not in it + assert element(1, "c", 0) not in it + assert element(2, "a", -3) not in it + assert element(2, "a", 0) not in it + assert {} not in it + assert { + "Param1": ParameterValue(type=ParameterValueType.INT, value="1"), + "Param2": ParameterValue(type=ParameterValueType.PATH, value="A"), + "Param3": ParameterValue(type=ParameterValueType.INT, value="-1"), + } not in it def test_associate_iteration(self) -> None: # GIVEN space = StepParameterSpace_2023_09( taskParameterDefinitions={ "Param1": RangeExpressionTaskParameterDefinition_2023_09( - type=ParameterValueType.INT, range="1-4" + type=ParameterValueType.INT, range=IntRangeExpr.from_str("1-4") ), "Param2": RangeListTaskParameterDefinition_2023_09( type=ParameterValueType.STRING, range=["a", "b", "c", "d"] @@ -395,7 +439,7 @@ def test_associate_len(self) -> None: type=ParameterValueType.STRING, range=["a", "b", "c", "d"] ), "Param3": RangeExpressionTaskParameterDefinition_2023_09( - type=ParameterValueType.INT, range="-1--4:-1" + type=ParameterValueType.INT, range=IntRangeExpr.from_str("-1--4:-1") ), }, combination="(Param1, Param2, Param3)", @@ -414,7 +458,7 @@ def test_associate_getitem(self) -> None: space = StepParameterSpace_2023_09( taskParameterDefinitions={ "Param1": RangeExpressionTaskParameterDefinition_2023_09( - type=ParameterValueType.INT, range="1-4" + type=ParameterValueType.INT, range=IntRangeExpr.from_str("1-4") ), "Param2": RangeListTaskParameterDefinition_2023_09( type=ParameterValueType.STRING, range=["a", "b", "c", "d"] @@ -427,7 +471,7 @@ def test_associate_getitem(self) -> None: ) # WHEN - space_iter = StepParameterSpaceIterator(space=space) + it = StepParameterSpaceIterator(space=space) # THEN element: Callable[[int, str, int], dict[str, ParameterValue]] = lambda p1, p2, p3: { @@ -442,13 +486,27 @@ def test_associate_getitem(self) -> None: element(4, "d", -4), ] with pytest.raises(IndexError): - space_iter[len(expected_values)] + it[len(expected_values)] with pytest.raises(IndexError): - space_iter[-len(expected_values) - 1] - assert expected_values == [space_iter[i] for i in range(0, len(expected_values))] + it[-len(expected_values) - 1] + assert expected_values == [it[i] for i in range(0, len(expected_values))] expected_reversed = expected_values.copy() expected_reversed.reverse() - assert expected_reversed == [space_iter[-i - 1] for i in range(0, len(expected_values))] + assert expected_reversed == [it[-i - 1] for i in range(0, len(expected_values))] + + # Validate that __contains__ returns True for all values included + for v in it: + assert v in it + # Validate that __contains__ returns False for some values outside + assert element(5, "a", -1) not in it + assert element(4, "d", -3) not in it + assert element(2, "c", -2) not in it + assert {} not in it + assert { + "Param1": ParameterValue(type=ParameterValueType.INT, value="4"), + "Param2": ParameterValue(type=ParameterValueType.STRING, value="d"), + "Param3": ParameterValue(type=ParameterValueType.FLOAT, value="-2"), + } not in it def test_nested_expr_iteration(self) -> None: # A more deeply nested test to hit all of the recursive edge cases. @@ -464,7 +522,7 @@ def test_nested_expr_iteration(self) -> None: type=ParameterValueType.STRING, range=["a", "b", "c", "d"] ), "Param3": RangeExpressionTaskParameterDefinition_2023_09( - type=ParameterValueType.INT, range="10-11" + type=ParameterValueType.INT, range=IntRangeExpr.from_str("10-11") ), "Param4": RangeListTaskParameterDefinition_2023_09( type=ParameterValueType.INT, range=["20", "21"] @@ -474,7 +532,7 @@ def test_nested_expr_iteration(self) -> None: ) # WHEN - result = StepParameterSpaceIterator(space=space) + it = StepParameterSpaceIterator(space=space) # THEN element: Callable[[int, str, int, int], dict[str, ParameterValue]] = ( @@ -495,4 +553,17 @@ def test_nested_expr_iteration(self) -> None: element(2, "c", 11, 20), element(2, "d", 11, 21), ] - assert expected_values == [v for v in result] + assert expected_values == [v for v in it] + + for value in expected_values: + assert value in it + assert element(1, "a", 10, 19) not in it + assert element(1, "a", 10, 22) not in it + assert element(0, "a", 10, 20) not in it + assert {} not in it + assert { + "Param1": ParameterValue(type=ParameterValueType.INT, value="1"), + "Param2": ParameterValue(type=ParameterValueType.STRING, value="c"), + "Param3": ParameterValue(type=ParameterValueType.STRING, value="11"), + "Param4": ParameterValue(type=ParameterValueType.INT, value="20"), + } not in it diff --git a/test/openjd/model/test_step_param_space_iter_with_chunks.py b/test/openjd/model/test_step_param_space_iter_with_chunks.py index 9bd16f4..2222bfd 100644 --- a/test/openjd/model/test_step_param_space_iter_with_chunks.py +++ b/test/openjd/model/test_step_param_space_iter_with_chunks.py @@ -34,8 +34,10 @@ defaultTaskCount=1, rangeConstraint=TaskChunksRangeConstraint_2023_09.CONTIGUOUS ), ), - ["1-1", "2-2"], + ["1-1", "2-2"], # [v for v in it] False, + ["1-2"], # "v in it" returns True + ["1", "2", "0-1"], # "v in it" returns False id="contig chunks, chunksize 1, range is short list", ), pytest.param( @@ -46,8 +48,10 @@ defaultTaskCount=2, rangeConstraint=TaskChunksRangeConstraint_2023_09.CONTIGUOUS ), ), - ["1-2"], + ["1-2"], # [v for v in it] False, + ["1-1", "2-2"], # "v in it" returns True + ["1", "2", "0-1"], # "v in it" returns False id="contig chunks, chunksize 2, range is short list", ), pytest.param( @@ -58,8 +62,10 @@ defaultTaskCount=1, rangeConstraint=TaskChunksRangeConstraint_2023_09.CONTIGUOUS ), ), - ["1-1", "2-2"], + ["1-1", "2-2"], # [v for v in it] False, + ["1-2"], # "v in it" returns True + ["1", "2", "0-1", "3-"], # "v in it" returns False id="contig chunks, chunksize 1, range is short range expr", ), pytest.param( @@ -70,8 +76,10 @@ defaultTaskCount=2, rangeConstraint=TaskChunksRangeConstraint_2023_09.CONTIGUOUS ), ), - ["1-2"], + ["1-2"], # [v for v in it] False, + ["1-1", "2-2"], # "v in it" returns True + ["1", "2", "0-1"], # "v in it" returns False id="contig chunks, chunksize 2, range is short range expr", ), pytest.param( @@ -82,8 +90,10 @@ defaultTaskCount=1, rangeConstraint=TaskChunksRangeConstraint_2023_09.NONCONTIGUOUS ), ), - ["1", "2"], + ["1", "2"], # [v for v in it] False, + ["1-1", "2-2", "1-2", "1-2:1"], # "v in it" returns True + ["0", "0-1"], # "v in it" returns False id="noncontig chunks, chunksize 1, range is short list", ), pytest.param( @@ -94,8 +104,10 @@ defaultTaskCount=2, rangeConstraint=TaskChunksRangeConstraint_2023_09.NONCONTIGUOUS ), ), - ["1,2"], + ["1,2"], # [v for v in it] False, + ["1-1", "2-2", "1-2", "1-2:1"], # "v in it" returns True + ["0", "0-1"], # "v in it" returns False id="noncontig chunks, chunksize 2, range is short list", ), pytest.param( @@ -106,8 +118,10 @@ defaultTaskCount=1, rangeConstraint=TaskChunksRangeConstraint_2023_09.NONCONTIGUOUS ), ), - ["1", "2"], + ["1", "2"], # [v for v in it] False, + ["1-1", "2-2", "1-2", "1-2:1"], # "v in it" returns True + ["0", "0-1"], # "v in it" returns False id="noncontig chunks, chunksize 1, range is short range expr", ), pytest.param( @@ -118,8 +132,10 @@ defaultTaskCount=2, rangeConstraint=TaskChunksRangeConstraint_2023_09.NONCONTIGUOUS ), ), - ["1,2"], + ["1,2"], # [v for v in it] False, + ["1-1", "2-2", "1-2", "1-2:1"], # "v in it" returns True + ["0", "0-1"], # "v in it" returns False id="noncontig chunks, chunksize 2, range is short range expr", ), pytest.param( @@ -130,8 +146,10 @@ defaultTaskCount=100, rangeConstraint=TaskChunksRangeConstraint_2023_09.CONTIGUOUS ), ), - ["1-1", "3-3", "5-5"], + ["1-1", "3-3", "5-5"], # [v for v in it] False, + [], # "v in it" returns True + ["0", "0-1", "2-2", "1-2", "1-2:1"], # "v in it" returns False id="contig chunks, chunksize 100, range is noncontig", ), pytest.param( @@ -143,8 +161,10 @@ rangeConstraint=TaskChunksRangeConstraint_2023_09.NONCONTIGUOUS, ), ), - ["1-5:2"], + ["1-5:2"], # [v for v in it] False, + ["1", "3", "5", "1-1", "3-3", "5-5", "1,3", "1-3:2", "1,3,5"], # "v in it" returns True + ["1-3", "1-5", "0", "2", "4", "6", "X", "--3"], # "v in it" returns False id="noncontig chunks, chunksize 100, range is noncontig", ), pytest.param( @@ -156,8 +176,10 @@ ), ), # Non-adaptive spreads out the chunks evenly - ["1-9", "10-18", "19-27", "28-35"], + ["1-9", "10-18", "19-27", "28-35"], # [v for v in it] False, + ["1-35", "1-1", "35-35"], # "v in it" returns True + ["0-0", "1", "35", "36-36", "0-35", "1-36"], # "v in it" returns False id="contig chunks, chunksize 10, range 1-35, non-adaptive", ), pytest.param( @@ -171,8 +193,10 @@ ), ), # Adaptive makes chunks as big as possible, so the last chunk ends up smaller - ["1-10", "11-20", "21-30", "31-35"], + ["1-10", "11-20", "21-30", "31-35"], # [v for v in it] True, + ["1-35", "1", "1-1", "35", "35-35"], # "v in it" returns True + ["0", "0-0", "36", "36-36", "0-35", "1-36", "12-"], # "v in it" returns False id="noncontig chunks, chunksize 10, range 1-35, adaptive", ), pytest.param( @@ -184,8 +208,19 @@ ), ), # Non-adaptive spreads out the chunks evenly - ["-20--17", "-16--13", "-12--9", "-8--5"], + ["-20--17", "-16--13", "-12--9", "-8--5"], # [v for v in it] False, + ["-20--5", "-20--10"], # "v in it" returns True + [ + "-21", + "-20--4", + "-", + "", + "-19", + "-5", + "-20,-19,-18,-15", + "-20--25:-3", + ], # "v in it" returns False id="contig chunks, chunksize 5, negative frames, non-adaptive", ), pytest.param( @@ -199,8 +234,20 @@ ), ), # Adaptive makes chunks as big as possible, so the last chunk ends up smaller - ["-20--16", "-15--11", "-10--6", "-5--5"], + ["-20--16", "-15--11", "-10--6", "-5--5"], # [v for v in it] True, + ["-20--5", "-20--10"], # "v in it" returns True + [ + "-21", + "-20--4", + "-", + "", + "-19", + "-5", + "-20,-19,-18,-15", + "-20--25:-3", + "--13", + ], # "v in it" returns False id="contig chunks, chunksize 5, negative frames, adaptive", ), pytest.param( @@ -214,15 +261,22 @@ ), ), # Adaptive makes chunks as big as possible, so the last chunk ends up smaller - ["-20--16", "-15--11", "-10--6", "-5"], + ["-20--16", "-15--11", "-10--6", "-5"], # [v for v in it] True, + ["-20--5", "-20", "-19", "-5", "-20,-19,-18,-15", "-6--18:-3"], # "v in it" returns True + ["-21", "-20--4", "-", "", "-20--25:-3"], # "v in it" returns False id="noncontig chunks, chunksize 5, negative frames, adaptive", ), ) -@pytest.mark.parametrize("range_int_param,expected,chunks_adaptive", PARAMETRIZE_CASES) -def test_single_param_chunked_iteration(range_int_param, expected, chunks_adaptive): +@pytest.mark.parametrize( + "range_int_param,expected,chunks_adaptive,expected_contains,expected_not_contains", + PARAMETRIZE_CASES, +) +def test_single_param_chunked_iteration( + range_int_param, expected, chunks_adaptive, expected_contains, expected_not_contains +): # GIVEN space = StepParameterSpace_2023_09( taskParameterDefinitions={ @@ -245,6 +299,13 @@ def test_single_param_chunked_iteration(range_int_param, expected, chunks_adapti assert [v for v in it] == [ {"Param1": ParameterValue(type=ParameterValueType.CHUNK_INT, value=v)} for v in expected ] + # Check that __contains__ is True/False for all provided cases + for v in expected: + assert {"Param1": ParameterValue(type=ParameterValueType.CHUNK_INT, value=v)} in it + for v in expected_contains: + assert {"Param1": ParameterValue(type=ParameterValueType.CHUNK_INT, value=v)} in it + for v in expected_not_contains: + assert {"Param1": ParameterValue(type=ParameterValueType.CHUNK_INT, value=v)} not in it # Check that the length and indexing work as expected it.reset_iter() if chunks_adaptive: @@ -264,6 +325,8 @@ def test_single_param_chunked_iteration(range_int_param, expected, chunks_adapti with pytest.raises(ValueError): it.chunks_default_task_count = 1 + assert it.chunks_parameter_name == "Param1" + PARAMETRIZE_CASES = ( pytest.param( @@ -280,10 +343,34 @@ def test_single_param_chunked_iteration(range_int_param, expected, chunks_adapti range=["A", "B"], ), }, - [("1-1", "A"), ("1-1", "B"), ("2-2", "A"), ("2-2", "B")], - False, + [("1-1", "A"), ("1-1", "B"), ("2-2", "A"), ("2-2", "B")], # [v for v in it] + False, # chunks_adaptive + None, # override_chunk_size + [("1-2", "A"), ("1-2", "B")], # "v in it" returns True + [("1", "A"), ("2", "A"), ("1", "B"), ("2", "B"), ("1-1", "C")], # "v in it" returns False id="2 dim, chunked outer, chunksize 1, non-adaptive", ), + pytest.param( + { + "Param1": RangeListTaskParameterDefinition_2023_09( + type=ParameterValueType.CHUNK_INT, + range=["1", "2"], + chunks=TaskChunksDefinition_2023_09( + defaultTaskCount=1, rangeConstraint=TaskChunksRangeConstraint_2023_09.CONTIGUOUS + ), + ), + "Param2": RangeListTaskParameterDefinition_2023_09( + type=ParameterValueType.STRING, + range=["A", "B"], + ), + }, + [("1-2", "A"), ("1-2", "B")], # [v for v in it] + False, # chunks_adaptive + 5, # override_chunk_size + [("1-1", "A"), ("2-2", "A"), ("1-1", "B"), ("2-2", "B")], # "v in it" returns True + [("1", "A"), ("2", "A"), ("1", "B"), ("2", "B"), ("1-1", "AB")], # "v in it" returns False + id="2 dim, chunked outer, chunksize 1, non-adaptive, override chunksize 5", + ), pytest.param( { "Param1": RangeListTaskParameterDefinition_2023_09( @@ -298,10 +385,34 @@ def test_single_param_chunked_iteration(range_int_param, expected, chunks_adapti range=["A", "B"], ), }, - [("1-2", "A"), ("1-2", "B")], - False, + [("1-2", "A"), ("1-2", "B")], # [v for v in it] + False, # chunks_adaptive + None, # override_chunk_size + [("1-1", "A"), ("2-2", "A"), ("1-1", "B"), ("2-2", "B")], # "v in it" returns True + [("1", "A"), ("2", "A"), ("1", "B"), ("2", "B"), ("1-1", "AB")], # "v in it" returns False id="2 dim, chunked outer, chunksize 2, non-adaptive", ), + pytest.param( + { + "Param1": RangeListTaskParameterDefinition_2023_09( + type=ParameterValueType.CHUNK_INT, + range=["1", "2"], + chunks=TaskChunksDefinition_2023_09( + defaultTaskCount=2, rangeConstraint=TaskChunksRangeConstraint_2023_09.CONTIGUOUS + ), + ), + "Param2": RangeListTaskParameterDefinition_2023_09( + type=ParameterValueType.STRING, + range=["A", "B"], + ), + }, + [("1-1", "A"), ("1-1", "B"), ("2-2", "A"), ("2-2", "B")], # [v for v in it] + False, # chunks_adaptive + 1, # override_chunk_size + [("1-2", "A"), ("1-2", "B")], # "v in it" returns True + [("1", "A"), ("2", "A"), ("1", "B"), ("2", "B"), ("1-1", "C")], # "v in it" returns False + id="2 dim, chunked outer, chunksize 2, non-adaptive, override chunksize 1", + ), pytest.param( { "Param1": RangeListTaskParameterDefinition_2023_09( @@ -320,8 +431,11 @@ def test_single_param_chunked_iteration(range_int_param, expected, chunks_adapti }, # The order is different from the equivalent non-adaptive, because the chunked dimension # is moved to the inside - [("1-1", "A"), ("2-2", "A"), ("1-1", "B"), ("2-2", "B")], - True, + [("1-1", "A"), ("2-2", "A"), ("1-1", "B"), ("2-2", "B")], # [v for v in it] + True, # chunks_adaptive + None, # override_chunk_size + [("1-2", "A"), ("1-2", "B")], # "v in it" returns True + [("1", "A"), ("2", "A"), ("1", "B"), ("2", "B"), ("0-0", "A")], # "v in it" returns False id="2 dim, chunked outer, chunksize 1, adaptive", ), pytest.param( @@ -340,40 +454,79 @@ def test_single_param_chunked_iteration(range_int_param, expected, chunks_adapti range=["A", "B"], ), }, - [("1-2", "A"), ("1-2", "B")], - True, + [("1-2", "A"), ("1-2", "B")], # [v for v in it] + True, # chunks_adaptive + None, # override_chunk_size + [("1-1", "A"), ("2-2", "A"), ("1-1", "B"), ("2-2", "B")], # "v in it" returns True + [("1", "A"), ("2", "A"), ("1", "B"), ("2", "B"), ("3-3", "B")], # "v in it" returns False id="2 dim, chunked outer, chunksize 2, adaptive", ), + pytest.param( + { + "Param1": RangeListTaskParameterDefinition_2023_09( + type=ParameterValueType.CHUNK_INT, + range=["1", "2"], + chunks=TaskChunksDefinition_2023_09( + defaultTaskCount=2, + targetRuntimeSeconds=20, + rangeConstraint=TaskChunksRangeConstraint_2023_09.NONCONTIGUOUS, + ), + ), + "Param2": RangeListTaskParameterDefinition_2023_09( + type=ParameterValueType.STRING, + range=["A", "B"], + ), + }, + [("1", "A"), ("1", "B"), ("2", "A"), ("2", "B")], # [v for v in it] + False, # chunks_adaptive + 1, # override_chunk_size + [("1-1", "A"), ("2-2", "A"), ("1-1", "B"), ("2-2", "B")], # "v in it" returns True + [("1", "C"), ("3-3", "B")], # "v in it" returns False + id="2 dim, chunked outer, chunksize 2, adaptive, noncontig, override chunksize 1 (turns off adaptive)", + ), ) -@pytest.mark.parametrize("param_defs,expected,chunks_adaptive", PARAMETRIZE_CASES) -def test_multi_param_chunked_iteration(param_defs: dict[str, Any], expected, chunks_adaptive): +@pytest.mark.parametrize( + "param_defs,expected,chunks_adaptive,override_chunk_size,expected_contains,expected_not_contains", + PARAMETRIZE_CASES, +) +def test_multi_param_chunked_iteration( + param_defs: dict[str, Any], + expected, + chunks_adaptive, + override_chunk_size, + expected_contains, + expected_not_contains, +): # GIVEN space = StepParameterSpace_2023_09(taskParameterDefinitions=param_defs) # WHEN - it = StepParameterSpaceIterator(space=space) + it = StepParameterSpaceIterator(space=space, chunks_task_count_override=override_chunk_size) # THEN - assert it.chunks_adaptive == chunks_adaptive - # Check that full iteration over the range gives the expected result - assert [v for v in it] == [ - { + def element(tup): + return { n: ParameterValue(type=ParameterValueType(param.type), value=v) - for (n, param), v in zip(param_defs.items(), item) + for (n, param), v in zip(param_defs.items(), tup) } - for item in expected - ] + + expected_values = [element(tup) for tup in expected] + assert it.chunks_adaptive == chunks_adaptive + # Check that full iteration over the range gives the expected result + assert [v for v in it] == expected_values # Check that resetting the iterator and re-iterating produces the same again it.reset_iter() - assert [v for v in it] == [ - { - n: ParameterValue(type=ParameterValueType(param.type), value=v) - for (n, param), v in zip(param_defs.items(), item) - } - for item in expected - ] + assert [v for v in it] == expected_values + # Check that __contains__ is True/False for all provided cases + for value in expected_values: + assert value in it + for tup in expected_contains: + assert element(tup) in it + for tup in expected_not_contains: + assert element(tup) not in it + assert it.chunks_parameter_name == "Param1" def test_adaptive_contiguous_chunked_iteration(): @@ -422,6 +575,8 @@ def make_item(v0, v1): with pytest.raises(StopIteration): next(it) + assert it.chunks_parameter_name == "P1" + def test_adaptive_noncontiguous_chunked_iteration(): # GIVEN @@ -466,6 +621,8 @@ def make_item(v0, v1): with pytest.raises(StopIteration): next(it) + assert it.chunks_parameter_name == "P1" + def test_divide_chunk_sizes(): # Some edge cases diff --git a/test/openjd/model/v2023_09/test_template_variables.py b/test/openjd/model/v2023_09/test_template_variables.py index da4c1cd..40f89c6 100644 --- a/test/openjd/model/v2023_09/test_template_variables.py +++ b/test/openjd/model/v2023_09/test_template_variables.py @@ -43,7 +43,7 @@ def make_script(env_or_task: str, scriptname: str, scriptdata: str) -> dict: } -class TestJobTemplate: +class TestJobTemplateVars: @pytest.mark.parametrize( "data", (