diff --git a/examples/recipes/impira_layoutlm-invoices/cpu/cpu/question-answering_fp16_config.json b/examples/recipes/impira_layoutlm-invoices/cpu/cpu/question-answering_fp16_config.json new file mode 100644 index 000000000..0dd56a0c3 --- /dev/null +++ b/examples/recipes/impira_layoutlm-invoices/cpu/cpu/question-answering_fp16_config.json @@ -0,0 +1,99 @@ +{ + "export": { + "opset_version": 17, + "batch_size": 1, + "export_params": true, + "do_constant_folding": true, + "verbose": false, + "dynamo": false, + "enable_hierarchy_tags": true, + "clean_onnx": false, + "hierarchy_tag_format": "full", + "input_tensors": [ + { + "name": "input_ids", + "dtype": "int32", + "shape": [ + 1, + 512 + ], + "value_range": [ + 0, + 50265 + ] + }, + { + "name": "bbox", + "dtype": "int32", + "shape": [ + 1, + 512, + 4 + ], + "value_range": [ + 0, + 1001 + ] + }, + { + "name": "attention_mask", + "dtype": "int32", + "shape": [ + 1, + 512 + ], + "value_range": [ + 0, + 2 + ] + }, + { + "name": "token_type_ids", + "dtype": "int32", + "shape": [ + 1, + 512 + ], + "value_range": [ + 0, + 1 + ] + } + ], + "output_tensors": [ + { + "name": "start_logits" + }, + { + "name": "end_logits" + } + ] + }, + "optim": {}, + "quant": { + "mode": "fp16", + "samples": 10, + "calibration_method": "minmax", + "weight_type": "uint8", + "activation_type": "uint8", + "per_channel": false, + "symmetric": false, + "weight_symmetric": null, + "activation_symmetric": null, + "save_calibration": false, + "distribution": "uniform", + "seed": null, + "calibration_load_path": null, + "calibration_save_path": null, + "op_types_to_quantize": null, + "nodes_to_exclude": null, + "fp16_keep_io_types": true, + "fp16_op_block_list": null + }, + "compile": null, + "loader": { + "task": "question-answering", + "model_class": "LayoutLMForQuestionAnswering", + "model_type": "layoutlm" + } +} diff --git a/examples/recipes/impira_layoutlm-invoices/cpu/cpu/question-answering_fp32_config.json b/examples/recipes/impira_layoutlm-invoices/cpu/cpu/question-answering_fp32_config.json new file mode 100644 index 000000000..ecad8d815 --- /dev/null +++ b/examples/recipes/impira_layoutlm-invoices/cpu/cpu/question-answering_fp32_config.json @@ -0,0 +1,82 @@ +{ + "export": { + "opset_version": 17, + "batch_size": 1, + "export_params": true, + "do_constant_folding": true, + "verbose": false, + "dynamo": false, + "enable_hierarchy_tags": true, + "clean_onnx": false, + "hierarchy_tag_format": "full", + "input_tensors": [ + { + "name": "input_ids", + "dtype": "int32", + "shape": [ + 1, + 512 + ], + "value_range": [ + 0, + 50265 + ] + }, + { + "name": "bbox", + "dtype": "int32", + "shape": [ + 1, + 512, + 4 + ], + "value_range": [ + 0, + 1001 + ] + }, + { + "name": "attention_mask", + "dtype": "int32", + "shape": [ + 1, + 512 + ], + "value_range": [ + 0, + 2 + ] + }, + { + "name": "token_type_ids", + "dtype": "int32", + "shape": [ + 1, + 512 + ], + "value_range": [ + 0, + 1 + ] + } + ], + "output_tensors": [ + { + "name": "start_logits" + }, + { + "name": "end_logits" + } + ] + }, + "optim": { + + }, + "quant": null, + "compile": null, + "loader": { + "task": "question-answering", + "model_class": "LayoutLMForQuestionAnswering", + "model_type": "layoutlm" + } +} diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index 30d533699..d81dd6771 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -262,6 +262,7 @@ def generate_random_inputs( symbolic_shapes = io_config.get("input_symbolic_shapes") or [ [None] * len(s) for s in io_config["input_shapes"] ] + value_ranges = io_config.get("input_value_ranges") or {} overrides = shape_config or {} specs: dict[str, dict[str, Any]] = {} @@ -290,6 +291,12 @@ def generate_random_inputs( "dtype": gen_dtype, "shape": list(resolved_shape), } + if name in value_ranges: + lo, hi = value_ranges[name] + # Build-config ranges are high-exclusive. The legacy NumPy + # generator accepts an inclusive integer maximum, so adapt only + # that boundary; float generation already uses [lo, hi). + specs[name]["range"] = [lo, hi - 1] if gen_dtype == "int" else [lo, hi] return generate_dummy_inputs_from_specs(specs) diff --git a/src/winml/modelkit/core/model_input_generator.py b/src/winml/modelkit/core/model_input_generator.py index c5623d997..bd5cbbc42 100644 --- a/src/winml/modelkit/core/model_input_generator.py +++ b/src/winml/modelkit/core/model_input_generator.py @@ -36,6 +36,28 @@ logger = logging.getLogger(__name__) +def _generate_ordered_bounding_boxes( + shape: list[int], min_value: int, max_value: int +) -> np.ndarray: + """Generate positive-area ``[x1, y1, x2, y2]`` boxes within inclusive bounds.""" + if max_value <= min_value: + raise ValueError("Bounding-box range must contain at least two coordinates") + + coordinates = np.random.randint(min_value, max_value + 1, size=shape) + x1 = np.minimum(coordinates[..., 0], coordinates[..., 2]) + y1 = np.minimum(coordinates[..., 1], coordinates[..., 3]) + x2 = np.maximum(coordinates[..., 0], coordinates[..., 2]) + y2 = np.maximum(coordinates[..., 1], coordinates[..., 3]) + + x_equal = x1 == x2 + y_equal = y1 == y2 + x1 = np.where(x_equal & (x2 == max_value), x1 - 1, x1) + y1 = np.where(y_equal & (y2 == max_value), y1 - 1, y1) + x2 = np.where(x_equal & (x2 < max_value), x2 + 1, x2) + y2 = np.where(y_equal & (y2 < max_value), y2 + 1, y2) + return np.stack((x1, y1, x2, y2), axis=-1).astype(np.int64) + + def generate_dummy_inputs_from_specs( input_specs: dict[str, dict[str, Any]], ) -> dict[str, np.ndarray]: @@ -106,7 +128,14 @@ def generate_dummy_inputs_from_specs( min_val, max_val = spec["range"] if dtype == np.int64: - inputs[name] = np.random.randint(min_val, max_val + 1, size=shape).astype(dtype) + if name == "bbox" and shape and shape[-1] == 4: + inputs[name] = _generate_ordered_bounding_boxes( + shape, int(min_val), int(max_val) + ) + else: + inputs[name] = np.random.randint(min_val, max_val + 1, size=shape).astype( + dtype + ) else: inputs[name] = np.random.rand(*shape).astype(dtype) * (max_val - min_val) + min_val else: diff --git a/src/winml/modelkit/loader/resolution.py b/src/winml/modelkit/loader/resolution.py index 548848175..659f3c3a5 100644 --- a/src/winml/modelkit/loader/resolution.py +++ b/src/winml/modelkit/loader/resolution.py @@ -132,6 +132,35 @@ class from the ``transformers`` package. ) from e +def _resolve_architecture_mapping( + config: PretrainedConfig, model_type_normalized: str +) -> tuple[str, type] | None: + """Resolve an unambiguous registered task from the checkpoint architecture. + + Per-architecture ``MODEL_CLASS_MAPPING`` entries normally override the loader + class after task detection. They also provide stronger task evidence when the + concrete class in ``config.architectures`` exactly matches a registered class. + This corrects incomplete upstream task inference without a model-ID default. + """ + try: + architecture_class = _resolve_model_class_from_config(config) + except ValueError: + return None + + from ..models.hf import MODEL_CLASS_MAPPING + + matches = { + task + for (model_type, task), model_class in MODEL_CLASS_MAPPING.items() + if model_type == model_type_normalized + and task is not None + and model_class is architecture_class + } + if len(matches) != 1: + return None + return matches.pop(), architecture_class + + def _detect_task_from_model_class(model_class: type) -> str: """Detect task from a model class via TasksManager. @@ -251,6 +280,7 @@ class TaskSource(str, Enum): USER_CLASS = "user-class" # user passed --model-class; task inferred MODEL_ID_DEFAULT = "model-id-default" # MODEL_TASK_MAPPING model-id default SENTINEL_DEFAULT = "sentinel-default" # (model_type, None) sentinel + ARCHITECTURE_MAPPING = "architecture-mapping" # exact registered architecture class TASKS_MANAGER = "tasks-manager" # Optimum inference (incl. fill-mask upgrade) WRAPPED_LIBRARY = "wrapped-library" # no architectures -> first supported task PIPELINE_TAG = "pipeline-tag" # Hub pipeline_tag fallback @@ -589,7 +619,14 @@ def resolve_task( else TaskSource.SENTINEL_DEFAULT ) - # 1b. no architectures -> first ONNX-exportable task + # 1b. exact architecture class -> registered task/class mapping + if opt_task is None and model_type_norm: + architecture_mapping = _resolve_architecture_mapping(config, model_type_norm) + if architecture_mapping is not None: + opt_task, resolved = architecture_mapping + source = TaskSource.ARCHITECTURE_MAPPING + + # 1c. no architectures -> first ONNX-exportable task # (merges the old timm wrapped-library stage AND the --model-type fallback) if opt_task is None and not getattr(config, "architectures", None) and model_type: # Populate Optimum's ONNX export-config registry before querying it; @@ -604,7 +641,7 @@ def resolve_task( # lookup failure here — e.g. a wrapped library whose classes aren't registered # under framework="pt" — can't escape as a raw KeyError. - # 1c. TasksManager (reads config.architectures) + # 1d. TasksManager (reads config.architectures) if opt_task is None: try: opt_task = _infer_task_from_architecture(config) @@ -612,7 +649,7 @@ def resolve_task( except ValueError: opt_task = None - # 1d. Hub pipeline_tag fallback + # 1e. Hub pipeline_tag fallback if opt_task is None and model_id and model_type: from ..utils.hub_utils import get_pipeline_tag @@ -625,7 +662,7 @@ def resolve_task( # reinforcement-learning, time-series-forecasting). Admitting one would flow a # non-exportable task into Stage 2 (model-class) / Stage 3 instead of degrading # to the last-resort default. Populate Optimum's ONNX export-config registry - # first (as Stage 1b does) so get_supported_tasks doesn't return []. + # first (as Stage 1c does) so get_supported_tasks doesn't return []. import optimum.exporters.onnx.model_configs # noqa: F401 if normalized_tag in get_supported_tasks( @@ -634,12 +671,12 @@ def resolve_task( opt_task = normalized_tag source = TaskSource.PIPELINE_TAG - # 1e. last-resort default + # 1f. last-resort default if opt_task is None: opt_task = next(iter(HF_TASK_DEFAULTS)) source = TaskSource.HF_TASK_DEFAULT - # --- Stage 2: model class (if not already resolved in 1b) ------------- + # --- Stage 2: model class (if not already resolved during detection) -- if resolved is None: resolved = _get_custom_model_class(model_type_norm, opt_task) if resolved is None: @@ -654,6 +691,6 @@ def resolve_task( # --- Stage 4: composite tag (detection path) -------------------------- composite = _composite_components_for_task(model_type, opt_task) if model_type else None - if source is None: # structural invariant: Stage 1d always sets a source + if source is None: # structural invariant: Stage 1 always sets a source raise RuntimeError("resolve_task: internal invariant violated — source was not set") return TaskResolution(surfaced, to_optimum_task(surfaced), resolved, source, composite) diff --git a/src/winml/modelkit/models/hf/__init__.py b/src/winml/modelkit/models/hf/__init__.py index 0339b2b8e..005abda48 100644 --- a/src/winml/modelkit/models/hf/__init__.py +++ b/src/winml/modelkit/models/hf/__init__.py @@ -48,6 +48,7 @@ from .depth_anything import DepthAnythingIOConfig as _DepthAnythingIOConfig # triggers registration from .depth_pro import DepthProIOConfig as _DepthProIOConfig # triggers registration from .detr import DETR_CONFIG +from .layoutlm import MODEL_CLASS_MAPPING as _LAYOUTLM_CLASS_MAPPING from .layoutlm import LayoutLMQAIOConfig as _LayoutLMQAIOConfig # triggers registration from .marian import MARIAN_CONFIG from .marian import MODEL_CLASS_MAPPING as _MARIAN_CLASS_MAPPING @@ -121,6 +122,7 @@ _BART_CLASS_MAPPING, _BLIP_CLASS_MAPPING, _CLIP_CLASS_MAPPING, + _LAYOUTLM_CLASS_MAPPING, _MARIAN_CLASS_MAPPING, _MU2_CLASS_MAPPING, _QWEN_CLASS_MAPPING, diff --git a/src/winml/modelkit/models/hf/layoutlm.py b/src/winml/modelkit/models/hf/layoutlm.py index 8085eb0ef..7ac1ba68a 100644 --- a/src/winml/modelkit/models/hf/layoutlm.py +++ b/src/winml/modelkit/models/hf/layoutlm.py @@ -11,6 +11,7 @@ from optimum.exporters.onnx.model_configs import LayoutLMOnnxConfig from optimum.utils import NormalizedTextConfig from optimum.utils.input_generators import DummyBboxInputGenerator, DummyVisionInputGenerator +from transformers import LayoutLMForQuestionAnswering from ...export import MaxLengthTextInputGenerator, register_onnx_overwrite @@ -19,8 +20,54 @@ import torch +MODEL_CLASS_MAPPING: dict[tuple[str, str], type] = { + ("layoutlm", "question-answering"): LayoutLMForQuestionAnswering, +} + + +class _LayoutLMNormalizedConfig(NormalizedTextConfig): # type: ignore[misc] + """LayoutLM text config with a metadata-derived usable sequence length.""" + + @property + def sequence_length(self) -> int: + """Account for the padding offset used by RoBERTa-style checkpoints.""" + max_positions = int(self.config.max_position_embeddings) + padding_idx = getattr(self.config, "pad_token_id", None) + if padding_idx: + return max_positions - int(padding_idx) - 1 + return max_positions + + @property + def bbox_coordinate_range(self) -> int: + """Return the high-exclusive normalized bbox coordinate bound.""" + max_2d_positions = int(self.config.max_2d_position_embeddings) + coordinate_range = min(1001, max_2d_positions) + if coordinate_range < 2: + raise ValueError("LayoutLM requires at least two 2D embedding positions") + return coordinate_range + + class ZeroTokenTypeLayoutLMTextInputGenerator(MaxLengthTextInputGenerator): - """LayoutLM text dummy generator that keeps token_type_ids within type_vocab_size=1.""" + """LayoutLM text dummy generator that bounds token_type_ids by type_vocab_size.""" + + def __init__( + self, + task: str, + normalized_config: NormalizedTextConfig, + sequence_length: int | None = None, + **kwargs: Any, + ) -> None: + """Cap generic shape inference to LayoutLM's usable position range.""" + self._type_vocab_size = max(1, int(normalized_config.config.type_vocab_size)) + usable_length = normalized_config.sequence_length + if sequence_length is None or sequence_length > usable_length: + sequence_length = usable_length + super().__init__( + task, + normalized_config, + sequence_length=sequence_length, + **kwargs, + ) def generate( self, @@ -29,8 +76,18 @@ def generate( int_dtype: str = "int64", float_dtype: str = "fp32", ) -> torch.Tensor: - """Generate LayoutLM text inputs, replacing token_type_ids with zeros.""" - tensor = cast( + """Generate LayoutLM text inputs with metadata-bounded token type IDs.""" + if input_name == "token_type_ids": + return cast( + "torch.Tensor", + self.random_int_tensor( + [self.batch_size, self.sequence_length], + max_value=self._type_vocab_size, + framework=framework, + dtype=int_dtype, + ), + ) + return cast( "torch.Tensor", super().generate( input_name, @@ -39,9 +96,89 @@ def generate( float_dtype=float_dtype, ), ) - if input_name == "token_type_ids": - return tensor.new_zeros(tensor.shape) - return tensor + + +class UsableLengthLayoutLMBboxInputGenerator(DummyBboxInputGenerator): # type: ignore[misc] + """Generate positive-area normalized boxes aligned with the text sequence.""" + + def __init__( + self, + task: str, + normalized_config: NormalizedTextConfig, + sequence_length: int | None = None, + **kwargs: Any, + ) -> None: + """Derive safe sequence and coordinate bounds from model metadata.""" + usable_length = normalized_config.sequence_length + if sequence_length is None or sequence_length > usable_length: + sequence_length = usable_length + self.coordinate_range = normalized_config.bbox_coordinate_range + super().__init__( + task, + normalized_config, + sequence_length=sequence_length, + **kwargs, + ) + + def generate( + self, + input_name: str, + framework: str = "pt", + int_dtype: str = "int64", + float_dtype: str = "fp32", + ) -> Any: + """Generate ``[x1, y1, x2, y2]`` with positive, in-bounds width and height.""" + del input_name, float_dtype + coordinates = self.random_int_tensor( + [self.batch_size, self.sequence_length, 4], + max_value=self.coordinate_range, + framework=framework, + dtype=int_dtype, + ) + max_coordinate = self.coordinate_range - 1 + if framework == "pt": + import torch + + x1 = torch.minimum(coordinates[..., 0], coordinates[..., 2]) + y1 = torch.minimum(coordinates[..., 1], coordinates[..., 3]) + x2 = torch.maximum(coordinates[..., 0], coordinates[..., 2]) + y2 = torch.maximum(coordinates[..., 1], coordinates[..., 3]) + + # Equal corners are expanded inward so width/height embedding indexes + # stay in [1, coordinate_range) without exceeding the coordinate table. + x_equal = x1 == x2 + y_equal = y1 == y2 + x1 = torch.where(x_equal & (x2 == max_coordinate), x1 - 1, x1) + y1 = torch.where(y_equal & (y2 == max_coordinate), y1 - 1, y1) + x2 = torch.where(x_equal & (x2 < max_coordinate), x2 + 1, x2) + y2 = torch.where(y_equal & (y2 < max_coordinate), y2 + 1, y2) + return torch.stack((x1, y1, x2, y2), dim=-1) + if framework == "np": + import numpy as np + + np_x1 = np.minimum(coordinates[..., 0], coordinates[..., 2]) + np_y1 = np.minimum(coordinates[..., 1], coordinates[..., 3]) + np_x2 = np.maximum(coordinates[..., 0], coordinates[..., 2]) + np_y2 = np.maximum(coordinates[..., 1], coordinates[..., 3]) + + np_x_equal = np_x1 == np_x2 + np_y_equal = np_y1 == np_y2 + np_x1 = np.where( + np_x_equal & (np_x2 == max_coordinate), np_x1 - 1, np_x1 + ) + np_y1 = np.where( + np_y_equal & (np_y2 == max_coordinate), np_y1 - 1, np_y1 + ) + np_x2 = np.where( + np_x_equal & (np_x2 < max_coordinate), np_x2 + 1, np_x2 + ) + np_y2 = np.where( + np_y_equal & (np_y2 < max_coordinate), np_y2 + 1, np_y2 + ) + return np.stack((np_x1, np_y1, np_x2, np_y2), axis=-1) + raise ValueError( + f"LayoutLM bbox generation supports only 'pt' and 'np', got {framework!r}" + ) @register_onnx_overwrite("layoutlm", "question-answering", library_name="transformers") @@ -51,18 +188,11 @@ class LayoutLMQAIOConfig(LayoutLMOnnxConfig): # type: ignore[misc] # optimum b # sequence_length is bound to the model's max_position_embeddings so # MaxLengthTextInputGenerator emits full-length text inputs instead of # Optimum's default of 16 (allow_new=True permits adding this mapping). - # We deliberately do NOT map max_2d_position_embeddings here: Optimum's - # DummyBboxInputGenerator hardcodes its coordinate range (its - # normalized_config.max_2d_position_embeddings read is commented out - # upstream), so such a mapping is inert and never becomes a sequence - # length. bbox coordinate bounds for the shipped recipe come from the - # recipe's `value_range` instead. - NORMALIZED_CONFIG_CLASS = NormalizedTextConfig.with_args( - sequence_length="max_position_embeddings", - allow_new=True, - ) + # The custom bbox generator separately derives normalized coordinate bounds + # from max_2d_position_embeddings and emits ordered, positive-area boxes. + NORMALIZED_CONFIG_CLASS = _LayoutLMNormalizedConfig DUMMY_INPUT_GENERATOR_CLASSES: tuple[type[Any], ...] = ( ZeroTokenTypeLayoutLMTextInputGenerator, DummyVisionInputGenerator, - DummyBboxInputGenerator, + UsableLengthLayoutLMBboxInputGenerator, ) diff --git a/src/winml/modelkit/onnx/io.py b/src/winml/modelkit/onnx/io.py index d2ad85878..d622e705e 100644 --- a/src/winml/modelkit/onnx/io.py +++ b/src/winml/modelkit/onnx/io.py @@ -113,11 +113,22 @@ def to_tensor(self) -> Any: # randint path below. Reorder so x0<=x1 and y0<=y1 to keep each box # well-formed for layout models. if self.name == "bbox" and len(concrete_shape) >= 1 and concrete_shape[-1] == 4: + if int(hi) - int(lo) < 2: + raise ValueError( + "Bounding-box value_range must contain at least two coordinates" + ) coords = torch.randint(int(lo), int(hi), concrete_shape, dtype=torch_dtype) x0 = torch.minimum(coords[..., 0], coords[..., 2]) y0 = torch.minimum(coords[..., 1], coords[..., 3]) x1 = torch.maximum(coords[..., 0], coords[..., 2]) y1 = torch.maximum(coords[..., 1], coords[..., 3]) + max_coordinate = int(hi) - 1 + x_equal = x0 == x1 + y_equal = y0 == y1 + x0 = torch.where(x_equal & (x1 == max_coordinate), x0 - 1, x0) + y0 = torch.where(y_equal & (y1 == max_coordinate), y0 - 1, y0) + x1 = torch.where(x_equal & (x1 < max_coordinate), x1 + 1, x1) + y1 = torch.where(y_equal & (y1 < max_coordinate), y1 + 1, y1) return torch.stack((x0, y0, x1, y1), dim=-1) return torch.randint(int(lo), int(hi), concrete_shape, dtype=torch_dtype) diff --git a/tests/unit/commands/test_perf_cli.py b/tests/unit/commands/test_perf_cli.py index be4984977..e5bbdaafc 100644 --- a/tests/unit/commands/test_perf_cli.py +++ b/tests/unit/commands/test_perf_cli.py @@ -849,6 +849,91 @@ def test_symbolic_override_rejects_non_integer_float_value(self) -> None: ) +class TestGenerateRandomInputs: + """Generated build-config ranges must reach default perf inputs.""" + + @pytest.mark.parametrize( + "value_range", + [ + [0, 1], + [3, 7], + ], + ) + def test_integer_ranges_remain_high_exclusive(self, value_range: list[int]) -> None: + import numpy as np + + from winml.modelkit.commands.perf import generate_random_inputs + + io_config = { + "input_names": ["tokens"], + "input_shapes": [[1, 32]], + "input_types": ["int32"], + "input_value_ranges": {"tokens": value_range}, + } + + with patch("numpy.random.randint", wraps=np.random.randint) as randint: + values = generate_random_inputs(io_config)["tokens"] + + assert randint.call_args.args[:2] == tuple(value_range) + assert np.all(values >= value_range[0]) + assert np.all(values < value_range[1]) + + def test_float_range_and_unspecified_defaults_are_preserved(self) -> None: + import numpy as np + + from winml.modelkit.commands.perf import generate_random_inputs + + io_config = { + "input_names": ["ranged", "default_int", "default_float"], + "input_shapes": [[1, 1024], [1, 1024], [1, 1024]], + "input_types": ["float32", "int64", "float32"], + "input_value_ranges": {"ranged": [-2.0, -1.0]}, + } + + inputs = generate_random_inputs(io_config) + + assert np.all(inputs["ranged"] >= -2.0) + assert np.all(inputs["ranged"] < -1.0) + assert set(np.unique(inputs["default_int"])) == {0, 1} + assert np.all(inputs["default_float"] >= 0.0) + assert np.all(inputs["default_float"] < 1.0) + + def test_bbox_range_generates_ordered_positive_area_boxes(self) -> None: + """Default perf inputs must preserve the canonical bbox corner contract.""" + import numpy as np + + from winml.modelkit.commands.perf import generate_random_inputs + + io_config = { + "input_names": ["bbox"], + "input_shapes": [[1, 512, 4]], + "input_types": ["int32"], + "input_value_ranges": {"bbox": [0, 1001]}, + } + + bbox = generate_random_inputs(io_config)["bbox"] + + assert bbox.shape == (1, 512, 4) + assert np.all(bbox >= 0) + assert np.all(bbox < 1001) + assert np.all(bbox[..., 0] < bbox[..., 2]) + assert np.all(bbox[..., 1] < bbox[..., 3]) + + def test_bbox_range_requires_two_coordinates(self) -> None: + """A degenerate bbox range must fail instead of producing invalid corners.""" + from winml.modelkit.commands.perf import generate_random_inputs + + io_config = { + "input_names": ["bbox"], + "input_shapes": [[1, 8, 4]], + "input_types": ["int32"], + "input_value_ranges": {"bbox": [0, 1]}, + } + + with pytest.raises(ValueError, match="at least two coordinates"): + generate_random_inputs(io_config) + + class TestEffectiveBatchSize: """Throughput must scale by the batch the session actually ran. diff --git a/tests/unit/export/test_onnx_config_overrides.py b/tests/unit/export/test_onnx_config_overrides.py index 29478512c..d91b893e0 100644 --- a/tests/unit/export/test_onnx_config_overrides.py +++ b/tests/unit/export/test_onnx_config_overrides.py @@ -23,6 +23,9 @@ from __future__ import annotations +import json +from pathlib import Path + import pytest from optimum.exporters.tasks import TasksManager @@ -33,6 +36,7 @@ _populate_image_size_from_preprocessor, ) from winml.modelkit.models.hf.roberta import _adjust_position_embeddings +from winml.modelkit.onnx.io import InputTensorSpec # ============================================================================= @@ -136,8 +140,33 @@ def test_bert_io_specs_shape_matches(self, bert_config) -> None: class TestLayoutLMQuestionAnsweringOverride: """LayoutLM QA export must include bbox and safe token_type_ids.""" - def test_layoutlm_qa_dummy_inputs_include_bbox_and_zero_token_types(self) -> None: - """Dummy inputs must keep bbox while forcing token_type_ids to zero.""" + @pytest.mark.parametrize("precision", ["fp32", "fp16"]) + def test_layoutlm_recipe_bbox_generates_valid_boxes(self, precision: str) -> None: + """Both explicit recipes must use the same meaningful bbox contract.""" + recipe_path = ( + Path(__file__).parents[3] + / "examples" + / "recipes" + / "impira_layoutlm-invoices" + / "cpu" + / "cpu" + / f"question-answering_{precision}_config.json" + ) + recipe = json.loads(recipe_path.read_text(encoding="utf-8")) + bbox_config = next( + tensor for tensor in recipe["export"]["input_tensors"] if tensor["name"] == "bbox" + ) + + assert bbox_config["value_range"] == [0, 1001] + bbox = InputTensorSpec.from_dict(bbox_config).to_tensor() + assert bbox.shape == (1, 512, 4) + assert bbox.min().item() >= 0 + assert bbox.max().item() < 1001 + assert (bbox[..., 0] < bbox[..., 2]).all().item() + assert (bbox[..., 1] < bbox[..., 3]).all().item() + + def test_layoutlm_qa_dummy_inputs_include_valid_boxes_and_zero_token_types(self) -> None: + """Dummy inputs must use meaningful boxes while forcing token types to zero.""" from transformers import LayoutLMConfig layoutlm_config = LayoutLMConfig( @@ -158,6 +187,91 @@ def test_layoutlm_qa_dummy_inputs_include_bbox_and_zero_token_types(self) -> Non assert inputs["bbox"].shape == (1, layoutlm_config.max_position_embeddings, 4) assert inputs["token_type_ids"].shape == (1, layoutlm_config.max_position_embeddings) assert inputs["token_type_ids"].max().item() == 0 + bbox = inputs["bbox"] + assert bbox.min().item() >= 0 + assert bbox.max().item() < 1001 + assert (bbox[..., 0] < bbox[..., 2]).all().item() + assert (bbox[..., 1] < bbox[..., 3]).all().item() + + specs = resolve_io_specs("layoutlm", "question-answering", layoutlm_config) + assert specs["value_ranges"]["bbox"] == (0, 1001) + + def test_layoutlm_qa_uses_usable_length_for_padding_offset(self) -> None: + """RoBERTa-style position offsets must not generate out-of-range positions.""" + from transformers import LayoutLMConfig + + layoutlm_config = LayoutLMConfig( + vocab_size=100, + hidden_size=64, + num_hidden_layers=2, + num_attention_heads=2, + intermediate_size=128, + max_position_embeddings=34, + max_2d_position_embeddings=1024, + type_vocab_size=1, + pad_token_id=1, + ) + + inputs = generate_dummy_inputs("layoutlm", "question-answering", layoutlm_config) + + assert inputs["input_ids"].shape == (1, 32) + assert inputs["bbox"].shape == (1, 32, 4) + assert inputs["attention_mask"].shape == (1, 32) + assert inputs["token_type_ids"].shape == (1, 32) + + specs = resolve_io_specs("layoutlm", "question-answering", layoutlm_config) + assert specs["value_ranges"]["token_type_ids"] == (0, 1) + + def test_layoutlm_qa_bbox_respects_smaller_2d_embedding_table(self) -> None: + """Coordinates and width/height indexes must fit the configured 2D table.""" + from transformers import LayoutLMConfig + + layoutlm_config = LayoutLMConfig( + vocab_size=100, + hidden_size=64, + num_hidden_layers=2, + num_attention_heads=2, + intermediate_size=128, + max_position_embeddings=32, + max_2d_position_embeddings=16, + type_vocab_size=1, + ) + + inputs = generate_dummy_inputs("layoutlm", "question-answering", layoutlm_config) + bbox = inputs["bbox"] + widths = bbox[..., 2] - bbox[..., 0] + heights = bbox[..., 3] - bbox[..., 1] + + assert bbox.shape == (1, layoutlm_config.max_position_embeddings, 4) + assert bbox.min().item() >= 0 + assert bbox.max().item() < layoutlm_config.max_2d_position_embeddings + assert ((widths > 0) & (widths < layoutlm_config.max_2d_position_embeddings)).all().item() + assert ((heights > 0) & (heights < layoutlm_config.max_2d_position_embeddings)).all().item() + + specs = resolve_io_specs("layoutlm", "question-answering", layoutlm_config) + assert specs["value_ranges"]["bbox"] == (0, 16) + + def test_layoutlm_qa_bbox_handles_minimum_valid_2d_table(self) -> None: + """The two-coordinate edge case must still produce positive-area boxes.""" + from transformers import LayoutLMConfig + + layoutlm_config = LayoutLMConfig( + vocab_size=100, + hidden_size=64, + num_hidden_layers=2, + num_attention_heads=2, + intermediate_size=128, + max_position_embeddings=8, + max_2d_position_embeddings=2, + type_vocab_size=1, + ) + + bbox = generate_dummy_inputs("layoutlm", "question-answering", layoutlm_config)["bbox"] + + assert bbox.shape == (1, 8, 4) + assert set(bbox.unique().tolist()) == {0, 1} + assert (bbox[..., 0] < bbox[..., 2]).all().item() + assert (bbox[..., 1] < bbox[..., 3]).all().item() def test_layoutlm_qa_io_specs_include_span_outputs(self) -> None: """LayoutLM QA specs expose document bbox input and span logits outputs.""" diff --git a/tests/unit/loader/test_hf_model_class_mapping.py b/tests/unit/loader/test_hf_model_class_mapping.py index ba10369eb..cb8177e84 100644 --- a/tests/unit/loader/test_hf_model_class_mapping.py +++ b/tests/unit/loader/test_hf_model_class_mapping.py @@ -70,6 +70,16 @@ def test_segformer_image_segmentation_returns_semantic_segmentation(self): ) assert result.__name__ == "AutoModelForSemanticSegmentation" + def test_layoutlm_question_answering_returns_concrete_qa_class(self): + """LayoutLM QA should use the concrete architecture registered for export.""" + from winml.modelkit.loader.resolution import _get_custom_model_class + + result = _get_custom_model_class( + model_type="layoutlm", + task="question-answering", + ) + assert result.__name__ == "LayoutLMForQuestionAnswering" + # ========================================================================= # Level 2: Task Defaults (tasks not in TasksManager) # ========================================================================= @@ -188,6 +198,12 @@ def test_segformer_class_mapping_registered(self): assert ("segformer", "image-segmentation") in HF_MODEL_CLASS_MAPPING + def test_layoutlm_class_mapping_registered(self): + """LayoutLM QA mapping should be aggregated from its architecture module.""" + from winml.modelkit.models import HF_MODEL_CLASS_MAPPING + + assert ("layoutlm", "question-answering") in HF_MODEL_CLASS_MAPPING + def test_segformer_module_class_mapping_structure(self): """Segformer module should export MODEL_CLASS_MAPPING dict.""" from winml.modelkit.models.hf.segformer import MODEL_CLASS_MAPPING diff --git a/tests/unit/loader/test_resolve_task.py b/tests/unit/loader/test_resolve_task.py index fffd20fe1..722b81620 100644 --- a/tests/unit/loader/test_resolve_task.py +++ b/tests/unit/loader/test_resolve_task.py @@ -35,6 +35,20 @@ def test_autodetect_text_classification_via_tasks_manager(): assert r.source == TaskSource.TASKS_MANAGER +def test_layoutlm_qa_uses_exact_architecture_mapping(): + r = resolve_task(_cfg("layoutlm", ["LayoutLMForQuestionAnswering"])) + assert r.task == "question-answering" + assert r.optimum_task == "question-answering" + assert r.model_class.__name__ == "LayoutLMForQuestionAnswering" + assert r.source == TaskSource.ARCHITECTURE_MAPPING + + +def test_layoutlm_non_qa_architecture_keeps_tasks_manager_routing(): + r = resolve_task(_cfg("layoutlm", ["LayoutLMForTokenClassification"])) + assert r.task == "token-classification" + assert r.source == TaskSource.TASKS_MANAGER + + def test_autodetect_modality_upgrade_for_vision_feature_extraction(): r = resolve_task(_cfg("vit", ["ViTModel"])) assert r.task == "image-feature-extraction"