diff --git a/docs/api/datamodel/metadata-formats.md b/docs/api/datamodel/metadata-formats.md new file mode 100644 index 00000000..bc23ce51 --- /dev/null +++ b/docs/api/datamodel/metadata-formats.md @@ -0,0 +1,17 @@ +# Metadata for format authors + +The public names of [`brainhops.datamodel.metadata`](metadata.md) are +those a user of the library needs, and the vocabulary groups that a +format names in `supports=`. The names below are for whoever adds +the metadata of a file format; the +[format author's guide](../../dev/metadata-formats.md) explains how they +fit together. They are imported from the private modules that define +them. + +# ::: brainhops.datamodel.metadata._vocabulary +# ::: brainhops.datamodel.metadata._operations +# ::: brainhops.datamodel.metadata._field +# ::: brainhops.datamodel.metadata._report +# ::: brainhops.datamodel.metadata._dtype +# ::: brainhops.datamodel.metadata._sentinel +# ::: brainhops.io.base._metadata_parser diff --git a/docs/api/datamodel/metadata.md b/docs/api/datamodel/metadata.md new file mode 100644 index 00000000..6f8b4f03 --- /dev/null +++ b/docs/api/datamodel/metadata.md @@ -0,0 +1 @@ +# ::: brainhops.datamodel.metadata diff --git a/docs/api/io/metadata/base.md b/docs/api/io/metadata/base.md new file mode 100644 index 00000000..fc9f4322 --- /dev/null +++ b/docs/api/io/metadata/base.md @@ -0,0 +1 @@ +# ::: brainhops.io.metadata._base diff --git a/docs/api/io/metadata/bids.md b/docs/api/io/metadata/bids.md new file mode 100644 index 00000000..1d548df3 --- /dev/null +++ b/docs/api/io/metadata/bids.md @@ -0,0 +1 @@ +# ::: brainhops.io.metadata.bids diff --git a/docs/api/io/metadata/index.md b/docs/api/io/metadata/index.md new file mode 100644 index 00000000..d4a95282 --- /dev/null +++ b/docs/api/io/metadata/index.md @@ -0,0 +1 @@ +# ::: brainhops.io.metadata diff --git a/docs/api/io/transformations/itk-h5.md b/docs/api/io/transformations/itk-h5.md index 9ca1d53f..644c0091 100644 --- a/docs/api/io/transformations/itk-h5.md +++ b/docs/api/io/transformations/itk-h5.md @@ -1 +1,3 @@ # ::: brainhops.io.transformations.itk.h5 + +# ::: brainhops.io.transformations.itk._metadata diff --git a/docs/design/format-metadata.md b/docs/design/format-metadata.md index 036a3461..3ae79e3c 100644 --- a/docs/design/format-metadata.md +++ b/docs/design/format-metadata.md @@ -990,30 +990,42 @@ to match on (`format`), and nothing is dispatched on a parsed name. Answer to issue question 4. Propagation is driven by the field `Scope` tags. The image class owns the operation: the metadata does not know the coordinate system, the shape or the transformation of the image, so the -image resolves what changed from its geometry, and hands the result to -the metadata. One method is public; two private hooks serve the image -operations. Each returns a new object of the same class. +image describes what it did as an operation object, and hands it to the +metadata, which returns a new object of the same class. ```python -def derive(self, *, history=None) -> tx.Self: - """Provenance only: same axes, same fields.""" +def derive(self, operation=None, *, history=None) -> tx.Self: + """Same axes without an operation; otherwise each value follows it.""" -def _select(self, axis_type, positions, *, history=None) -> tx.Self: - """The axes of one non-spatial type kept `positions` (or None).""" +class Operation: # what an image did to its axes + moves_space: bool # did the spatial axes change? + voxel_map: tx.Optional[np.ndarray] # old voxels -> new, linear part -def _reslice(self, linear, *, history=None) -> tx.Self: - """The spatial axes changed; `linear` maps old voxels to new.""" +class Indexed(Operation): # image[index] + index: tuple + shape: tuple + system: tx.Optional[CoordinateSystem] + + def positions(self, axis: AxisType) -> tx.Optional[np.ndarray]: ... + + +class Resampled(Operation): # image.reslice(geometry) + transformation: Transformation # new voxels -> old voxels + geometry: tx.Optional[Geometry] # None when not known ``` -`positions` is a resolved integer array (`None` when the axis was -dropped), and `linear` the linear part of the old-to-new voxel map, as an -ndarray (`None` when it is unknown or not affine): the metadata never -interprets a raw index or a `Transformation`. `history` is one entry (a -string) or several. The three share one pipeline, -`_derive_values(changed={AxisType: positions or linear}, history=)`, -which applies the rules below field by field: +`Indexed.positions(axis)` is `np.arange(size)[component]` for the first +axis of that type (`None` when the axis was dropped or the selection is +not 1-D), and `voxel_map` the linear part of the old-to-new voxel map +(`None` when it is unknown or not affine). `history` is one entry (a +string) or several. Each value goes through `propagate(value, +operation, name=, scope=, source=)`, which looks up a handler +registered with `@propagates(key, operation_type)`: first by the type +of the value (its MRO, and for each class the MRO of the operation), +then by the scope of its field; a value no handler takes is kept. The +scope defaults apply the rules below field by field: - `FILE`-scoped fields are kept, except `creation_time` (cleared) and `history`, to which the `history` entries (a short string such as @@ -1021,20 +1033,21 @@ which applies the rules below field by field: brainhops entry once. - `ACQUISITION`-scoped fields are kept. - `SPATIAL`-scoped fields (`slice_timing`, `slice_encoding_direction`, - `phase_encoding_direction`) are cleared by `_reslice`, with one + `phase_encoding_direction`) are cleared when `moves_space`, with one exception: an encoding direction in voxel axes is pushed through - `linear` (`v' = normalize(L @ v)`, section 4.2), so a + `voxel_map` (`v' = normalize(L @ v)`, section 4.2) by the handler of + `EncodingDirection`, so a permutation or flip remaps `"j-"` and an oblique resampling keeps an oblique direction. The map of a 4-D image has more axes than the direction, which lies in its first three; the direction is kept only when it still lies there. Without a map, or with a map that is not affine, the direction is cleared; a direction in a named world space does not move with the voxels. The slice timing is always cleared. -- `AXIS`-scoped fields (`channels`, `bvalues`, `bvectors`) keep, by - `_select`, their entries at the positions the image kept along the - axis type their `Along` names (the image resolves them with - `np.arange(size)[component]`, so a slice, integers or a boolean mask - all work). A dropped axis (an integer index) or a position beyond the +- `AXIS`-scoped fields (`channels`, `bvalues`, `bvectors`) keep, under + an `Indexed`, their entries at `positions` along the axis type their + `Along` names (`np.arange(size)[component]`, so a slice, integers or + a boolean mask all work); they are kept when the image has no such + axis, or when the index leaves it untouched. A dropped axis (an integer index) or a position beyond the field clears it. This is today's AFNI `_PER_BRICK` and NRRD `_PER_AXIS` rule, made generic. - `extra` is kept verbatim; nothing in it is understood. @@ -1046,13 +1059,13 @@ which applies the rules below field by field: resampling, the keyval `SliceTiming` is removed. Record content that is tied to some axes but outside the vocabulary (NIfTI `slice_start/end`, AFNI `TAXIS_OFFSETS` and the `_PER_GRID`/ - `_PER_BRICK` attributes, NRRD `_PER_AXIS` fields) is scrubbed by a - per-format hook, `_derive_raw(raw, *, changed)`, which receives the - same mapping and only tests which axis types are in it. Without both, - case 1 would write stale slice timing from an untouched record. The - hook's default is a deep copy of the record (a derived object never - shares it, as M10 copies rather than aliases), so a format overrides - it only to scrub. + `_PER_BRICK` attributes, NRRD `_PER_AXIS` fields) is scrubbed by the + handler of the class of the record (`propagate_raw`), which the + format registers next to its metadata class, and which only reads + the operation. Without both, case 1 would write stale slice timing + from an untouched record. Without a handler, the record is deep + copied (a derived object never shares it, as M10 copies rather than + aliases), so a format registers one only to scrub. Where it is called: @@ -1063,10 +1076,10 @@ Where it is called: per file). Each level is a `SingleScaleImage` and so has a `metadata` field too; it holds a *derived copy*, built when the level is materialised, never the parent object by identity: the first level - gets `derive()`, and level `i` gets `_reslice` with the linear part of - `level_i.transformation.inverse() @ level_0.transformation`, the map - from the voxels of the first level to its own, so a direction in voxel - axes survives the coarser sampling. Sharing by identity would let `levels[2].metadata.description + gets `derive()`, and level `i` gets `derive(Resampled(...))` with + `level_0.transformation.inverse() @ level_i.transformation`, the map + from the voxels of level `i` to those of the first level, so a + direction in voxel axes survives the coarser sampling. Sharing by identity would let `levels[2].metadata.description = ...` silently edit the pyramid, and would make a level's `slice_timing` wrong. The price is that editing a level's metadata does not reach the pyramid, which is the right direction: the pyramid @@ -1087,19 +1100,19 @@ Where it is called: A call on the sequence itself is covered by `DataModelBase.from_other`, which passes the `metadata` of a data model it hands to a constructor (section 10). -- **Data-model operations.** `SingleScaleImage.__getitem__` expands - its index to one component per data axis: a component that changes an - axis typed `time`, `channel`, ... calls `_select(type, - np.arange(size)[component])` (`None` for an integer), and one that - changes a spatial axis (or an untyped one, or moves one to another - position) calls `_reslice` last, with the pseudo-inverse of the linear - part of the map of the index (`_index2transform`); the step is - recorded once, as `"getitem"`. `reslice` calls `_reslice` with the - linear part of `new2old.inverse()`, where `new2old` is the - voxel-to-voxel map without the grid (a field, which has no affine - form), and `history="reslice"`. `__call__` keeps the metadata as it - is. A `MultiScaleImage` reslices through one of its levels, so it - inherits the rule. +- **Data-model operations.** `SingleScaleImage.__getitem__` derives + with `Indexed(index, self.shape, grid.input)`, which expands its index + to one component per data axis and reads the type of each axis: an + axis typed `time`, `channel`, ... selects the entries of the fields + along it, and a change of a spatial axis (or of an untyped one, or a + move to another position) moves space, through the pseudo-inverse of + the linear part of the map of the index (`_index2transform`); the step + is recorded once, as `"getitem"`. `reslice` derives with + `Resampled(new2old, geometry)`, where `new2old` is the voxel-to-voxel + map without the grid (a field, which has no affine form), and + `history="reslice"`. `__call__` keeps the metadata as it is (a copy, + with no provenance). A `MultiScaleImage` reslices through one of its + levels, so it inherits the rule. ## 10. Datamodel field (M10) and the `metadata_fields` clash (M11) @@ -1510,8 +1523,8 @@ for a JSON-capable node. > `omero` key was not worth it); a `Channel.unit` and a non-opaque alpha > are reported as approximated. The writer keeps the multiscale `name`, > `type` and downsampling `metadata` of the record. Levels get -> `derive()` (the first) or `_reslice()` (the others, section 9) when -> the pyramid is read; a pyramid built in memory keeps the levels it was +> `derive()` (the first) or `derive(Resampled(...))` (the others, +> section 9) when the pyramid is read; a pyramid built in memory keeps the levels it was > given. For plain Zarr the > generic option was taken: `ZarrMetadata` (format `"zarr"`) stores the > vocabulary as a BIDS sidecar (the codec of `to_bids`) under the array @@ -1917,6 +1930,128 @@ holds. `ConversionReport.passed_through`, which only that hook filled, was dropped with it. +### Addendum: the seventh review + +Where this addendum and the sections above disagree, this addendum +holds. + +- **`repr` (M5).** `Metadata` has no `__repr__` of its own: `bagof` + builds it, with policies on the fields. `Metadata` and the groups + (`Vocabulary`) hide a field that holds `None` or `UNSUPPORTED` + (`repr=HideIf(...)`), and `extra` is hidden when empty too. The + fields are listed in reverse (`reverse=True`), with the groups listed + in the reverse of their order, so that `repr(Metadata(...))` shows + `format`, `extra`, then the vocabulary in its declared order. A + format class declares the vocabulary again (`supports=`), and + `bagof` has no way to place a field (`format`) before the fields a + class declares, so the `repr` of a format would show `format` last. + It hides it instead, since the name of the class says the format: + `FileBasedMetadata` declares `format` again with `HideIfDefault`, + which `bagof` binds again on each class against the value the class + pins, so `repr` gives `NiftiMetadata(description='T1w')`. Generic + `Metadata` keeps `HideIf(...)` on it, and shows its format. + +- **No input or output on `Metadata` (M8).** `Metadata.load`, + `Metadata.from_bids` and `Metadata.to_bids` are gone: the data model + reads and writes no file. The metadata of a file is read by the + dispatcher, `FileBasedMetadata.load(path, hint=...)` (or by the class + of a format, `NiftiMetadata.load(path)`), and a BIDS sidecar by the + functions `from_bids` and `to_bids` of `brainhops.io.metadata.bids`, + whose `BidsSidecar` is the reader `FileBasedMetadata.load` picks for + a `.json` file. + +- **Conversion helpers are functions.** No class overrides the + conversion, so `Metadata._convert_from` is the module function + `_convert_from(cls, other, args, kwargs)`, which `to`, + `from_instance` and `FileBasedMetadata.writable` call. The methods + `_accepts_raw` (on `Metadata` and `FileBasedMetadata`) and + `FileBasedMetadata._raw_type` are gone too: the class variable + `_raw_class` says it all, read by the function `_accepts_raw(cls, + raw)` (`_raw_class is None or isinstance(raw, _raw_class)`). + `Metadata` declares `None` (any record), and `FileBasedMetadata` + `type(None)` (none) until a format declares the type of its own + record, which keeps the behaviour of the two former methods. + +- **Encoding directions (M3).** The `space` of an `EncodingDirection` + may also be a brainhops `CoordinateSystem`, kept as it is; it equals + an equal system, never a label (not even its own name). A file names + the space with a string, so the JSON codec writes a system by its + name, which reads back as a label, and reports as lost a direction in + a system without a name (`encode_changes(obj, changed, *, report)`); + a sidecar already reports any direction in a space as lost. + `EncodingDirection.transform` also takes a `Transformation`: the + linear part of the affine it reduces to, and a `TypeError` for one + that does not (a field). + +- **The parsers (M8).** `MetadataParser` only reads, as a `FileParser` + does: its `to_file`, which refused, is gone, and plain Zarr defines + its own. Its `from_bytes` override is gone too: since #311 (issue + #297), `FileParser.from_bytes` hands the bytes to `from_fileobj` in a + stream when a class implements `from_fileobj`, as every format + does. `Hdf5MetadataParser` + lives next to `Hdf5Parser`, in `brainhops.io.base.hdf5`, and is an + `Hdf5Parser`, whose routing of paths, streams and bytes to `sniff_h5` + and `from_h5` it uses rather than its own. `ItkH5Metadata` (and + `read_h5_header`) moved next to the `.h5` parser, + `brainhops.io.transformations.itk.h5`, since it needs `h5py`; the + package `itk` exports it when `h5py` is installed. + +- **Imports at the top.** The modules of the metadata import at the + top, except where a cycle forbids it, which a comment says at each + import: `_nifti_metadata` and `_mgh_metadata` import from `nifti` and + `mgh`, which import them; `EncodingDirection.transform` imports the + transformations, whose `metadata` field imports the metadata; + and `_with_brainhops` reads `brainhops.__version__`, defined after + the package imports the data model. The Zarr attribute helpers + (`node_attributes`, `write_attributes`) moved from `_image` to + `_metadata`, which the image imports. + +- **Layout of the modules.** Each module of the metadata lists its + public classes and main functions first, then its public helpers, + then its private helpers. A private definition that must exist + before a public one at import time (a converter that a field + annotation evaluates, a base class, a default or a `TypeVar`) stays + above it, with a comment that says why. + +### Addendum: operation objects + +Where this addendum and the sections above disagree, this addendum +holds. + +- **One entry point (M9).** The private hooks `_select(axis_type, + positions)` and `_reslice(linear)`, and the `changed=` mapping of + `_derive_values`, are gone. An image operation describes what it did + as an `Operation` (`Indexed`, `Resampled`, exported by + `brainhops.datamodel.metadata`), and derives with + `derive(operation, history=...)`. `derive()` without an operation is + unchanged, and so is the automatic provenance. +- **Dispatch.** One registry, in `_operations`: + `@propagates(key, operation_type)` registers a handler + `handler(value, operation, *, name, source)`, where `key` is a value + class or a `Scope` member, never `object`. `propagate` looks up the + type of the value (its MRO, the most specific operation first), then + the scope; a value no handler takes is kept, and `UNSUPPORTED` + always is. Plain tuples (`bvalues`, `display_range`) register + nothing, and go to the scope default, which reads `ALONG`. Each + handler lives next to its type: the encoding direction in `_terms`, + the scope defaults in `_vocabulary`, under `Scope`. +- **The raw record.** `FileBasedMetadata._derive_values` propagates the + record with `propagate_raw`, the same lookup by the type of the + record, and a deep copy without a handler. A format registers a + handler for the class of its own record, never for a widely used + type: NIfTI's `_derive_header` replaces the `_reslice` override. +- **Imports.** The transformations import the metadata, so + `_operations` imports neither the transformations nor the geometry at + the top: `Resampled` annotates them as strings, without conversion, + and `voxel_map` imports what it needs when it runs. +- **`Resampled.geometry` is optional.** A level of an OME-Zarr pyramid + is derived through `Resampled` when the pyramid is opened, before any + level is read, and its geometry needs its shape, which needs its + data; the level passes no geometry, and no handler reads it. +- **Not done.** `image(transform)` records no provenance; it is a plain + copy. A warp's local Jacobian is an extension point of `Operation`, + which no operation implements. + ## Open questions for the maintainer 1. **Where the field lives (M10).** On the datamodel roots (`Image`, diff --git a/docs/dev/metadata-formats.md b/docs/dev/metadata-formats.md new file mode 100644 index 00000000..465dba9b --- /dev/null +++ b/docs/dev/metadata-formats.md @@ -0,0 +1,349 @@ +# Writing the metadata of a format + +This page is for whoever adds a file format to brainhops, or the +metadata of one that has none yet. What the metadata *is*, and how a user +reads, edits and converts it, is in the user guide +(`docs/start/metadata.md`); why it is built this way is in the +design memo (`docs/design/format-metadata.md`). + +A format's metadata is one class, `Metadata`, a subclass of +[`FileBasedMetadata`][brainhops.io.metadata.FileBasedMetadata] +that lives next to the format's parser. It says which vocabulary fields +the format can store, and how to decode them from the format's own +**raw record** (a `nibabel` header, a dict of attributes, ...) and encode +them back. The framework does the rest: the read-time snapshot, change +detection, conversion between formats, loss reports, propagation +(`derive`, and the image operations), the `metadata` field of images +and transformations. + +`brainhops/io/base/_nifti_metadata.py` is the worked example. A format's +metadata module reads in one order: its docstring (what each field is +stored as, and what is lossy), its constants, the raw-record type when +there is one (`MghRaw`, `ZarrRaw`: it must exist before the class names +it as the type argument of its base), the metadata class (its public +methods: the read aliases, the parser's `from_*` and sniffers, a public +override such as `check_writable`; then the two hooks), its private +codec helpers (decode side, then encode side), and last the helpers its +image classes import. + +## Where the names live + +The package `brainhops.datamodel.metadata` exports what a user needs +(`Metadata`, `UNSUPPORTED`, `Scope`, the value classes, the report and +the loss policy), and the vocabulary groups that `supports=` names +(`ProvenanceVocabulary`, `MRIVocabulary`, `DiffusionVocabulary`, +`DisplayVocabulary`, `StorageVocabulary`, `MicroscopyVocabulary`, +`TransformVocabulary`, and their base `Vocabulary`). The base of a +format's metadata lives in `brainhops.io`, as `FileBasedImage` does, +since it derives from the dispatcher of the formats: `from +brainhops.io.metadata import FileBasedMetadata, OpaqueMetadata`. The +data model never imports it. A format author imports +the rest from the private modules that define it: + +| Module | Names | +|---|---| +| `brainhops.datamodel.metadata._vocabulary` | the annotations `Bids`, `Scoped` and `Along`, and the tables `VOCABULARY`, `GROUPS`, `BIDS_KEYS`, `SCOPES`, `ALONG` | +| `brainhops.datamodel.metadata._field` | `MetadataField` | +| `brainhops.datamodel.metadata._report` | `apply_loss_policy`, `collect_loss_reports`, `OnLoss`, `LossPolicy` | +| `brainhops.datamodel.metadata._dtype` | `preferred_dtype`, `preferred_storage`, `stored_values` | +| `brainhops.datamodel.metadata._sentinel` | `ALL`, `Maybe`, `Unsupported` | +| `brainhops.io.base._metadata_parser` | `MetadataParser` | +| `brainhops.io.base.hdf5` | `Hdf5MetadataParser` (needs `h5py`) | +| `brainhops.io.metadata._json` | the JSON codec of key/value stores | +| `brainhops.io.metadata._sync` | `sync_metadata` | + +## The class + +```python +@register_format +class MyMetadata( + MetadataParser, # reads the raw record of a file (see below) + FileBasedMetadata[MyRaw], # the type of the raw record + on={"format": "my"}, # polymorphic discriminant + supports=( + ProvenanceVocabulary, + "echo_time", + ), # everything else UNSUPPORTED +): + EXTENSIONS = (".my",) + HINTS = ("my",) + + @classmethod + def sniff_fileobj(cls, file, error=False, **kwargs) -> float: ... + + @classmethod + def from_fileobj(cls, file, **kwargs) -> "MyMetadata": + return cls.from_raw(read_my_record(file)) + + def to_file(self, file, **kwargs) -> None: ... # a record of its own + + @classmethod + def _decode_raw(cls, raw, *, image=None) -> dict: ... + + def _encode_raw(self, raw, changed, *, image=None, report) -> MyRaw: ... +``` + +`format` is the discriminant of the polymorphic root, and `on=` declares +it: `bagof` narrows the field to the literal `"my"`, with that default, +so the class does not declare `format` itself. `Metadata(format="my", +...)` builds a `MyMetadata`, and `"my"` is the name `metadata.to("my")` +takes. The one exception is a format that subclasses another format +(`ItkMetadata` under `OpaqueMetadata`): its field is narrowed to the +value of its parent, so it declares its own. + +The type argument of `FileBasedMetadata` declares the type of the raw +record, `raw`, which stays out of `repr` and `==`; the class does not +declare `raw` again (what the record holds goes in its docstring). A +new, empty record is that type called without arguments (`MyRaw()`), for +an object built in memory or converted from another format. Do not +derive a shared base of several formats from `FileBasedMetadata`: its +field would win by the MRO over the type argument of each format (the +Zarr formats share `_ZarrMetadataParser`, a `MetadataParser` only). The +type matters beyond documentation. `raw` and the read-time snapshot are +fields of `Metadata`, so generic metadata carries the record of the +metadata it was converted from, and a conversion gives the record back +to a format only when the record is an instance of the type that the +format declares (`_raw_class`, which `FileBasedMetadata` sets to +`type(None)` until a format declares its own). Every format must +therefore declare a type of its own: wrap a plain `dict` or `tuple` in a +small class (`ZarrRaw`, `X5Raw`), never share a type with another +format, and do not declare a subclass of the type of another format. +`tests/test_io_metadata_matrix.py` checks it. A format with no record +declares `FileBasedMetadata[None]`. A read alias under the familiar name +of the record (`header`, `tags`, `node`) is a plain property over `raw`. + +`FileBasedMetadata` is the dispatcher of the formats whose files hold +metadata: `FileBasedMetadata.load(path, hint=...)` picks among the +classes registered into it with `@register_format`, by their +`EXTENSIONS`, `HINTS` and sniffers, as `brainhops.io.load` picks an +image parser. It is not a `FileBasedObject`, so `brainhops.io.load` +never returns metadata. The parser of a format, a `MetadataParser`, +comes first among its bases, and owns no registry. It is a `FileParser`: +the format implements `from_fileobj(file)`, which reads the raw record +of an open binary file, and nothing else (a NIfTI header, never the +voxels), then builds the metadata with `from_raw`; `from_filename` opens +a path in binary mode and hands it over, and `FileParser.from_bytes` +wraps the bytes in a stream, since the class implements `from_fileobj`. A format that reads a path otherwise overrides +`from_filename` too (MGH, whose tags are read lazily from a path). A +format stored in HDF5 derives from `Hdf5MetadataParser` instead (next to +`Hdf5Parser`, in `brainhops.io.base.hdf5`), and implements +`sniff_h5(h5file, error=False)` and `from_h5(h5file, **kwargs)`, as an +`Hdf5Parser` format does; a Zarr format implements `sniff_node(node)` +and `from_node(node)`, as `ZarrImage` does. A format whose record is an +object of its own on disk (the attributes of a Zarr array) defines +`to_file(file)` to write `to_raw()` there; no other format has a +`to_file`, since its record is written along with the data. The reader +of the image or transformation shares the code that reads the record +with the parser (`_load_nifti_header`, `read_mgh_raw`, +`read_h5_header`). A format whose files hold no metadata (FLIRT, ITK +`.tfm`) is not a `MetadataParser`, and overrides `load` to refuse +(`OpaqueMetadata.load`, from which `ItkMetadata` inherits, and +`FlirtMetadata.load`). + +## Class keyword + +- `supports=`: the vocabulary fields (and `"extra"`) the format can + store, as names, group classes (all the fields of the group) or + [`ALL`][brainhops.datamodel.metadata._sentinel.ALL]. Omitted, a subclass keeps + its parent's capabilities. Every other field defaults to + [`UNSUPPORTED`][brainhops.datamodel.metadata.UNSUPPORTED] and is + refused at construction. `supported_fields` lists what the class + stores, `unsupported_fields` the rest. A new vocabulary field is + unsupported by every format until one opts in. + +## Hooks + +A format implements two hooks, both private and both optional: + +| Hook | Called by | Gives | +|---|---|---| +| `_decode_raw(raw, *, image)` | `from_raw` | the vocabulary values of a raw record | +| `_encode_raw(raw, changed, *, image, report)` | `update_raw` | the raw record to write | + +- `_decode_raw(raw, *, image=None) -> dict`: raw record to vocabulary, + on read. Returns field names (and `"extra"`) to values; `None` values + may be left out. `image` is the image or transformation the raw record + belongs to, for fields that need it. A value for a field the class + does not support is a bug of the format class, and `from_raw` raises + `TypeError` rather than drop it unreported. +- `_encode_raw(raw, changed, *, image=None, report) -> raw`: vocabulary + to raw record, on write. `raw` is the record to write over (already a + copy, or the writer's own fresh record) and `changed` holds only the + fields that differ from the read-time snapshot, so an untouched field + keeps the record's value. A `None` in `changed` *clears* the slot. + `changed["extra"]` is a per-key diff whose `None` values remove a key. + Value-dependent loss goes in `report` (`report.lost[name] = value`, + `report.approximated[name] = reason`). Returns the raw record to write. + +**Fields the data model owns.** Some formats store, in the same slot, a +vocabulary field and a piece of the geometry of the data model: NIfTI +stores the repetition time as the time step of the image (`pixdim[4]`), +and a Zarr array is stored with the data type of its values. The writer +stores what the data model says, whatever the metadata says, so +`_encode_raw` does not write such a field when the data model has a +value for it (NIfTI writes `repetition_time` only for an image without a +time step; the Zarr formats never write `data_type`). The writer then +calls the public +[`check_raw`][brainhops.io.metadata.FileBasedMetadata.check_raw] +on its finished record: it decodes the record as a reader would, and +reports as approximated each changed field whose value the record does +not hold. So a field the data model owns needs no declaration. + +**Public overrides.** What else a format needs, it does by overriding a +public method and calling `super()`: + +- [`check_writable(*, image=None, raw=None)`][brainhops.io.metadata.FileBasedMetadata.check_writable] + runs `update_raw` and `check_raw` on a scratch record, a copy of `raw` + by default. A format whose writer starts from another record builds it + and passes it on (NIfTI: the record, with the shape and the time step + of `image`), so that a value-dependent check reads the same state as a + real write. + +**Image operations.** An image operation that changes axes describes +what it did as an +[`Operation`][brainhops.datamodel.metadata.Operation] (`image[index]` +gives an [`Indexed`][brainhops.datamodel.metadata.Indexed], +`image.reslice(...)` a +[`Resampled`][brainhops.datamodel.metadata.Resampled]), and derives the +metadata of its result with +[`derive(operation)`][brainhops.datamodel.metadata.Metadata.derive]. +Each field propagates by the handler of its value, or else by that of +its scope, and the raw record of a format by the handler of the class +of the record, or else as a deep copy. A format whose record holds +content tied to some axes, outside the vocabulary, registers a handler +for the class of its record, which is its own (never a widely used type +such as `dict`), with `propagates` from +`brainhops.datamodel.metadata._operations`. The handler returns the +record of the derived metadata, a copy it may edit; NIfTI's clears its +slice slots and `dim_info` when the spatial axes move: + +```python +@propagates(nb.Nifti1Header, Operation) +def _derive_header(raw, operation, *, name, source): + raw = copy.deepcopy(raw) + if operation.moves_space: + _clear_slices(raw) + raw.set_dim_info(None, None, None) + return raw +``` + +The lookup order (the type of the value before its scope, the most +specific class of the operation first) is documented in that module. + +## Scopes and axes + +Every vocabulary field declares how it propagates to a derived image, +with an annotation: `Scoped(Scope.FILE)`, `Scoped(Scope.ACQUISITION)`, +`Scoped(Scope.SPATIAL)`, or, for a field with one entry per index along +a non-spatial axis, `Along(AxisType.time)` or `Along(AxisType.channel)` +(the `AXIS` scope). These live in +`brainhops.datamodel.metadata._vocabulary`, with the groups. + +A writer of an arbitrary image maps the axes of its coordinate system to +the slots of the format by type, not by name: NIfTI stores the time axis +as its fourth dimension and the channel axis as its fifth, and OME-Zarr +names them `t` and `c`. A per-axis field goes with the axis its `Along` +names. When the image has no axis of that type, or when the length of +the field differs from the length of the axis, `_encode_raw` reports the +field as lost rather than writing entries that describe nothing. + +## Reading and writing + +**One word for the raw record.** Every name says "raw". A reader builds +the metadata with +[`from_raw`][brainhops.io.metadata.FileBasedMetadata.from_raw], +which decodes the raw record and keeps the read-time snapshot. A parser +given a raw record and a `metadata` that is not that record's (explicit, +or carried by `replace()`) uses +[`update_from_raw`][brainhops.io.metadata.FileBasedMetadata.update_from_raw], +which decodes the new raw record and keeps the changes. A parser does +both in one call from its `__post_init__`, +`sync_metadata(self, MyMetadata, raw, image=self)` +(`brainhops/io/metadata/_sync.py`): `same=` tells whether the metadata +holds the parser's record already (default: by identity), `raw` may be +a function that reads the record, called only when needed (Zarr, MGH), +and `force=True` decodes it afresh. + +A writer starts from +`metadata, report = MyMetadata.writable(obj.metadata)` +([`writable`][brainhops.io.metadata.FileBasedMetadata.writable]: +the metadata converted to its class when it is of another, and a report +that holds the conversion's losses, with no policy applied yet), builds +its raw record, calls +[`update_raw`][brainhops.io.metadata.FileBasedMetadata.update_raw] +with it and that report, as `on_loss=report` (a report given as +`on_loss` is filled, never warned about; `force=` names a writer keyword +that must win over the record, such as MGH `tr=`), sets what it takes +from the data model, calls +[`check_raw`][brainhops.io.metadata.FileBasedMetadata.check_raw] +on the finished record with the same report, and hands the report to +[`apply_loss_policy`][brainhops.datamodel.metadata._report.apply_loss_policy]: +one write, one report, one warning. +`io.save` collects the reports of a conversion and of the write that +follows ([`collect_loss_reports`][brainhops.datamodel.metadata._report.collect_loss_reports]) +and warns once, with +[`ConversionReport.merged`][brainhops.datamodel.metadata.ConversionReport.merged]. + +**JSON and key/value stores.** A format whose store is a JSON object or +a set of key/value pairs (x5 node `Metadata`, Zarr attributes, and the +MRtrix and NRRD headers to come) does not write its own codec: +`brainhops/io/metadata/_json.py` holds the one BIDS sidecars use (the +sidecars themselves are read and written by +`brainhops.io.metadata.bids`: `from_bids`, `to_bids`, and the +`BidsSidecar` reader that `FileBasedMetadata.load` picks for a `.json` +file; the data model does no input or output). `decode_object(obj, +names)` splits an object into the values of the vocabulary fields +`names` (read from their sidecar keys, BIDS keys or `CamelCase` names) +and the other keys, which are `extra`; `encode_changes(obj, changed, +report=)` writes the changed fields back under their keys (`None` +removes one, and a value JSON cannot hold is reported as lost), and +`encode_extra(obj, diff, report=, reserved=)` applies the `extra` diff, +reporting the keys the format keeps for itself as lost. `X5Metadata` is +the shortest example. + +## The `metadata` field of a format class + +`Image` and `Transformation` declare `metadata: Optional[Metadata]` +(keyword-only, out of `repr` and `==`). A format narrows it to its own +class, with a default factory, which is what makes a change of format +convert (and report). Both are written with +[`MetadataField`][brainhops.datamodel.metadata._field.MetadataField]`[hint, +*annotations]`, whose converter also converts on a class that does not +convert its fields (a plain `Magic` parser), converts a metadata object +of another class into the field's class (a `NiftiMetadata` given to a +field typed `Metadata` becomes generic `Metadata`), and *copies* a +metadata object that already is of the field's class +([`copy`][brainhops.datamodel.metadata.Metadata.copy]: the raw +record is shared, the snapshot and `extra` are not): two objects never +hold the same metadata, so editing the result of `replace()` or +`from_other` never edits the original. + +!!! warning "The first base wins" + `bagof` takes a field from the *first* base that has it, so the + narrowed declaration must sit on the class itself or on its first + base: a transformation format whose first base is a data model + transformation must declare it again (see + `NiftiBasedTransformation`). + +`DataModelBase.from_other` passes the `metadata` of a data model it hands +to a constructor (an x5 chain made into a NIfTI field), so the field +converts it and reports the loss. + +## Checklist + +1. Write `Metadata` next to the parser: `on=`, `supports=`, + `FileBasedMetadata[]`, then `_decode_raw` and + `_encode_raw`. +2. When the file holds metadata, put a `MetadataParser` first among its + bases (`@register_format`, `EXTENSIONS`, `HINTS`, a sniffer and + `from_fileobj`), so that `FileBasedMetadata.load` reads it; + otherwise, override `load` to refuse. +3. In the parser's `__post_init__`, read the metadata from the raw + record (`sync_metadata`); in the writer, encode it (`update_raw`), + check the finished record (`check_raw`) and apply the loss policy. +4. Narrow the `metadata` field of the image or transformation class + with `MetadataField`. +5. Add the class to the matrix test, `tests/test_io_metadata_matrix.py`: + a format class that is not in its table fails it. +6. Describe what the format stores in the "Formats" section of the + user guide. diff --git a/pyproject.toml b/pyproject.toml index 6ec7fef3..fa3fb99d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ dependencies = [ # typed inverses declare; 0.1 raises on it. 0.3 registers a subclass # with every polymorphic level above it and conjoins the constraints # it inherits, which the axes and coordinate systems rely on. - "bagof-magic >= 0.3.dev0", + "bagof-magic >= 0.3.dev2", "bagof-paths >= 0.2", # Units are parsed by pint, imported on first use. 0.21.1 is the last # release for Python 3.8; 0.24 is the first to support numpy 2. diff --git a/src/brainhops/datamodel/__init__.py b/src/brainhops/datamodel/__init__.py index 5ae2bafe..119ec2c3 100644 --- a/src/brainhops/datamodel/__init__.py +++ b/src/brainhops/datamodel/__init__.py @@ -26,6 +26,7 @@ "enums", "kinds", "images", + "metadata", "orientation", "systems", "transformations", @@ -39,6 +40,7 @@ enums, images, kinds, + metadata, orientation, systems, transformations, diff --git a/src/brainhops/datamodel/metadata/__init__.py b/src/brainhops/datamodel/metadata/__init__.py new file mode 100644 index 00000000..7e5c153b --- /dev/null +++ b/src/brainhops/datamodel/metadata/__init__.py @@ -0,0 +1,100 @@ +""" +Non-spatial metadata, shared across file formats. + +Every file format keeps descriptive metadata, such as a description, a +repetition time, a slice timing or a provenance, under its own names +and types. This package gives that metadata one representation, in a +class hierarchy that mirrors the hierarchy of the images (`Image`, +`FileBasedImage`, `NiftiImage`): + +- [`Metadata`][brainhops.datamodel.metadata.Metadata] holds the common + vocabulary, one field per concept, named after its BIDS key in snake + case and stored in BIDS units, plus `extra`, a free-form store of + string keys. In-memory images and transformations carry it, and + formats convert through it. +- [`FileBasedMetadata`][brainhops.io.metadata.FileBasedMetadata], which + lives in `brainhops.io.metadata` as `FileBasedImage` lives in + `brainhops.io.images`, is the base of the metadata of a file format, + which reads its fields from the raw record of the format (a `nibabel` + header, the attributes of a Zarr array, ...) and writes them back. +- Each format has its own `Metadata` class, next to its parser + under `brainhops.io`. `FileBasedMetadata.load(path)` reads the + metadata of a file without its data, and `brainhops.io.metadata.bids` + reads and writes BIDS sidecars. The data model does no input or + output. + +A vocabulary field holds a value, `None` when the value is unknown, or +[`UNSUPPORTED`][brainhops.datamodel.metadata.UNSUPPORTED] when the format +has no slot for the field. Converting into a format, with +`metadata.to(NiftiMetadata)`, records what the format cannot hold in a +[`ConversionReport`][brainhops.datamodel.metadata.ConversionReport], and +the loss policy (`"ignore"`, `"warn"` or `"raise"`, see +[`metadata_loss_policy`][brainhops.datamodel.metadata.metadata_loss_policy]) +decides what happens to the report. A +[`Scope`][brainhops.datamodel.metadata.Scope] says how each field +propagates to a derived image, through the +[`Operation`][brainhops.datamodel.metadata.Operation] an image operation +describes (an [`Indexed`][brainhops.datamodel.metadata.Indexed] image or +a [`Resampled`][brainhops.datamodel.metadata.Resampled] one). + +The names exported here are those a user of the library needs, and the +vocabulary groups (`ProvenanceVocabulary`, `MRIVocabulary`, ..., and +their base `Vocabulary`), which a format names in its `supports=` +declaration. What else a format author needs (the field annotations, +the `metadata` field of images, the loss helpers) is imported from the +private modules of this package, which the format author's guide lists +(`docs/dev/metadata-formats.md`). The user guide is +`docs/start/metadata.md`. +""" + +__all__ = [ + "Metadata", + "UNSUPPORTED", + "Scope", + "Operation", + "Indexed", + "Resampled", + "GeneratedBy", + "Channel", + "EncodingDirection", + "ConversionReport", + "MetadataLossWarning", + "MetadataLossError", + "metadata_loss_policy", + "Vocabulary", + "ProvenanceVocabulary", + "MRIVocabulary", + "DiffusionVocabulary", + "DisplayVocabulary", + "StorageVocabulary", + "MicroscopyVocabulary", + "TransformVocabulary", +] + +from ._base import Metadata +from ._operations import Indexed, Operation, Resampled +from ._report import ( + ConversionReport, + MetadataLossError, + MetadataLossWarning, + metadata_loss_policy, +) +from ._sentinel import UNSUPPORTED +from ._terms import Channel, EncodingDirection, GeneratedBy +from ._vocabulary import ( + DiffusionVocabulary, + DisplayVocabulary, + MicroscopyVocabulary, + MRIVocabulary, + ProvenanceVocabulary, + Scope, + StorageVocabulary, + TransformVocabulary, + Vocabulary, +) + +# The public names keep the `__module__` of the private module that +# defines them: rewriting it to this package's name would break +# `inspect.getsource`, IPython's `??` and doctest discovery, which look +# the source up through `__module__`. Pickles name the private module, +# and load as well. diff --git a/src/brainhops/datamodel/metadata/_base.py b/src/brainhops/datamodel/metadata/_base.py new file mode 100644 index 00000000..d29e87cc --- /dev/null +++ b/src/brainhops/datamodel/metadata/_base.py @@ -0,0 +1,659 @@ +"""`Metadata`: the common vocabulary, the hub through which formats convert.""" + +__all__ = ["Metadata"] + +# stdlib +import copy +import operator + +# externals +import typing_extensions as tx +from bagof.magic import Factory, HideIf, NoEq, NoRepr, fields + +# internals +from ..base import DataModelBase +from ._meta import MetadataMeta +from ._operations import Operation, propagate +from ._report import ( + ConversionReport, + OnLoss, + apply_loss_policy, +) +from ._sentinel import UNSUPPORTED, Maybe +from ._terms import GeneratedBy, _is_absent +from ._vocabulary import ( + FIELDS, + SCOPES, + VOCABULARY, + DiffusionVocabulary, + DisplayVocabulary, + MicroscopyVocabulary, + MRIVocabulary, + ProvenanceVocabulary, + Scope, + Scoped, + StorageVocabulary, + TransformVocabulary, +) + +# The `history=` argument of `derive`: one entry, several, or none. +# Above `Metadata`, whose signatures evaluate it. +_History = tx.Union[str, tx.Sequence[str], None] + + +# The groups are listed in the reverse of their order because the fields +# are (`reverse=True`: the fields of a class before those it inherits), +# so that `repr` shows `format`, `extra`, then the vocabulary in its +# declared order. The class of a format hides `format`, which its name +# says (see `FileBasedMetadata.format`). +class Metadata( + DataModelBase, + TransformVocabulary, + MicroscopyVocabulary, + StorageVocabulary, + DisplayVocabulary, + DiffusionVocabulary, + MRIVocabulary, + ProvenanceVocabulary, + metaclass=MetadataMeta, + polymorphic=True, + kw_only=True, + reverse=True, + # `UNSUPPORTED` and `None` are hidden from `repr`, or a format that + # stores three fields would print forty (see also `extra`). + repr=HideIf(_is_absent), +): + """ + Metadata that does not depend on a file format: the common + vocabulary, and the free-form store `extra`. + + In-memory images and transformations carry this class of metadata, + and every conversion between two formats goes through it, so that a + conversion from NIfTI to MGH loses exactly what the two conversions + from NIfTI to `Metadata` and from `Metadata` to MGH lose. The BIDS + sidecar codec (`brainhops.io.metadata.bids`) reads and writes it + too. `Metadata` supports every field of the vocabulary, and does no + input or output: the metadata of a file is read by + [`FileBasedMetadata.load`][brainhops.io.metadata.FileBasedMetadata.load]. + + `Metadata` is also the root of the metadata classes, and selects the + subclass from the `format` field: once `brainhops.io` is imported, + `Metadata(format="nifti", ...)` builds a `NiftiMetadata`, and an + unknown format builds a plain `Metadata`. + + The vocabulary is declared by seven groups, which `Metadata` inherits: + [`ProvenanceVocabulary`][], [`MRIVocabulary`][], [`DiffusionVocabulary`][], + [`DisplayVocabulary`][], [`StorageVocabulary`][], + [`MicroscopyVocabulary`][] and + [`TransformVocabulary`][]. Each field holds a value, `None` when the + value is unknown, or `UNSUPPORTED` when a format has no slot for the + field. The hooks that a format implements are described in the format + author's guide (`docs/dev/metadata-formats.md`). + """ + + # --- class attributes --------------------------------------------- + + supported_fields: tx.Annotated[ + tx.ClassVar[tx.FrozenSet[str]], + tx.Doc( + "The vocabulary fields (and `'extra'`) this class can store, " + "from its `supports=` declaration." + ), + ] = frozenset(("extra",) + VOCABULARY) + + unsupported_fields: tx.Annotated[ + tx.ClassVar[tx.FrozenSet[str]], + tx.Doc( + "The vocabulary fields this class cannot store: the " + "complement of `supported_fields`." + ), + ] = frozenset() + + # The type of raw record that the class declares, as the type argument + # of a generic base (`FileBasedMetadata[nb.Nifti1Header]`), set by the + # metaclass: `type(None)` for `[None]`. `None` on `Metadata`, which + # keeps any record (see `raw`, and `_accepts_raw` below). + _raw_class: tx.ClassVar[tx.Optional[type]] = None + + # --- format and extras -------------------------------------------- + + format: tx.Annotated[ + str, + tx.Doc("The format this metadata belongs to; selects the subclass."), + ] = "generic" + + extra: tx.Annotated[ + Maybe[tx.Dict[str, tx.Any]], + tx.Doc( + "Free-form keys the vocabulary does not cover, copied into " + "any free-form store a format has." + ), + Scoped(Scope.FILE), + # Not `Factory()`: inferred from `Maybe[...]`, a union with + # `None`, the default would be `None`. + Factory(dict), + # Hidden from `repr` when empty, as well as when absent. + HideIf(operator.not_), + ] + + # --- the raw record ----------------------------------------------- + + raw: tx.Annotated[ + tx.Any, + tx.Doc( + """ + The raw record of the file the metadata was read from (a + `nibabel` header, the attributes of a Zarr array, ...), or + `None`. Edit it only for what the vocabulary does not cover: + on write, a field left untouched keeps the value of the + record, and a field that was set wins over it. + + Generic metadata keeps the record of the metadata it was + converted from, so that converting back to the format of the + record keeps it: `NiftiMetadata -> Metadata -> NiftiMetadata` + round-trips, header extensions included. A conversion into + another format leaves the record behind, since every format + declares its own type of record. + """ + ), + NoRepr(), + NoEq(), + ] = None + + _snapshot: tx.Annotated[ + tx.Dict[str, tx.Any], + tx.Doc( + """ + The read-time snapshot: field name to the value that the + reader decoded from `raw`, and empty for metadata built in + memory. On write, a field is encoded over the raw record only + when it differs from its snapshot. The reader fills it; it is + never set by hand. + """ + ), + NoRepr(), + NoEq(), + Factory(), + ] + + # --- construction ------------------------------------------------- + + def __post_init__(self) -> None: + parent = getattr(super(), "__post_init__", None) + if parent is not None: + parent() + cls = type(self) + for name in cls.unsupported_fields: + value = getattr(self, name, None) + if value is None: + setattr(self, name, UNSUPPORTED) + elif value is not UNSUPPORTED: + raise ValueError( + f"{cls.__name__} cannot store {name!r} (it is " + f"UNSUPPORTED by this format), so {value!r} is " + f"refused." + ) + + def copy(self) -> tx.Self: + """ + Copy this metadata, so that the copy can be edited without editing + the original. + + An image or a transformation holds a copy when it is given + metadata that another object already holds (through `replace()`, + `from_other` or `metadata=`). + + The raw record is shared, as `replace()` shares it, while the + read-time snapshot and `extra` are copied, so that editing the + copy never edits the original. + + Returns + ------- + Metadata + The copy, of the same class. + """ + new = copy.copy(self) + new.extra = copy.copy(new.extra) + new._snapshot = copy.copy(self._snapshot) + return new + + # --- capabilities ------------------------------------------------- + + @classmethod + def supports(cls, name: str) -> bool: + """ + Whether this class can store a vocabulary field, as its + `supports=` declaration says. + + A format whose capabilities depend on the instance holds + `UNSUPPORTED` in a field that a particular instance cannot store, + so `meta.name is UNSUPPORTED` is the test for an instance. + + Parameters + ---------- + name : str + The name of a vocabulary field, or `"extra"`. + + Returns + ------- + bool + Whether the class can store the field. + + Raises + ------ + KeyError + If `name` is not a vocabulary field. + """ + if name not in FIELDS: + raise KeyError(f"{name!r} is not a vocabulary field.") + return name in cls.supported_fields + + # --- conversion --------------------------------------------------- + + def to( + self, + cls: tx.Union[None, str, tx.Type["Metadata"]] = None, + *, + on_loss: tx.Optional[OnLoss] = None, + **values: tx.Any, + ) -> "Metadata": + """ + Convert this metadata into another class, as images and + transformations convert with `to()`. + + Each value of the vocabulary is copied, except where the target + class cannot store the field. Such a value is lost, and reported. + + Parameters + ---------- + cls : type or str, optional + The `Metadata` subclass to convert to, or the name of its + format (`"generic"`, `"nifti"`, ...). By default, the class of + this object, which makes a copy. + on_loss : {"ignore", "warn", "raise"} or ConversionReport, optional + What to do when something is lost. By default, the policy in + effect (see [`metadata_loss_policy`][]). A + [`ConversionReport`][] is filled with what was lost or + approximated, and nothing is warned or raised. + **values + Fields to set on the result. + + Returns + ------- + Metadata + The converted metadata. + + Raises + ------ + MetadataLossError + If something is lost under the `"raise"` policy. + ValueError + If `cls` names no known format. + + Examples + -------- + ```python + report = ConversionReport() + nifti = meta.to(NiftiMetadata, on_loss=report) + if report.lossy: + ... + ``` + """ + target = type(self) if cls is None else _metadata_class(cls) + obj, found = _convert_from(target, self, (), values) + apply_loss_policy(found, on_loss, stacklevel=2) + return obj + + @classmethod + def from_instance(cls, other: tx.Any, *args, **kwargs) -> tx.Self: + """ + Convert the metadata of another format into this class. + + This is the conversion that field converters trigger. It behaves + as [`to`][brainhops.datamodel.metadata.Metadata.to] does, and hands + the report to the loss policy in effect (see + [`metadata_loss_policy`][]). An object that is not metadata is + handed to the data model. + + Parameters + ---------- + other : object + The object to convert. + *args, **kwargs + Constructor arguments. + + Returns + ------- + Metadata + The converted metadata. + + Raises + ------ + MetadataLossError + If something is lost under the `"raise"` policy. + """ + if not isinstance(other, Metadata): + return super().from_instance(other, *args, **kwargs) + obj, report = _convert_from(cls, other, args, kwargs) + apply_loss_policy(report, stacklevel=3) + return obj + + # --- propagation -------------------------------------------------- + + def derive( + self, + operation: tx.Optional[Operation] = None, + *, + history: _History = None, + ) -> tx.Self: + """ + Build the metadata of an image computed from the image this + metadata describes. + + Without an operation, the derived image lies on the same axes, + as a smoothed or a denoised version of it does, and every field + is kept. An image operation that changes axes describes what it + did as an [`Operation`][brainhops.datamodel.metadata.Operation]: + `image[index]` gives an + [`Indexed`][brainhops.datamodel.metadata.Indexed], and + `image.reslice(...)` a + [`Resampled`][brainhops.datamodel.metadata.Resampled]. Each + field then propagates through it, by the handler of its value + or of its scope (see `brainhops.datamodel.metadata._operations`): + an index along a time or a channel axis selects the entries of + the fields that run along it, and a change of the spatial axes + clears the fields tied to the spatial sampling, except an + encoding direction, which follows the map of the voxel axes. + + Either way, three provenance fields record the derivation: + `creation_time` is cleared, the entries of `history` are appended + to `history`, and `generated_by` gains a brainhops entry, once. + + Parameters + ---------- + operation : Operation, optional + What the image operation did to the axes of the image. + history : str or sequence of str, optional + A description of the derivation, appended to `history`: one + entry, or several. + + Returns + ------- + Metadata + New metadata, of the same class. Generic metadata derived this + way carries no raw record, since no format hook can scrub it. + The metadata of a file format keeps a copy of its record, + propagated through the operation by the handler of the + record's type (see `propagate_raw`), and a copy of its + read-time snapshot. + + Examples + -------- + ```python + meta = Metadata(description="T1w", history=("acquired",)) + meta.derive(history="smooth").history # ('acquired', 'smooth') + ``` + """ + values = self._derive_values(operation=operation, history=history) + return type(self)(**values) + + # --- internals ---------------------------------------------------- + + def _derive_values( + self, + *, + operation: tx.Optional[Operation], + history: _History, + ) -> tx.Dict[str, tx.Any]: + """ + The constructor values of `derive`, field by field: each value + propagated through `operation` (see `propagate`), and the + derivation recorded in the provenance fields. + """ + values: tx.Dict[str, tx.Any] = {} + for name in FIELDS: + value = getattr(self, name, None) + if value is UNSUPPORTED or name in self.unsupported_fields: + continue + if name == "extra": + value = dict(value or {}) + if operation is not None: + value = propagate( + value, + operation, + name=name, + scope=SCOPES.get(name, Scope.FILE), + source=self, + ) + values[name] = value + _derive_provenance(values, history) + return values + + +# ---------------------------------------------------------------------- +# PRIVATE +# ---------------------------------------------------------------------- + + +def _convert_from( + cls: tx.Type[Metadata], + other: Metadata, + args: tx.Tuple[tx.Any, ...] = (), + kwargs: tx.Optional[tx.Dict[str, tx.Any]] = None, +) -> tx.Tuple[Metadata, ConversionReport]: + """ + Convert metadata into a class, as `Metadata.from_instance` does, and + return the report instead of acting on it. + + It works across the metadata classes, which do not override it: each + value of the vocabulary is copied, except where `cls` cannot store + the field, and is then reported as lost. The raw record and its + snapshot go along when `cls` keeps the record (see `_accepts_raw`). + + Parameters + ---------- + cls : type + The metadata class to convert into. + other : Metadata + The metadata to convert. + args : tuple, optional + Positional arguments of the constructor. + kwargs : dict, optional + Fields to set on the result. + + Returns + ------- + metadata : Metadata + The converted metadata: of `cls`, or of the class of `other` when + `other` already stands for `cls` (a copy). + report : ConversionReport + What was lost. + """ + kwargs = dict(kwargs or {}) + same = _is_already(other, cls) + # A copy keeps the most specific class. + target = type(other) if same else cls + report = ConversionReport( + source=_format_name(other), target=_format_name(target) + ) + values: tx.Dict[str, tx.Any] = {} + if same or (other.raw is not None and _accepts_raw(target, other.raw)): + # The record and its snapshot go along, shared and copied as + # `copy()` does: to a copy, to the generic hub, or back to the + # format whose class declares the type of the record. + values["raw"] = other.raw + values["snapshot"] = copy.copy(other._snapshot) + unsupported = target.unsupported_fields + for name in FIELDS: + value = getattr(other, name, None) + if value is None or value is UNSUPPORTED: + continue + if name == "extra": + if not value: + continue + value = dict(value) + if name in unsupported: + report.lost[name] = value + continue + values[name] = value + values.update(kwargs) + return target(*args, **values), report + + +def _accepts_raw(cls: type, raw: tx.Any) -> bool: + """ + Whether a conversion into a metadata class keeps a raw record. + + A class keeps a record of the type it declares (`_raw_class`), and a + class that declares none (`None`: generic `Metadata`) keeps any + record. Formats declare distinct types, so that a record only ever + goes back to its own format; `FileBasedMetadata` declares + `type(None)` until a format declares its own type, so that a format + keeps no record of another one. + + Parameters + ---------- + cls : type + The metadata class converted into. + raw : object + The raw record of the source metadata, not `None`. + + Returns + ------- + bool + Whether the record is kept. + """ + declared = cls._raw_class + return declared is None or isinstance(raw, declared) + + +def _derive_provenance( + values: tx.Dict[str, tx.Any], history: _History +) -> None: + """ + Record a derivation in the provenance fields, in place. + + The derived object is a new object, so its `creation_time` is + cleared. The entries of `history` (one, when it is a string) are + appended to the `history` field, and brainhops is added to + `generated_by` unless it is there already. A field that the format + does not support is absent from `values`, and is left alone. + """ + if isinstance(history, str): + steps: tx.Tuple[str, ...] = (history,) + else: + steps = tuple(history or ()) + if "creation_time" in values: + values["creation_time"] = None + if "history" in values and steps: + values["history"] = tuple(values["history"] or ()) + steps + if "generated_by" in values: + values["generated_by"] = _with_brainhops(values["generated_by"]) + + +def _with_brainhops( + generated_by: tx.Optional[tx.Tuple[GeneratedBy, ...]], +) -> tx.Tuple[GeneratedBy, ...]: + entries = tuple(generated_by or ()) + if any(getattr(g, "name", None) == "brainhops" for g in entries): + return entries + try: + # Not at the top: `brainhops` imports the data model before it + # defines `__version__`. + from brainhops import __version__ as version + except ImportError: # pragma: no cover + version = None + return entries + (GeneratedBy(name="brainhops", version=version),) + + +def _is_already(value: tx.Any, cls: type) -> bool: + """ + Whether a metadata object can stand for `cls` as it is, without a + conversion. + + An object of `cls` itself can, and so can an object of a subclass of + a format class: a subclass of `NiftiMetadata` is still NIfTI + metadata. Generic `Metadata` is the exception, because its subclasses + are the formats: a `NiftiMetadata` given where generic metadata is + expected is converted, so that the result supports every field. + + Parameters + ---------- + value : object + The metadata object. + cls : type + The metadata class that is expected. + + Returns + ------- + bool + Whether `value` is used as it is. + """ + if type(value) is cls: + return True + return cls is not Metadata and isinstance(value, cls) + + +def _format_name(obj: tx.Any) -> str: + """ + The name of the format of a metadata class or object. + + For a class, the name is the default of its `format` field, which the + class pins with `on={"format": ...}` (`"nifti"` for `NiftiMetadata`, + `"generic"` for `Metadata`). A class whose `format` has no string + default is named after the class. For an object, the name is the + value of its `format` field. + + Parameters + ---------- + obj : type or object + A metadata class or object. + + Returns + ------- + str + The name of the format, as conversion reports print it. + """ + if isinstance(obj, type): + for field in fields(obj): + if field.name == "format": + default = field.default + if isinstance(default, str): + return default + return obj.__name__ + return str(getattr(obj, "format", type(obj).__name__)) + + +def _metadata_class(target: tx.Any) -> tx.Type[Metadata]: + """ + The class that a `to()` target names. + + Parameters + ---------- + target : type or str + A `Metadata` subclass, or the name of a format. A name is resolved + by the polymorphic constructor of `Metadata`, which selects the + subclass from `format`. + + Returns + ------- + type + The metadata class. + + Raises + ------ + ValueError + If `target` names no known format. + TypeError + If `target` is neither a class nor a string. + """ + if isinstance(target, type) and issubclass(target, Metadata): + return target + if isinstance(target, str): + cls = type(Metadata(format=target)) + if cls is Metadata and target != _format_name(Metadata): + raise ValueError(f"No metadata class for the format {target!r}.") + return cls + raise TypeError( + f"Expected a Metadata subclass or a format name, got {target!r}." + ) diff --git a/src/brainhops/datamodel/metadata/_dtype.py b/src/brainhops/datamodel/metadata/_dtype.py new file mode 100644 index 00000000..1bedfd70 --- /dev/null +++ b/src/brainhops/datamodel/metadata/_dtype.py @@ -0,0 +1,265 @@ +"""The element type, and the intensity scaling, a writer stores an +array with.""" + +__all__ = ["preferred_dtype", "preferred_storage", "stored_values"] + +# externals +import numpy as np +import typing_extensions as tx + +# internals +from ._report import ConversionReport, OnLoss, apply_loss_policy +from ._sentinel import UNSUPPORTED + + +def preferred_dtype( + metadata: tx.Any, + array_dtype: tx.Any, + dtype: tx.Any = None, + *, + on_loss: tx.Optional[OnLoss] = None, +) -> np.dtype: + """ + Choose the element type that a writer stores an array as. + + An explicit `dtype`, the writer option, always wins. Otherwise, the + `data_type` of the metadata is used when the values of the array are + of the same kind (integers, floating-point numbers or complex + numbers). The `data_type` is the type the file had when it was read, + or the type set since, so a label map read as `uint8` is written as + `uint8` again, while a resampled floating-point version of the same + map is not quantised. In every other case the array keeps its own + type. + + A `data_type` that was set or converted by hand, but cannot be used + because the values are of another kind, is reported as approximated. + A `data_type` that was only read is dropped silently, since the data + changed kind after the read. + + Parameters + ---------- + metadata : Metadata or None + The metadata of the object being written. + array_dtype : dtype-like + The element type of the array. + dtype : dtype-like, optional + The element type requested by the caller of the writer. + on_loss : {"ignore", "warn", "raise"} or ConversionReport, optional + What to do with an unused `data_type` that was set by hand. By + default, the policy in effect; a writer passes the report of its + write. + + Returns + ------- + numpy.dtype + The element type to store the array as. + + Raises + ------ + MetadataLossError + If an unused `data_type` is reported under the `"raise"` policy. + """ + array_dtype = np.dtype(array_dtype) + if dtype is not None: + return np.dtype(dtype) + wanted = getattr(metadata, "data_type", None) + if wanted is None or wanted is UNSUPPORTED: + return array_dtype + wanted = np.dtype(wanted) + if _same_kind(array_dtype, wanted): + return wanted + changed = _changed_fields(metadata, {"data_type": wanted}) + if "data_type" in changed: + report = ConversionReport(target=getattr(metadata, "format", None)) + report.approximated["data_type"] = ( + f"stored as {array_dtype.name}: the values are not " + f"{wanted.name} values" + ) + apply_loss_policy(report, on_loss, stacklevel=2) + return array_dtype + + +def preferred_storage( + metadata: tx.Any, + data: tx.Any, + dtype: tx.Any = None, + *, + on_loss: tx.Optional[OnLoss] = None, +) -> tx.Tuple[np.dtype, tx.Optional[float], tx.Optional[float]]: + """ + Choose how a writer that supports an intensity scaling stores an + array: its element type, and the scaling. + + The metadata describes the storage of the file that was read with three + fields: `data_type`, `scale_slope` and `scale_intercept`. Without a + scaling, the element type is chosen as [`preferred_dtype`][] chooses + it. With a scaling, the array is stored as `data_type` with that + scaling when `data_type` is an integer type and every value of the + array is, up to a thousandth of a step, `stored * scale_slope + + scale_intercept` for an integer `stored` that `data_type` can hold. A + scaled integer file that was read and left unchanged is therefore + written back exactly as it was. When the values do not fit the scaling + (after a resampling, for instance), the array is stored unscaled, and + the storage fields that were changed by hand are reported as + approximated. + + Parameters + ---------- + metadata : Metadata or None + The metadata of the image being written. + data : array-like + The values to write. They are read only when a scaling has to be + checked. + dtype : dtype-like, optional + The element type requested by the caller of the writer. It wins + over the metadata, and no scaling is applied. + on_loss : {"ignore", "warn", "raise"} or ConversionReport, optional + What to do with storage fields that were set by hand but cannot + be used. By default, the policy in effect; a writer passes the + report of its write. + + Returns + ------- + dtype : numpy.dtype + The element type to store the array as. + slope, intercept : float or None + The scaling to store the array with, or `None` for none. When a + scaling is returned, the writer stores `(value - intercept) / + slope` as `dtype`, rounded only when `dtype` is an integer type + (see [`stored_values`][]). A scaling is only returned with an + integer type, the one case where it makes sense. + + Raises + ------ + MetadataLossError + If an unused storage field is reported under the `"raise"` + policy. + """ + array_dtype = np.dtype(getattr(data, "dtype", np.float64)) + if dtype is not None: + return np.dtype(dtype), None, None + slope = _present(getattr(metadata, "scale_slope", None)) + intercept = _present(getattr(metadata, "scale_intercept", None)) + if slope in (None, 1.0) and intercept in (None, 0.0): + return ( + preferred_dtype(metadata, array_dtype, on_loss=on_loss), + None, + None, + ) + slope = 1.0 if slope is None else float(slope) + intercept = 0.0 if intercept is None else float(intercept) + wanted = _present(getattr(metadata, "data_type", None)) + if wanted is not None and _fits(data, np.dtype(wanted), slope, intercept): + return np.dtype(wanted), slope, intercept + report = ConversionReport(target=getattr(metadata, "format", None)) + stored = preferred_dtype(metadata, array_dtype, on_loss=report) + changed = _changed_fields( + metadata, {"scale_slope": slope, "scale_intercept": intercept} + ) + for name in ("scale_slope", "scale_intercept"): + if name in changed and changed[name] is not None: + report.approximated[name] = ( + f"stored unscaled as {stored.name}: the values do not fit " + f"the scaling" + ) + apply_loss_policy(report, on_loss, stacklevel=2) + return stored, None, None + + +def stored_values( + data: tx.Any, + dtype: tx.Any, + slope: tx.Optional[float] = None, + intercept: tx.Optional[float] = None, +) -> np.ndarray: + """ + The values a writer stores, with the type and the scaling that + [`preferred_storage`][] chose. + + Each value is stored as `(value - intercept) / slope`, rounded to the + nearest integer only when `dtype` is an integer type, then cast to + `dtype`. + + Parameters + ---------- + data : array-like + The values to write. + dtype : dtype-like + The element type to store them as. + slope, intercept : float, optional + The scaling, or `None` for none (a slope of 1 and an intercept of + 0). + + Returns + ------- + numpy.ndarray + The stored values, of type `dtype`. + """ + dtype = np.dtype(dtype) + if slope is None and intercept is None: + return np.asarray(data).astype(dtype, copy=False) + slope = 1.0 if slope is None else float(slope) + intercept = 0.0 if intercept is None else float(intercept) + stored = (np.asarray(data, dtype=np.float64) - intercept) / slope + if dtype.kind in "iu": + stored = np.round(stored) + return stored.astype(dtype) + + +# ---------------------------------------------------------------------- +# PRIVATE +# ---------------------------------------------------------------------- + + +def _same_kind(source: np.dtype, target: np.dtype) -> bool: + """Whether values of `source` may be stored as `target` without + changing their kind: integers (booleans included) as integers, + floats as floats, complex numbers as complex numbers.""" + + def kind(dtype: np.dtype) -> str: + return "i" if dtype.kind in "biu" else dtype.kind + + return kind(source) == kind(target) and target.kind != "b" + + +def _present(value: tx.Any) -> tx.Any: + return None if value is UNSUPPORTED else value + + +def _fits( + data: tx.Any, dtype: np.dtype, slope: float, intercept: float +) -> bool: + """Whether every value is an integer of `dtype` once unscaled.""" + if dtype.kind not in "iu" or not slope: + return False + try: + values = np.asarray(data, dtype=np.float64) + except Exception: + return False + if not values.size or not np.all(np.isfinite(values)): + return bool(values.size == 0) + stored = (values - intercept) / slope + rounded = np.round(stored) + info = np.iinfo(dtype) + return bool( + np.all(np.abs(stored - rounded) <= 1e-3) + and rounded.min() >= info.min + and rounded.max() <= info.max + ) + + +def _changed_fields( + metadata: tx.Any, assigned: tx.Dict[str, tx.Any] +) -> tx.Dict[str, tx.Any]: + """ + The fields of `metadata` that changed since it was read. + + The metadata of a file format (`FileBasedMetadata`, in `brainhops.io`, + which this package does not import) knows what changed since the + read, with `_changed_fields`. For any other metadata, every field in + `assigned` counts as changed. + """ + changed_fields = getattr(metadata, "_changed_fields", None) + if changed_fields is None: + return assigned + return changed_fields() diff --git a/src/brainhops/datamodel/metadata/_field.py b/src/brainhops/datamodel/metadata/_field.py new file mode 100644 index 00000000..50699cbe --- /dev/null +++ b/src/brainhops/datamodel/metadata/_field.py @@ -0,0 +1,99 @@ +"""`MetadataField`: the `metadata` field of images and transformations.""" + +__all__ = ["MetadataField"] + +# externals +import typing_extensions as tx +from bagof.converters import Converter +from bagof.magic import ConvertTo, KwOnly, NoEq, NoRepr + +# internals +from ._base import Metadata, _is_already + + +class MetadataField: + """ + The annotation of a `metadata` field: + `MetadataField[hint, *annotations]` is + `Annotated[hint, ConvertTo(...), KwOnly(), NoRepr(), NoEq(), + *annotations]`. + + The field is keyword-only, out of `repr` and `==`, converted on + assignment (even on a class that does not convert its fields) and + copied rather than shared. `hint` is a `Metadata` class, or + `Optional` of one; the annotations add the rest: + + ```python + metadata: MetadataField[tx.Optional[Metadata], tx.Doc("...")] = None + metadata: MetadataField[ + NiftiMetadata, Factory(), tx.Doc("...") + ] + ``` + + A narrowed field must be declared on the class itself or on its + first base: `bagof` takes a field from the first base that has it + (see the format author's guide, `docs/dev/metadata-formats.md`). + """ + + def __class_getitem__(cls, params: tx.Any) -> tx.Any: + if not isinstance(params, tuple): + params = (params,) + hint, *extras = params + return tx.Annotated[ + ( + hint, + ConvertTo(_EnsureCopy(hint)), + KwOnly(), + NoRepr(), + NoEq(), + *extras, + ) + ] + + +# ---------------------------------------------------------------------- +# PRIVATE +# ---------------------------------------------------------------------- + + +class _EnsureCopy: + """ + The converter of a `metadata` field: converts what it is given into + the field's class, as `bagof` would (and a metadata object of another + class, even a subclass of a generic field's `Metadata`, by + conversion, which reports the loss), and copies a metadata object + that already is of that class (`Metadata.copy`), so that two images + or transformations never hold the same one. + """ + + def __init__(self, hint: tx.Any) -> None: + self.hint = hint + self._convert: tx.Optional[tx.Callable[[tx.Any], tx.Any]] = None + self._target: tx.Optional[type] = None + + def __call__(self, value: tx.Any) -> tx.Any: + if self._convert is None: + self._convert = Converter.get(self.hint) + self._target = _target_class(self.hint) + target = self._target + if ( + isinstance(value, Metadata) + and target is not None + and not _is_already(value, target) + ): + return target.from_instance(value) + out = self._convert(value) + if out is value and isinstance(out, Metadata): + out = out.copy() + return out + + +def _target_class(hint: tx.Any) -> tx.Optional[type]: + """The `Metadata` class of a field hint (`Optional[X]` gives `X`).""" + if isinstance(hint, type) and issubclass(hint, Metadata): + return hint + for arg in tx.get_args(hint): + found = _target_class(arg) + if found is not None: + return found + return None diff --git a/src/brainhops/datamodel/metadata/_meta.py b/src/brainhops/datamodel/metadata/_meta.py new file mode 100644 index 00000000..313237e4 --- /dev/null +++ b/src/brainhops/datamodel/metadata/_meta.py @@ -0,0 +1,192 @@ +""" +The metaclass of `Metadata`: it reads the `supports=` class keyword of a +subclass, and sets its capabilities. +""" + +# externals +import typing_extensions as tx +from bagof.magic import Factory, fields +from bagof.magic._options import Options + +# internals +from brainhops._core.compat import own_annotations + +from ..base import DataModelBase +from ._sentinel import ALL, UNSUPPORTED +from ._vocabulary import FIELDS, GROUPS + + +class MetadataMeta(type(DataModelBase)): + """ + Reads the `supports=` class keyword of a `Metadata` subclass. + + A class hook cannot read it: `bagof` builds the fields of a class + before `__init_subclass__` runs, and ignores the class keywords it + does not know. So the keyword is popped here, the namespace is + completed (`_declare`) before `bagof` reads it, and the class is + finished (`_finish`) once built; any other keyword `bagof` does not + read is refused, rather than ignored. No metadata class overrides + either step, so they live here rather than on `Metadata`. + """ + + def __new__( + metacls, + name: str, + bases: tx.Tuple[type, ...], + namespace: tx.Dict[str, tx.Any], + supports: tx.Any = None, + **kwargs: tx.Any, + ) -> type: + unknown = sorted(set(kwargs) - _BAGOF_KEYWORDS) + if unknown: + raise TypeError( + f"{name}: unknown class keyword(s) {unknown}; a metadata " + f"class takes supports= and the bagof options." + ) + parent = next((b for b in bases if isinstance(b, metacls)), None) + if parent is None: + # `Metadata` itself, whose capabilities are in its body. + if supports is not None: + raise TypeError( + f"{name}: supports= is for Metadata subclasses." + ) + return super().__new__(metacls, name, bases, namespace, **kwargs) + if supports is not None: + # `Metadata`: the last class of the MRO built by this metaclass. + root = [b for b in parent.__mro__ if isinstance(b, metacls)][-1] + metacls._declare(root, name, namespace, supports) + cls = super().__new__(metacls, name, bases, namespace, **kwargs) + metacls._read_raw_class(cls) + if "__magic_discard__" not in name: + # Not one of the transient classes `bagof` builds. + cls._finish() + return cls + + @staticmethod + def _declare( + root: type, + name: str, + namespace: tx.Dict[str, tx.Any], + supports: tx.Union[str, tx.Iterable[tx.Any]], + ) -> None: + """ + Read `supports=` into the namespace of the class `name`, before + `bagof` builds its fields: a supported field is declared again + (in case a parent did not support it), and any other one is + declared with the default `UNSUPPORTED`, which is exactly what + writing `x: T = UNSUPPORTED` in the class body does. A field + written out in the body is left as it is. `root` is `Metadata`, + whose declarations are copied. + """ + supported = _supported_names(name, supports) + annotations = own_annotations(namespace) + for field in FIELDS: + if field in annotations or field in namespace: + continue + hint = _declared_hint(root, field) + if field in supported: + annotations[field] = hint + if field != "extra": # `extra` has a factory + namespace[field] = None + else: + annotations[field] = _without_factory(hint) + namespace[field] = UNSUPPORTED + # On Python 3.14+ the body's annotations come as a lazy annotate + # function; it is replaced by the plain dict, as up to 3.13. + for key in ("__annotate__", "__annotate_func__"): + namespace.pop(key, None) + namespace["__annotations__"] = annotations + + @staticmethod + def _read_raw_class(cls: type) -> None: + """ + Set `_raw_class` from the type argument of a generic base, as in + `class NiftiMetadata(FileBasedMetadata[nb.Nifti1Header])`: + `type(None)` for `[None]`. A class without such a base inherits + the value of its parent. + """ + for base in cls.__dict__.get("__orig_bases__", ()): + origin = tx.get_origin(base) + args = tx.get_args(base) + if not isinstance(origin, MetadataMeta) or len(args) != 1: + continue + (arg,) = args + if isinstance(arg, tx.TypeVar): + continue + cls._raw_class = type(None) if arg is None else arg + + def _finish(cls) -> None: + """ + Set the capabilities of a class once `bagof` has built it: its + `supported_fields` (the fields whose default is not + `UNSUPPORTED`) and `unsupported_fields`. + """ + supported = frozenset( + field.name + for field in fields(cls) + if field.name in FIELDS and field.default is not UNSUPPORTED + ) + cls.supported_fields = supported + cls.unsupported_fields = frozenset(FIELDS) - supported + + +# ---------------------------------------------------------------------- +# PRIVATE +# ---------------------------------------------------------------------- + + +# The class keywords `bagof` reads (it ignores any other one silently). +_BAGOF_KEYWORDS = frozenset(Options._DEFAULTS) | {"on", "priority"} + + +def _supported_names( + name: str, supports: tx.Union[str, tx.Iterable[tx.Any]] +) -> tx.FrozenSet[str]: + """The field names a `supports=` declaration stands for.""" + if isinstance(supports, str): + if supports != ALL: + raise TypeError( + f"{name}: supports= takes a sequence of field names or " + f"vocabulary groups, or ALL, not {supports!r}." + ) + return frozenset(FIELDS) + names: tx.Set[str] = set() + for item in supports: + if not isinstance(item, type): + names.add(item) + elif item in GROUPS: + names.update(GROUPS[item]) + else: + raise TypeError( + f"{name}: supports= names {item.__name__}, which is not a " + f"vocabulary group; expected one of " + f"{[g.__name__ for g in GROUPS]}." + ) + unknown = names - frozenset(FIELDS) + if unknown: + raise TypeError( + f"{name}: supports= names {sorted(unknown)}, which are not " + f"vocabulary fields; expected some of {sorted(FIELDS)}." + ) + return frozenset(names) + + +def _declared_hint(root: type, name: str) -> tx.Any: + """The annotation of a field as `Metadata` (`root`, or one of its + groups) declares it.""" + return next( + own_annotations(klass)[name] + for klass in root.__mro__ + if name in own_annotations(klass) + ) + + +def _without_factory(hint: tx.Any) -> tx.Any: + """An annotation without its default factory (that of `extra`), for a + field whose default becomes `UNSUPPORTED`. Its converters stay: they + all let `UNSUPPORTED` through.""" + if tx.get_origin(hint) is not tx.Annotated: + return hint + base, *extras = tx.get_args(hint) + kept = [extra for extra in extras if not isinstance(extra, Factory)] + return tx.Annotated[(base, *kept)] if kept else base diff --git a/src/brainhops/datamodel/metadata/_operations.py b/src/brainhops/datamodel/metadata/_operations.py new file mode 100644 index 00000000..d165a4a4 --- /dev/null +++ b/src/brainhops/datamodel/metadata/_operations.py @@ -0,0 +1,462 @@ +""" +Operations: what an image operation did to the axes of an image, and how +each metadata value follows it. + +An image operation that changes axes (`image[index]`, +`image.reslice(...)`) describes what it did as an [`Operation`][]: +[`Indexed`][] for an index, [`Resampled`][] for a resampling. It hands +the operation to +[`Metadata.derive`][brainhops.datamodel.metadata.Metadata.derive], which +gives every value of the metadata to [`propagate`][], and the raw record +of a format to [`propagate_raw`][]. Both look up a handler registered +with [`propagates`][], in this order: + +1. a handler of the type of the value (or of the record), the most + specific class of the value first, and, for each class, the most + specific class of the operation first; +2. for a vocabulary value, a handler of the scope of its field (a + [`Scope`][brainhops.datamodel.metadata.Scope] member), the most + specific class of the operation first; +3. otherwise, a value is kept, and a record is deep-copied. + +A handler lives next to the type it handles: that of an encoding +direction in `_terms`, the scope defaults in `_vocabulary`, that of the +NIfTI header next to `NiftiMetadata`. A format whose raw record holds +content tied to some axes, outside the vocabulary, registers a handler +for the class of its record, which is its own (never a widely used type +such as `dict`). + +This module imports neither the transformations nor the geometry at +the top: the transformations import the metadata (their `metadata` +field). The two computations that need them import them when they run. +""" + +__all__ = [ + "Handler", + "Indexed", + "Operation", + "Resampled", + "propagate", + "propagate_raw", + "propagates", +] + +# stdlib +import copy + +# externals +import numpy as np +import typing_extensions as tx +from bagof.magic import Field, NoEq, NoInit, NoRepr + +# internals +from ..base import DataModelBase +from ..enums import AxisType +from ..systems import CoordinateSystem, _axes_or_unknown +from ._sentinel import UNSUPPORTED + +if tx.TYPE_CHECKING: # pragma: no cover + from .._transformations.base import Transformation + from ..geometry import Geometry + from ._base import Metadata + +Handler = tx.Callable[..., tx.Any] +""" +A propagation handler, `handler(value, operation, *, name, source)`: the +value after the operation. `name` is the name of the field (`"raw"` for +the raw record), and `source` is the metadata the value is read from. +""" + + +class Operation(DataModelBase, frozen=True, eq=False, repr=False): + """ + What an image operation did to the axes of an image. + + An operation is handed to + [`Metadata.derive`][brainhops.datamodel.metadata.Metadata.derive], + which propagates each value of the metadata through it (see + [`propagate`][]). A subclass that no handler names takes the + handlers of its bases: the scope defaults read its `moves_space`, + and the handler of an encoding direction its `voxel_map`. A + subclass overrides those two properties; their defaults (the + spatial axes moved, through an unknown map) clear whatever depends + on the spatial axes. + + A warp would describe the local map of its voxels (its Jacobian), + which an encoding direction could follow voxel by voxel; it is a + possible extension, which no operation implements. + """ + + @property + def moves_space(self) -> bool: + """Whether the spatial axes changed (the default: they did).""" + return True + + @property + def voxel_map(self) -> tx.Optional[np.ndarray]: + """ + The linear part of the map from the old voxel coordinates to the + new ones (new axes by old axes), or `None` when it is unknown or + not affine (the default). + """ + return None + + +class Indexed(Operation): + """ + An image indexed as its data array is, `image[index]`. + + The index is expanded once, on construction, into one component per + axis of the data (`expanded`), and the type of each voxel axis is + read from the coordinate system (`axes`). + """ + + index: tx.Annotated[ + tuple, + tx.Doc("The full index, as given to `image[index]`."), + Field(convert=False), + ] + shape: tx.Annotated[ + tx.Tuple[int, ...], + tx.Doc("The shape of the indexed image."), + ] + system: tx.Annotated[ + tx.Optional[CoordinateSystem], + tx.Doc( + "The coordinate system of the voxels of the image (the input " + "of its grid), which gives the type of each axis; `None` when " + "it is not known." + ), + Field(convert=False), + ] = None + expanded: tx.Annotated[ + tuple, + tx.Doc( + "The index, with its `...` replaced by as many full slices as " + "the axes it stands for, and the axes it leaves out filled at " + "the end." + ), + NoInit(), + NoRepr(), + NoEq(), + ] = () + axes: tx.Annotated[ + tx.Tuple[tx.Optional[AxisType], ...], + tx.Doc( + "The type of each axis of the image, `None` where it is not known." + ), + NoInit(), + NoRepr(), + NoEq(), + ] = () + + def __post_init__(self) -> None: + parent = getattr(super(), "__post_init__", None) + if parent is not None: + parent() + index = self.index if isinstance(self.index, tuple) else (self.index,) + ndim = len(self.shape) + object.__setattr__(self, "index", index) + object.__setattr__(self, "expanded", _expand_index(index, ndim)) + object.__setattr__(self, "axes", _axis_types(self.system, ndim)) + + def positions(self, axis: AxisType) -> tx.Optional[np.ndarray]: + """ + The positions that the index keeps along an axis of a type, in + order. + + Parameters + ---------- + axis : AxisType + The type of the axis. When the image has several axes of this + type, the first one is read. + + Returns + ------- + np.ndarray or None + The kept positions: the full range when the axis is + untouched. `None` when the index drops the axis (an + integer), when it keeps a selection that is not 1-D, when it + is out of the range of the axis, or when the image has no + axis of this type. + """ + axis = AxisType(axis) + old = 0 + for component in self.expanded: + if component is None: + continue + if self.axes[old] is axis: + try: + kept = np.arange(self.shape[old])[component] + except IndexError: + return None + return kept if np.ndim(kept) == 1 else None + old += 1 + return None + + @property + def moves_space(self) -> bool: + """ + Whether the index changes a spatial axis, or an axis of unknown + type, or moves one to another position (an integer or a `None` + before it). + """ + old = new = 0 + for component in self.expanded: + if component is None: + new += 1 + continue + kind, size = self.axes[old], self.shape[old] + changes = not ( + isinstance(component, slice) + and range(*component.indices(size)) == range(size) + ) + if kind in (None, AxisType.space) and (changes or old != new): + return True + old += 1 + if not isinstance(component, (int, np.integer)): + new += 1 + return False + + @property + def voxel_map(self) -> tx.Optional[np.ndarray]: + """ + The linear part of the map from the old voxels to the new ones, + with a zero column for each axis an integer dropped; `None` for + an index other than integers, slices and `None`. + """ + if not all( + c is None or isinstance(c, (int, slice)) for c in self.expanded + ): + return None + # Not at the top: see the docstring of the module. + from ..geometry import _index2transform + + # The map of the index goes from the new voxels to the old ones; + # its pseudo-inverse goes back. + sub2full, _ = _index2transform(self.index, self.shape, self.system) + matrix = np.asarray(sub2full.matrix, dtype=float) + return np.linalg.pinv(matrix[:, :-1]) + + +class Resampled(Operation): + """ + An image resampled onto a new geometry, `image.reslice(geometry)`. + """ + + transformation: tx.Annotated[ + "Transformation", + tx.Doc( + "The map from the new voxel coordinates to the old ones, " + "without the grid of the new geometry." + ), + Field(convert=False), + ] + geometry: tx.Annotated[ + tx.Optional["Geometry"], + tx.Doc( + "The geometry the image is resampled onto, or `None` when it " + "is not known (a level of a pyramid, whose shape is not known " + "until its data is read)." + ), + Field(convert=False), + ] = None + + @property + def moves_space(self) -> bool: + """Always `True`: a resampling moves the spatial axes.""" + return True + + @property + def voxel_map(self) -> tx.Optional[np.ndarray]: + """ + The linear part of the inverse of `transformation`, from the old + voxels to the new ones; `None` when it does not reduce to an + affine. + """ + # Not at the top: see the docstring of the module. + from .._transformations.multiscale import _as_affine + + affine = _as_affine(self.transformation.inverse()) + if affine is None: + return None + return np.asarray(affine.matrix, dtype=float)[:, :-1] + + +def propagates( + key: tx.Any, operation_type: tx.Type[Operation] +) -> tx.Callable[[Handler], Handler]: + """ + Register a propagation handler: a decorator. + + Parameters + ---------- + key : type or Scope + The class of the values the handler propagates (a vocabulary + value, or the class of a raw record), or a scope, whose handler + propagates the values of the fields in that scope that no + handler of their type takes. + operation_type : type + The class of the operations the handler takes, a subclass of + [`Operation`][] (or `Operation` itself, for every operation). + + Returns + ------- + callable + The decorator, which registers the handler and returns it + unchanged. + + Raises + ------ + TypeError + If `key` is `object`, which would take every value, or if + `operation_type` is not an `Operation` class. + """ + if key is object: + raise TypeError("A handler of `object` would take every value.") + if not ( + isinstance(operation_type, type) + and issubclass(operation_type, Operation) + ): + raise TypeError( + f"Expected an Operation class, got {operation_type!r}." + ) + + def register(handler: Handler) -> Handler: + _HANDLERS[(key, operation_type)] = handler + return handler + + return register + + +def propagate( + value: tx.Any, + operation: Operation, + *, + name: str, + scope: tx.Any, + source: "Metadata", +) -> tx.Any: + """ + A value of the metadata after an operation, from the handler of its + type, or else from that of its scope (see the lookup order in the + docstring of the module). A value no handler takes is kept, and + `UNSUPPORTED` always is. + + Parameters + ---------- + value : object + The value of the field. + operation : Operation + What the image operation did. + name : str + The name of the field. + scope : Scope + The scope of the field. + source : Metadata + The metadata the value is read from. + + Returns + ------- + object + The value of the field of the derived metadata. + """ + if value is UNSUPPORTED: + return value + handler = _lookup(type(value).__mro__, operation) + if handler is None: + handler = _lookup((scope,), operation) + if handler is None: + return value + return handler(value, operation, name=name, source=source) + + +def propagate_raw( + raw: tx.Any, + operation: tx.Optional[Operation], + *, + source: "Metadata", +) -> tx.Any: + """ + The raw record of the metadata of a format after an operation, from + the handler of its type, or else a deep copy of it. + + Parameters + ---------- + raw : object + The raw record, or `None`. + operation : Operation, optional + What the image operation did; `None` for a derivation on the same + axes. + source : Metadata + The metadata the record belongs to. + + Returns + ------- + object + The record of the derived metadata: never the record itself, so + that the derived metadata can edit it. + """ + handler = None + if operation is not None: + handler = _lookup(type(raw).__mro__, operation) + if handler is None: + return copy.deepcopy(raw) + return handler(raw, operation, name="raw", source=source) + + +# ---------------------------------------------------------------------- +# PRIVATE +# ---------------------------------------------------------------------- + + +# (class of value or scope, class of operation) -> handler. +_HANDLERS: tx.Dict[tx.Tuple[tx.Any, type], Handler] = {} + + +def _lookup( + keys: tx.Iterable[tx.Any], operation: Operation +) -> tx.Optional[Handler]: + """The handler of the first key that has one, the most specific class + of the operation first, for each key.""" + for key in keys: + for kind in type(operation).__mro__: + handler = _HANDLERS.get((key, kind)) + if handler is not None: + return handler + return None + + +def _expand_index( + index: tx.Tuple[tx.Any, ...], ndim: int +) -> tx.Tuple[tx.Any, ...]: + """An index with its `...` replaced by as many full slices as the axes + it stands for, and the axes it leaves out filled at the end, as + `_index2transform` expands it.""" + # Compared by identity, so that an array in the index is not compared + # with `...` element by element. + at = next((i for i, c in enumerate(index) if c is ...), None) + if at is None: + index, at = (*index, ...), len(index) + used = sum(1 for c in index if c is not None and c is not ...) + fill = (slice(None),) * (ndim - used) + return index[:at] + fill + index[at + 1 :] + + +def _axis_types( + system: tx.Optional[CoordinateSystem], ndim: int +) -> tx.Tuple[tx.Optional[AxisType], ...]: + """The type of each voxel axis of an image, `None` where it is not + known.""" + axes = _axes_or_unknown(system) + if axes == [...]: + return (None,) * ndim + if axes.is_open: + axes = axes.expand(ndim) + kinds = [] # type: tx.List[tx.Optional[AxisType]] + for axis in list(axes)[:ndim]: + try: + kinds.append(AxisType(getattr(axis, "type", None))) + except ValueError: + kinds.append(None) + return tuple(kinds) + (None,) * (ndim - len(kinds)) diff --git a/src/brainhops/datamodel/metadata/_report.py b/src/brainhops/datamodel/metadata/_report.py new file mode 100644 index 00000000..94564d06 --- /dev/null +++ b/src/brainhops/datamodel/metadata/_report.py @@ -0,0 +1,379 @@ +"""Loss reports, and the loss policy.""" + +__all__ = [ + "ConversionReport", + "LossPolicy", + "MetadataLossError", + "MetadataLossWarning", + "OnLoss", + "apply_loss_policy", + "collect_loss_reports", + "metadata_loss_policy", +] + +# stdlib +import contextlib +import contextvars +import enum +import warnings + +# externals +import typing_extensions as tx +from bagof.magic import Factory + +# internals +from ..base import DataModelBase + + +class ConversionReport(DataModelBase): + """ + The record of what a conversion or a write could not carry over. + + A conversion between metadata classes, or the write of metadata into + a file, fills a report as it goes. A field whose value was dropped is + listed in `lost`, with the value. A field whose value was stored, but + not exactly, is listed in `approximated`, with a short description of + the change (such as `"truncated to 80 bytes"`). + + The loss policy decides what happens to a report that is not empty + (see [`metadata_loss_policy`][]). + """ + + source: tx.Annotated[ + tx.Optional[str], tx.Doc("The format converted from.") + ] = None + target: tx.Annotated[ + tx.Optional[str], tx.Doc("The format converted to.") + ] = None + lost: tx.Annotated[ + tx.Dict[str, tx.Any], + tx.Doc("Field name -> the value that was dropped."), + Factory(), + ] + approximated: tx.Annotated[ + tx.Dict[str, str], + tx.Doc("Field name -> what changed in the stored value."), + Factory(), + ] + + @property + def lossy(self) -> bool: + """Whether the report lists a field as lost or approximated.""" + return bool(self.lost or self.approximated) + + def merge(self, other: "ConversionReport") -> tx.Self: + """ + Add the entries of another report to this report, in place. + + Parameters + ---------- + other : ConversionReport + The report whose entries are added. An entry of `other` + replaces the entry of this report for the same field. + + Returns + ------- + ConversionReport + This report. + """ + self.lost.update(other.lost) + self.approximated.update(other.approximated) + return self + + @classmethod + def merged(cls, reports: tx.Sequence["ConversionReport"]) -> tx.Self: + """ + Merge several reports, in order, into a new report. + + The merged report goes from the source of the first report to the + target of the last one, and holds the entries of all of them. + When two reports have an entry for the same field, the later + entry wins. + + Parameters + ---------- + reports : sequence of ConversionReport + The reports to merge, in the order the steps happened. + + Returns + ------- + ConversionReport + The merged report. + """ + merged = cls( + source=reports[0].source if reports else None, + target=reports[-1].target if reports else None, + ) + for report in reports: + merged.merge(report) + return merged + + def raise_if_lossy(self) -> None: + """ + Raise an error when anything was lost or approximated. + + Raises + ------ + MetadataLossError + If the report lists a field as lost or approximated. + """ + if self.lossy: + raise MetadataLossError(self) + + def __str__(self) -> str: + where = f"{self.source or '?'} -> {self.target or '?'}" + if not self.lossy: + return f"Metadata conversion {where}: nothing lost." + parts = [] + if self.lost: + items = ", ".join(f"{k}={short(v)}" for k, v in self.lost.items()) + parts.append(f"lost {items}") + if self.approximated: + items = ", ".join( + f"{k} ({v})" for k, v in self.approximated.items() + ) + parts.append(f"approximated {items}") + return f"Metadata conversion {where}: " + "; ".join(parts) + "." + + +class MetadataLossWarning(UserWarning): + """ + The warning issued when some metadata could not be carried over. + + The `report` attribute holds the [`ConversionReport`][] that says what + was lost or approximated. + """ + + def __init__(self, report: ConversionReport) -> None: + """ + Parameters + ---------- + report : ConversionReport + What was lost or approximated. + """ + super().__init__(str(report)) + self.report = report + + +class MetadataLossError(Exception): + """ + The error raised, under the `"raise"` policy, when some metadata could + not be carried over. + + The `report` attribute holds the [`ConversionReport`][] that says what + was lost or approximated. The error is deliberately neither a + `TypeError` nor a `ValueError`, because field converters turn those + two into conversion errors, and a refused loss must surface as itself. + """ + + def __init__(self, report: ConversionReport) -> None: + """ + Parameters + ---------- + report : ConversionReport + What was lost or approximated. + """ + super().__init__(str(report)) + self.report = report + + +LossPolicy = tx.Literal["ignore", "warn", "raise"] +"""A loss policy: ignore a loss, warn about it, or raise.""" + +OnLoss = tx.Union[LossPolicy, ConversionReport] +""" +What an `on_loss=` argument takes: a [`LossPolicy`][], or a +[`ConversionReport`][] to fill with what is lost, instead of warning or +raising. +""" + + +@contextlib.contextmanager +def metadata_loss_policy(policy: LossPolicy) -> tx.Iterator[None]: + """ + Set the loss policy of the conversions and writes made in a block. + + The policy in effect governs every conversion that takes no + `on_loss=` argument, in particular the implicit conversions that field + converters trigger: assigning a `NiftiMetadata` to a field typed + `Metadata`, or saving an MGH image as NIfTI. Outside of any block, the + policy is `"warn"`. + + Parameters + ---------- + policy : {"ignore", "warn", "raise"} + The policy: ignore a loss, warn about it, or raise + [`MetadataLossError`][]. + + Yields + ------ + None + + Raises + ------ + ValueError + If `policy` is not one of the three policies. + + Examples + -------- + ```python + with metadata_loss_policy("raise"): + nifti = NiftiImage.from_other(mgh) # raises if anything is lost + ``` + """ + token = _POLICY.set(_check_policy(policy)) + try: + yield + finally: + _POLICY.reset(token) + + +def apply_loss_policy( + report: ConversionReport, + on_loss: tx.Optional[OnLoss] = None, + *, + stacklevel: int = 2, +) -> ConversionReport: + """ + Act on a report, according to a loss policy. + + Under the `"ignore"` policy nothing happens, under `"warn"` a + [`MetadataLossWarning`][] is issued, and under `"raise"` a + [`MetadataLossError`][] is raised. A report that lists nothing as lost + or approximated is silent under every policy. Inside a + [`collect_loss_reports`][] block, a report that would be warned about + is collected instead. + + When `on_loss` is itself a report, the entries of `report` are added + to it, whether anything was lost or not, and nothing is warned or + raised: the caller acts on the report it passed. Its `source` and + `target` are set from `report` when it has none yet. + + Parameters + ---------- + report : ConversionReport + The report to act on. + on_loss : {"ignore", "warn", "raise"} or ConversionReport, optional + The policy to apply, or a report to fill. By default, the policy + in effect (see [`metadata_loss_policy`][]). + stacklevel : int, optional + The stack level of the warning, counted from the caller of this + function. + + Returns + ------- + ConversionReport + `report` itself. + + Raises + ------ + MetadataLossError + If the policy is `"raise"` and the report is lossy. + ValueError + If `on_loss` is not a policy nor a report. + """ + if isinstance(on_loss, ConversionReport): + if on_loss is not report: + on_loss.source = on_loss.source or report.source + on_loss.target = on_loss.target or report.target + on_loss.merge(report) + return report + policy = _check_policy(on_loss or _POLICY.get()) + if not report.lossy or policy == "ignore": + return report + if policy == "raise": + raise MetadataLossError(report) + collected = _COLLECTED.get() + if collected is not None: + # Inside `collect_loss_reports`: the caller acts on it. + collected.append(report) + return report + warnings.warn(MetadataLossWarning(report), stacklevel=stacklevel + 1) + return report + + +@contextlib.contextmanager +def collect_loss_reports() -> tx.Iterator[tx.List[ConversionReport]]: + """ + Collect the reports that the conversions and writes made in a block + would warn about, instead of warning about them. + + Under the `"raise"` policy, a loss still raises where it happens. + `io.save` uses this context manager to warn once for a save that + first converts the object into the format of the file and then + writes it. + + Yields + ------ + list of ConversionReport + The list that the collected reports are appended to. + + Examples + -------- + ```python + with collect_loss_reports() as reports: + nifti = NiftiImage.from_other(mgh) + nifti.save("out.nii.gz") + if reports: + apply_loss_policy(ConversionReport.merged(reports), "warn") + ``` + """ + reports: tx.List[ConversionReport] = [] + token = _COLLECTED.set(reports) + try: + yield reports + finally: + _COLLECTED.reset(token) + + +# ---------------------------------------------------------------------- +# PRIVATE +# ---------------------------------------------------------------------- + + +def short(value: tx.Any, width: int = 40) -> str: + """ + A short text for a value in a report. + + Parameters + ---------- + value : object + The value. A known term reads as its string. + width : int, optional + The maximum number of characters. + + Returns + ------- + str + The `repr` of the value, cut with `...` when it is too long. + """ + if isinstance(value, enum.Enum): + # A known term reads as the term (`'scanner'`). + value = value.value + text = repr(value) + return text if len(text) <= width else text[: width - 3] + "..." + + +_POLICIES = ("ignore", "warn", "raise") + + +def _check_policy(policy: str) -> str: + if policy not in _POLICIES: + raise ValueError( + f"A metadata loss policy is one of {_POLICIES}, not {policy!r}." + ) + return policy + + +# Not annotated: `ContextVar[...]` is not subscriptable on Python 3.8, and +# the type is inferred from the default anyway. +_POLICY = contextvars.ContextVar( + "brainhops_metadata_loss_policy", default="warn" +) + + +# The reports collected by `collect_loss_reports`, when one is active (a +# list of `ConversionReport`), else `None`. +_COLLECTED = contextvars.ContextVar( + "brainhops_metadata_loss_reports", default=None +) diff --git a/src/brainhops/datamodel/metadata/_sentinel.py b/src/brainhops/datamodel/metadata/_sentinel.py new file mode 100644 index 00000000..bc20cb58 --- /dev/null +++ b/src/brainhops/datamodel/metadata/_sentinel.py @@ -0,0 +1,52 @@ +"""The `UNSUPPORTED` sentinel and the `Maybe` value type.""" + +__all__ = ["ALL", "UNSUPPORTED", "Maybe", "Unsupported"] + +# externals +import typing_extensions as tx + + +class Unsupported: + """ + The type of [`UNSUPPORTED`][brainhops.datamodel.metadata.UNSUPPORTED]. + + A field of a format that cannot store it holds `UNSUPPORTED`, which + is distinct from `None` ("nobody set it"). There is one instance: + it is falsy, never equal to anything but itself, and survives + copying and pickling as the same object. + """ + + __slots__ = () + _instance: tx.ClassVar[tx.Optional["Unsupported"]] = None + + def __new__(cls) -> tx.Self: + if cls._instance is None: + cls._instance = object.__new__(cls) + return cls._instance + + def __bool__(self) -> bool: + return False + + def __repr__(self) -> str: + return "UNSUPPORTED" + + def __reduce__(self) -> str: + # A string makes pickle (and `copy`) look the singleton up by name. + return "UNSUPPORTED" + + +UNSUPPORTED = Unsupported() +"""The format cannot store this field. Singleton, falsy, not `None`.""" + +# Above `Maybe`, which is built from it at import time. +_T = tx.TypeVar("_T") + +Maybe = tx.Union[_T, None, Unsupported] +""" +A vocabulary value: a value, `None` (unknown) or `UNSUPPORTED`. + +`Maybe[float]` is `Union[float, None, Unsupported]`. +""" + +ALL = "all" +"""`supports=ALL` declares that a format can store every field.""" diff --git a/src/brainhops/datamodel/metadata/_terms.py b/src/brainhops/datamodel/metadata/_terms.py new file mode 100644 index 00000000..b94293d9 --- /dev/null +++ b/src/brainhops/datamodel/metadata/_terms.py @@ -0,0 +1,445 @@ +"""The structured values of the vocabulary, and the field converters.""" + +__all__ = ["Channel", "EncodingDirection", "GeneratedBy"] + +# stdlib +import math + +# externals +import numpy as np +import typing_extensions as tx +from bagof.magic import ConvertTo, replace + +# internals +from brainhops._core.enum import EnumConverter +from brainhops._core.typing import ArrayLike + +from ..base import DataModelBase +from ..enums import SpaceEnum +from ..systems import CoordinateSystem +from ..units import Unit +from ._operations import Operation, propagates +from ._sentinel import UNSUPPORTED + +AXES = "ijk" +"""The voxel axes of a BIDS direction, in order.""" + + +class GeneratedBy(DataModelBase): + """One entry of BIDS `GeneratedBy`: a program that produced the data.""" + + name: tx.Annotated[str, tx.Doc("Name of the program (BIDS `Name`).")] + version: tx.Annotated[ + tx.Optional[str], tx.Doc("Its version (BIDS `Version`).") + ] = None + description: tx.Annotated[ + tx.Optional[str], tx.Doc("What it did (BIDS `Description`).") + ] = None + code_url: tx.Annotated[ + tx.Optional[str], tx.Doc("Where its code lives (BIDS `CodeURL`).") + ] = None + + +# The converter of `Channel.color`; above the class, which evaluates it. +def _rgba(value: tx.Any) -> tx.Any: + """A color as an RGBA hex string: upper case, without `#`, with an + opaque alpha when it has none.""" + if not isinstance(value, str): + return value + value = value.lstrip("#").upper() + if len(value) == 6: + value += "FF" + return value or None + + +class Channel(DataModelBase): + """The description of one channel of an image.""" + + name: tx.Annotated[tx.Optional[str], tx.Doc("The channel label.")] = None + color: tx.Annotated[ + tx.Optional[str], + tx.Doc( + """ + Display color, as an RGBA hex string. It is held in upper + case, without `#`, and an RGB string (6 digits) is given an + opaque alpha (`"0000ff"` is `"0000FFFF"`). + """ + ), + ConvertTo(_rgba), + ] = None + display_range: tx.Annotated[ + tx.Optional[tx.Tuple[float, float]], + tx.Doc("Display window `(min, max)`."), + ] = None + unit: tx.Annotated[ + tx.Optional[str], tx.Doc("Unit of the channel's values.") + ] = None + + +# The converters of `EncodingDirection.vector` and `.space`; above the +# class, which evaluates them. +def _vector(value: tx.Union[str, ArrayLike]) -> tx.Tuple[float, ...]: + if isinstance(value, str): + return _bids_vector(value) + return tuple(float(v) for v in np.ravel(np.asarray(value, dtype=float))) + + +def _space(value: tx.Any) -> tx.Any: + """The converter of `EncodingDirection.space`: a coordinate system is + kept as it is, and a label becomes a known space when it is one.""" + if isinstance(value, CoordinateSystem): + return value + return _SPACE_TERM(value) + + +class EncodingDirection(DataModelBase): + """ + The direction of an encoding axis, such as the phase-encoding or the + slice-encoding axis, as a unit vector in a coordinate system. + + When `space` is `None`, the vector is expressed in the voxel axes of + the image, which is the frame BIDS uses for `PhaseEncodingDirection`. + In that frame, `EncodingDirection("j-")` is the same direction as + `EncodingDirection((0, -1, 0))`. A direction that is aligned with a + voxel axis reads and writes as a BIDS string (see `to_bids`). Any + other direction, such as an oblique direction obtained after a + resampling, is kept exactly, and a format that can only store an axis + reports it as lost. + + The space is the label of a world space (a `SpaceEnum` term, or any + string), or a brainhops + [`CoordinateSystem`][brainhops.datamodel.systems.CoordinateSystem]. + A file stores the space as a string, so a coordinate system is + written as its `name`, which reads back as a label; a direction in a + coordinate system without a name cannot be written to JSON, and is + reported as lost. + + Two directions are equal when their vectors and their spaces are + equal: a coordinate system is equal to an equal coordinate system, + never to a label, not even its own name. The vector is normalised on + construction, and a component within `1e-9` of 0, 1 or -1 is snapped + to that value, so that `(0, 0, 2)`, `"k"`, and `"k"` mapped through a + permutation of the axes are the same vector. + """ + + vector: tx.Annotated[ + ArrayLike, + tx.Doc( + "The direction: an array-like vector (or a BIDS string, " + "`'j-'`), stored as a tuple of floats, normalised and with " + "its near-axis components snapped on construction." + ), + ConvertTo(_vector), + ] + space: tx.Annotated[ + tx.Optional[tx.Union[SpaceEnum, CoordinateSystem, str]], + tx.Doc( + "The coordinate system of `vector`: `None` for the image's " + "voxel axes, the label of a world space, or a " + "`CoordinateSystem`." + ), + ConvertTo(_space), + ] = None + + def __post_init__(self) -> None: + parent = getattr(super(), "__post_init__", None) + if parent is not None: + parent() + vector = np.asarray(self.vector, dtype=float) + norm = float(np.linalg.norm(vector)) + if not vector.size or not norm or not math.isfinite(norm): + raise ValueError( + f"A direction is a non-zero vector, not {self.vector!r}." + ) + self.vector = tuple(_snapped(float(v)) for v in vector / norm) + + def to_bids(self) -> tx.Optional[str]: + """ + The BIDS string of this direction, such as `"j-"`. + + Returns + ------- + str or None + The BIDS string, or `None` when the direction is not along one + of the first three voxel axes (because it is expressed in a + world space, or because it is oblique). + """ + if self.space is not None: + return None + vector = np.asarray(self.vector, dtype=float) + index = int(np.argmax(np.abs(vector))) + if index >= len(AXES) or not math.isclose( + abs(vector[index]), 1.0, abs_tol=1e-6 + ): + return None + return AXES[index] + ("-" if vector[index] < 0 else "") + + def transform(self, linear: tx.Any) -> tx.Self: + """ + The direction after a linear map of its coordinate system. + + Parameters + ---------- + linear : array-like or Transformation + The matrix that maps the old axes to the new ones, or a + brainhops `Transformation` that reduces to an affine, whose + linear part is used (the translation does not move a + direction). + + Returns + ------- + EncodingDirection + The mapped direction, normalised, in the same `space`. + + Raises + ------ + TypeError + If `linear` is a transformation that does not reduce to an + affine (a field, for instance), which maps no direction to a + single direction. + """ + matrix = _linear_part(linear) + return replace(self, vector=tuple(matrix @ np.asarray(self.vector))) + + def __repr__(self) -> str: + bids = self.to_bids() + if bids is not None: + return f"EncodingDirection({bids!r})" + vector = tuple(round(v, 6) for v in self.vector) + if self.space is None: + return f"EncodingDirection({vector!r})" + if isinstance(self.space, CoordinateSystem): + return f"EncodingDirection({vector!r}, space={self.space!r})" + return f"EncodingDirection({vector!r}, space={str(self.space)!r})" + + +@propagates(EncodingDirection, Operation) +def _propagate_direction( + value: EncodingDirection, + operation: Operation, + *, + name: str, + source: tx.Any, +) -> tx.Optional[EncodingDirection]: + """ + An encoding direction after an operation: it follows the voxel axes. + + A direction is kept when the spatial axes did not move, or when it + is expressed in a world space. One in voxel axes goes through the + map of the voxels (`operation.voxel_map`), which may have more axes + than the direction, which lies in the first ones (`ijk`), as for a + 4-D image: the direction is then mapped as a vector of the old axes + that is zero beyond its own, and is kept only when it still lies in + the first axes. It is cleared when the map is unknown, when it moves + the direction off those axes, or when it maps the direction to zero + (its axis was dropped). + """ + if not operation.moves_space or value.space is not None: + return value + linear = operation.voxel_map + if linear is None: + return None + linear = np.asarray(linear, dtype=float) + vector = np.asarray(value.vector, dtype=float) + size = len(vector) + if linear.ndim != 2 or min(linear.shape) < size: + return None + beyond = linear[size:, :size] @ vector + if np.any(np.abs(beyond) > 1e-9 * np.abs(linear).max()): + # The direction moved onto an axis it cannot be expressed on. + return None + try: + return value.transform(linear[:size, :size]) + except ValueError: + # Mapped to zero: its axis was dropped. + return None + + +# ---------------------------------------------------------------------- +# CONVERTERS +# ---------------------------------------------------------------------- + + +class MaybeEnumConverter(EnumConverter): + """ + The converter of a vocabulary field that has a list of known terms. + + It converts as [`EnumConverter`][brainhops._core.enum.EnumConverter] + does, and lets `UNSUPPORTED` through, as every vocabulary converter + does. + """ + + __slots__ = () + + def __call__(self, value: tx.Any) -> tx.Any: + """ + Convert a value. + + Parameters + ---------- + value : Enum, str, None or UNSUPPORTED + The value to convert. + + Returns + ------- + Enum, str, None or UNSUPPORTED + The converted value. + + Raises + ------ + TypeError + If `value` is neither a member, a string, `None` nor + `UNSUPPORTED`. + """ + if value is UNSUPPORTED: + return value + return super().__call__(value) + + +def unit(value: tx.Any) -> tx.Any: + """ + Convert the value of a unit field. + + A unit name that the units module parses becomes a `Unit`. A name + that it cannot parse stays a string, so that the spelling of a file + survives. + + Parameters + ---------- + value : Unit, str, None or UNSUPPORTED + The value to convert. + + Returns + ------- + Unit, str, None or UNSUPPORTED + The converted value. + + Raises + ------ + TypeError + If `value` is neither a unit nor a string. + """ + if _is_absent(value) or isinstance(value, Unit): + return value + if isinstance(value, str): + try: + return Unit(value) + except ValueError: + return value + raise TypeError(f"Expected a Unit or a str, not {type(value).__name__}.") + + +def dtype(value: tx.Any) -> tx.Any: + """ + Convert the value of a data type field to a numpy data type in native + byte order. + + The byte order is a detail of how a file encodes its values, never + metadata, so it is dropped. + + Parameters + ---------- + value : dtype-like, None or UNSUPPORTED + The value to convert. + + Returns + ------- + numpy.dtype, None or UNSUPPORTED + The converted value. + """ + if _is_absent(value): + return value + return np.dtype(value).newbyteorder("=") + + +def direction(value: tx.Any) -> tx.Any: + """ + Convert the value of an encoding direction field. + + Parameters + ---------- + value : EncodingDirection, str, array-like, mapping, None or UNSUPPORTED + A direction, a BIDS string such as `"j-"`, a vector, or a mapping + with the keys `vector` and `space` (or `Vector` and `Space`, as + JSON spells them). + + Returns + ------- + EncodingDirection, None or UNSUPPORTED + The converted value. + + Raises + ------ + ValueError + If the value does not describe a direction. + """ + if _is_absent(value) or isinstance(value, EncodingDirection): + return value + if isinstance(value, tx.Mapping): + vector = value.get("vector", value.get("Vector")) + space = value.get("space", value.get("Space")) + return EncodingDirection(vector, space=space) + return EncodingDirection(value) + + +# ---------------------------------------------------------------------- +# PRIVATE +# ---------------------------------------------------------------------- + + +def _is_absent(value: tx.Any) -> bool: + """Whether a value is absent, so that there is nothing to convert: + `None` (unknown) or `UNSUPPORTED` (no slot). Every vocabulary + converter lets an absent value through unchanged.""" + return value is None or value is UNSUPPORTED + + +# A component this close to 0 or to +-1, once normalised, is snapped to +# it: a direction through a permutation or a flip of the axes, or a +# rotation by a multiple of 90 degrees, is then exactly an axis, and +# compares equal to it with the default (field by field) equality. +_SNAP = 1e-9 + + +def _snapped(value: float) -> float: + for exact in (0.0, 1.0, -1.0): + if abs(value - exact) <= _SNAP: + return exact + return value + + +# The converter of a label of a space (see `_space`). +_SPACE_TERM = EnumConverter(SpaceEnum) + + +def _bids_vector(value: str) -> tx.Tuple[float, ...]: + """The unit vector of a BIDS direction (`"i"`, `"j-"`, `"k"`).""" + axis = value[:-1] if value.endswith("-") else value + if len(axis) != 1 or axis not in AXES: + raise ValueError( + f"A BIDS direction is one of 'i', 'j', 'k', optionally " + f"followed by '-', not {value!r}." + ) + vector = [0.0] * len(AXES) + vector[AXES.index(axis)] = -1.0 if value.endswith("-") else 1.0 + return tuple(vector) + + +def _linear_part(linear: tx.Any) -> np.ndarray: + """The matrix of `EncodingDirection.transform`: an array as it is, or + the linear part of a transformation that reduces to an affine.""" + # Not at the top: the transformations import the metadata (their + # `metadata` field), which imports this module. + from .._transformations.base import Transformation + from .._transformations.multiscale import _as_affine + + if not isinstance(linear, Transformation): + return np.asarray(linear, dtype=float) + affine = _as_affine(linear) + if affine is None: + raise TypeError( + f"A direction is mapped by a linear map; {type(linear).__name__} " + f"does not reduce to an affine." + ) + matrix = np.asarray(affine.matrix, dtype=float) + return matrix[:-1, :-1] diff --git a/src/brainhops/datamodel/metadata/_vocabulary.py b/src/brainhops/datamodel/metadata/_vocabulary.py new file mode 100644 index 00000000..2e44c1b2 --- /dev/null +++ b/src/brainhops/datamodel/metadata/_vocabulary.py @@ -0,0 +1,718 @@ +"""The common vocabulary: its scopes, its annotations and its groups.""" + +__all__ = [ + "ALONG", + "BIDS_KEYS", + "FIELDS", + "GROUPS", + "SCOPES", + "VOCABULARY", + "Vocabulary", + "Along", + "Bids", + "DiffusionVocabulary", + "DisplayVocabulary", + "MRIVocabulary", + "MicroscopyVocabulary", + "ProvenanceVocabulary", + "Scope", + "Scoped", + "StorageVocabulary", + "TransformVocabulary", +] + +# stdlib +import datetime +import itertools + +# externals +import numpy as np +import typing_extensions as tx +from bagof.magic import ConvertTo, Field, HideIf, Magic, fields + +# internals +from brainhops._core.enum import StrEnum + +from ..enums import ( + AxisType, + ContrastMethod, + IlluminationType, + IntentEnum, + Manufacturer, + SpaceEnum, +) +from ..units import Unit +from ._operations import Indexed, Operation, propagates +from ._sentinel import Maybe +from ._terms import ( + Channel, + EncodingDirection, + GeneratedBy, + MaybeEnumConverter, + _is_absent, + direction, + dtype, + unit, +) + + +class Scope(StrEnum): + """ + How a vocabulary field propagates when an image is derived from + another one (see `Metadata.derive`). + + +-----------------+-------------------+---------------------------------+ + | Member | Value | Meaning | + +=================+===================+=================================+ + | `FILE` | `"file"` | The field is about the file or | + | | | about the data as a whole. It | + | | | is kept. | + +-----------------+-------------------+---------------------------------+ + | `ACQUISITION` | `"acquisition"` | The field describes the | + | | | acquisition, and does not | + | | | change under a resampling. It | + | | | is kept. | + +-----------------+-------------------+---------------------------------+ + | `SPATIAL` | `"spatial"` | The field is tied to the | + | | | spatial sampling. It is cleared | + | | | when the spatial axes change, | + | | | or mapped along with them. | + +-----------------+-------------------+---------------------------------+ + | `AXIS` | `"axis"` | The field has one entry per | + | | | index along one non-spatial | + | | | axis (see [`Along`][]). It is | + | | | indexed when that axis changes. | + +-----------------+-------------------+---------------------------------+ + """ + + FILE = "file" + ACQUISITION = "acquisition" + SPATIAL = "spatial" + AXIS = "axis" + + +# The scope defaults: how the value of a field propagates through an +# operation when no handler of its type takes it (see `_operations`). + + +@propagates(Scope.FILE, Operation) +@propagates(Scope.ACQUISITION, Operation) +@propagates(Scope.AXIS, Operation) +def _keep( + value: tx.Any, operation: Operation, *, name: str, source: tx.Any +) -> tx.Any: + """A field that the operation does not change: it is kept.""" + return value + + +@propagates(Scope.SPATIAL, Operation) +def _clear_if_space_moves( + value: tx.Any, operation: Operation, *, name: str, source: tx.Any +) -> tx.Any: + """A field tied to the spatial sampling: it is cleared when the + spatial axes move.""" + return None if operation.moves_space else value + + +@propagates(Scope.AXIS, Indexed) +def _index_along( + value: tx.Any, operation: Indexed, *, name: str, source: tx.Any +) -> tx.Any: + """ + A field with one entry per index along an axis, after an index: the + entries at the positions the index keeps along the first axis of + the type the field runs along (`ALONG`). The field is kept when the + image has no such axis, or when the index leaves it untouched, and + cleared when the index drops it. + """ + axis = ALONG[name] + if axis not in operation.axes: + return value + positions = operation.positions(axis) + size = operation.shape[operation.axes.index(axis)] + if positions is not None and np.array_equal(positions, np.arange(size)): + return value + return _select_entries(value, positions) + + +class Bids(Field): + """ + The annotation that gives a vocabulary field its BIDS sidecar key. + + The key lands in the `metadata` of the field, under `"bids"`, where + the sidecar codec reads it. + + Examples + -------- + ```python + repetition_time: tx.Annotated[ + Maybe[float], Bids("RepetitionTime") + ] = None + ``` + """ + + def __init__(self, key: str) -> None: + """ + Parameters + ---------- + key : str + The BIDS key, such as `"RepetitionTime"`. + """ + super().__init__(metadata={"bids": key}) + + +class Scoped(Field): + """ + The annotation that gives a vocabulary field its [`Scope`][]. + + The scope lands in the `metadata` of the field, under `"scope"`. A + field without a scope is in the `FILE` scope. A field in the `AXIS` + scope is declared with [`Along`][] instead, which also names its axis. + + Examples + -------- + ```python + echo_time: tx.Annotated[Maybe[float], Scoped(Scope.ACQUISITION)] = None + ``` + """ + + def __init__(self, scope: tx.Union[Scope, str]) -> None: + """ + Parameters + ---------- + scope : Scope or str + The scope of the field. + + Raises + ------ + ValueError + If `scope` is not a scope, or is `AXIS` (use [`Along`][]). + """ + scope = Scope(scope) + if scope is Scope.AXIS: + raise ValueError( + "A field in the AXIS scope names its axis: use Along(...)." + ) + super().__init__(metadata={"scope": scope}) + + +class Along(Field): + """ + The annotation of a vocabulary field that has one entry per index + along one non-spatial axis of an image. + + The field is in the `AXIS` scope, and its entries run along the axes + of the given type. NIfTI and OME-Zarr both map their time and channel + dimensions to brainhops axis types, so the b-values of a diffusion + image run along the time axis (NIfTI dimension 4), and the + descriptions of the channels run along the channel axis. + + The axis type lands in the `metadata` of the field, under `"along"`, + next to the scope. + + Examples + -------- + ```python + channels: tx.Annotated[ + Maybe[tx.Tuple[Channel, ...]], Along(AxisType.channel) + ] = None + ``` + """ + + def __init__(self, axis: tx.Union[AxisType, str]) -> None: + """ + Parameters + ---------- + axis : AxisType or str + The type of the axis the entries run along. It cannot be + `space`: a field tied to the spatial axes is in the `SPATIAL` + scope. + + Raises + ------ + ValueError + If `axis` is not an axis type, or is `space`. + """ + axis = AxisType(axis) + if axis is AxisType.space: + raise ValueError( + "A field tied to the spatial axes is Scoped(Scope.SPATIAL)." + ) + super().__init__(metadata={"scope": Scope.AXIS, "along": axis}) + + +# ---------------------------------------------------------------------- +# VOCABULARY GROUPS +# ---------------------------------------------------------------------- + + +class Vocabulary( + Magic, + kw_only=True, + convert=True, + # A field is shown in `repr` only when it holds a value: not `None` + # (unknown), not `UNSUPPORTED` (no slot). + repr=HideIf(_is_absent), +): + """ + The base class of the vocabulary groups. + + A vocabulary group is a `Magic` mixin that declares a few fields of + the common vocabulary, and nothing else. [`Metadata`][] inherits every + group, and a format names the groups it can store in its `supports=` + declaration. A group is not meant to be instantiated. The groups + convert their fields (`convert=True`), and hide from `repr` a field + that holds `None` or `UNSUPPORTED`, because the fields of a mixin + keep the options of the class that declares them. + """ + + +class ProvenanceVocabulary(Vocabulary): + """ + Vocabulary group: what the data is and where it comes from (`file` + scope). Not meant to be instantiated; see [`Metadata`][]. + """ + + name: tx.Annotated[ + Maybe[str], tx.Doc("A short name for the data."), Scoped(Scope.FILE) + ] = None + + description: tx.Annotated[ + Maybe[str], + tx.Doc("A free-text description."), + Bids("Description"), + Scoped(Scope.FILE), + ] = None + + history: tx.Annotated[ + Maybe[tx.Tuple[str, ...]], + tx.Doc("The commands that produced the data, oldest first."), + Scoped(Scope.FILE), + ] = None + + generated_by: tx.Annotated[ + Maybe[tx.Tuple[GeneratedBy, ...]], + tx.Doc("The programs that produced the data."), + Bids("GeneratedBy"), + Scoped(Scope.FILE), + ] = None + + creation_time: tx.Annotated[ + Maybe[datetime.datetime], + tx.Doc("When the file was created."), + Scoped(Scope.FILE), + ] = None + + sources: tx.Annotated[ + Maybe[tx.Tuple[str, ...]], + tx.Doc("Files the data was derived from (BIDS provenance)."), + Bids("Sources"), + Scoped(Scope.FILE), + ] = None + + space: tx.Annotated[ + Maybe[tx.Union[SpaceEnum, str]], + tx.Doc( + "The label of the world space (`'MNI152NLin6Asym'`, " + "`'scanner'`, ...; a known one is a `SpaceEnum`); the space " + "itself is geometry." + ), + Bids("SpatialReference"), + Scoped(Scope.FILE), + ConvertTo(MaybeEnumConverter(SpaceEnum)), + ] = None + + intent: tx.Annotated[ + Maybe[tx.Union[IntentEnum, str]], + tx.Doc( + "What the values are, as a NIfTI intent name (a known one is " + "an `IntentEnum`)." + ), + Scoped(Scope.FILE), + ConvertTo(MaybeEnumConverter(IntentEnum)), + ] = None + + +class MRIVocabulary(Vocabulary): + """ + Vocabulary group: MRI acquisition parameters. They are in the + `ACQUISITION` scope, except the encoding directions and the slice + timing, which are tied to the spatial sampling (`SPATIAL` scope). Not + meant to be instantiated; see [`Metadata`][]. + """ + + repetition_time: tx.Annotated[ + Maybe[float], + tx.Doc("Repetition time, in seconds."), + Bids("RepetitionTime"), + Scoped(Scope.ACQUISITION), + ] = None + + echo_time: tx.Annotated[ + Maybe[float], + tx.Doc("Echo time, in seconds."), + Bids("EchoTime"), + Scoped(Scope.ACQUISITION), + ] = None + + inversion_time: tx.Annotated[ + Maybe[float], + tx.Doc("Inversion time, in seconds."), + Bids("InversionTime"), + Scoped(Scope.ACQUISITION), + ] = None + + flip_angle: tx.Annotated[ + Maybe[float], + tx.Doc("Flip angle, in degrees."), + Bids("FlipAngle"), + Scoped(Scope.ACQUISITION), + ] = None + + magnetic_field_strength: tx.Annotated[ + Maybe[float], + tx.Doc("Nominal field strength, in tesla."), + Bids("MagneticFieldStrength"), + Scoped(Scope.ACQUISITION), + ] = None + + manufacturer: tx.Annotated[ + Maybe[tx.Union[Manufacturer, str]], + tx.Doc( + "Manufacturer of the equipment (a known one is a `Manufacturer`)." + ), + Bids("Manufacturer"), + Scoped(Scope.ACQUISITION), + ConvertTo(MaybeEnumConverter(Manufacturer)), + ] = None + + manufacturers_model_name: tx.Annotated[ + Maybe[str], + tx.Doc("Model name of the equipment."), + Bids("ManufacturersModelName"), + Scoped(Scope.ACQUISITION), + ] = None + + institution_name: tx.Annotated[ + Maybe[str], + tx.Doc("Institution responsible for the equipment."), + Bids("InstitutionName"), + Scoped(Scope.ACQUISITION), + ] = None + + acquisition_time: tx.Annotated[ + Maybe[datetime.datetime], + tx.Doc("When the acquisition started."), + Bids("AcquisitionTime"), + Scoped(Scope.ACQUISITION), + ] = None + + phase_encoding_direction: tx.Annotated[ + Maybe[EncodingDirection], + tx.Doc( + "Phase-encoding direction: a unit vector, by default in voxel " + "axes (BIDS `'j-'` is accepted)." + ), + Bids("PhaseEncodingDirection"), + Scoped(Scope.SPATIAL), + ConvertTo(direction), + ] = None + + total_readout_time: tx.Annotated[ + Maybe[float], + tx.Doc("Total readout time, in seconds."), + Bids("TotalReadoutTime"), + Scoped(Scope.ACQUISITION), + ] = None + + effective_echo_spacing: tx.Annotated[ + Maybe[float], + tx.Doc("Effective echo spacing, in seconds."), + Bids("EffectiveEchoSpacing"), + Scoped(Scope.ACQUISITION), + ] = None + + slice_encoding_direction: tx.Annotated[ + Maybe[EncodingDirection], + tx.Doc( + "Slice-encoding direction: a unit vector, by default in voxel " + "axes (BIDS `'k'` is accepted)." + ), + Bids("SliceEncodingDirection"), + Scoped(Scope.SPATIAL), + ConvertTo(direction), + ] = None + + slice_timing: tx.Annotated[ + Maybe[tx.Tuple[float, ...]], + tx.Doc( + "Acquisition time of each slice, in seconds, along the slice " + "encoding direction." + ), + Bids("SliceTiming"), + Scoped(Scope.SPATIAL), + ] = None + + multiband_acceleration_factor: tx.Annotated[ + Maybe[int], + tx.Doc("Multiband acceleration factor."), + Bids("MultibandAccelerationFactor"), + Scoped(Scope.ACQUISITION), + ] = None + + +class DiffusionVocabulary(Vocabulary): + """ + Vocabulary group: the diffusion gradient table, one entry per index + along the time axis (`AXIS` scope). BIDS stores it in `.bval` and + `.bvec` files, not in the sidecar. Not meant to + be instantiated; see [`Metadata`][]. + """ + + bvalues: tx.Annotated[ + Maybe[tx.Tuple[float, ...]], + tx.Doc( + "One b-value per index along the time axis (the volumes of a " + "diffusion series), in s/mm^2." + ), + Along(AxisType.time), + ] = None + + bvectors: tx.Annotated[ + Maybe[tx.Tuple[tx.Tuple[float, float, float], ...]], + tx.Doc( + "One unit gradient direction per index along the time axis, " + "in world (RAS) coordinates." + ), + Along(AxisType.time), + ] = None + + +class DisplayVocabulary(Vocabulary): + """ + Vocabulary group: how the values are shown and what they are. The + channels run along the channel axis (`AXIS` scope); the display range + and the unit apply to every value (`FILE` scope). Not meant to be + instantiated; see [`Metadata`][]. + """ + + display_range: tx.Annotated[ + Maybe[tx.Tuple[float, float]], + tx.Doc("Display window `(min, max)`."), + Scoped(Scope.FILE), + ] = None + + channels: tx.Annotated[ + Maybe[tx.Tuple[Channel, ...]], + tx.Doc("One description per channel."), + Along(AxisType.channel), + ] = None + + data_unit: tx.Annotated[ + Maybe[tx.Union[Unit, str]], + tx.Doc( + "Unit of the data values: a `Unit` whenever the units module " + "parses the name (`'a.u.'`, `'mm/s'`, `'HU'`), and the name " + "itself, as a string, when it does not, so that a file with " + "an odd unit still reads. Formats write it as its symbol " + "(`Unit.symbol`), which parses back to the same unit." + ), + Scoped(Scope.FILE), + ConvertTo(unit), + ] = None + + +class StorageVocabulary(Vocabulary): + """ + Vocabulary group: how the values of the data are stored in the file + (`file` scope). Not meant to be instantiated; see [`Metadata`][]. + + The loaded array always holds the values themselves. A file that + stores integers with an intensity scaling (NIfTI `scl_slope` and + `scl_inter`) loads as floating-point values, which are + `stored * scale_slope + scale_intercept`. A writer stores the data as + `data_type`, with the scaling, when the values of the array are of + that kind and fit the scaling, so that a scaled integer file is + written back as it was read. Otherwise the writer stores the array as + it is, and reports a changed field it could not use. + """ + + data_type: tx.Annotated[ + Maybe[np.dtype], + tx.Doc( + "The element type of the data as stored, in native byte " + "order. It may differ from the type of the loaded array: a " + "scaled integer file loads as floating-point values. A " + "`dtype=` writer option wins over it." + ), + Scoped(Scope.FILE), + ConvertTo(dtype), + ] = None + + scale_slope: tx.Annotated[ + Maybe[float], + tx.Doc( + "The slope of the intensity scaling: a value is `stored * " + "scale_slope + scale_intercept`. `None` means no scaling." + ), + Scoped(Scope.FILE), + ] = None + + scale_intercept: tx.Annotated[ + Maybe[float], + tx.Doc( + "The intercept of the intensity scaling (see `scale_slope`). " + "`None` means no intercept." + ), + Scoped(Scope.FILE), + ] = None + + +class MicroscopyVocabulary(Vocabulary): + """ + Vocabulary group: microscopy acquisition (`acquisition` scope). Not + meant to be instantiated; see [`Metadata`][]. + """ + + objective_magnification: tx.Annotated[ + Maybe[float], + tx.Doc("Nominal magnification of the objective."), + Scoped(Scope.ACQUISITION), + ] = None + + objective_numerical_aperture: tx.Annotated[ + Maybe[float], + tx.Doc("Numerical aperture of the objective."), + Scoped(Scope.ACQUISITION), + ] = None + + illumination_type: tx.Annotated[ + Maybe[tx.Union[IlluminationType, str]], + tx.Doc("Illumination type (a known one is an `IlluminationType`)."), + Scoped(Scope.ACQUISITION), + ConvertTo(MaybeEnumConverter(IlluminationType)), + ] = None + + contrast_method: tx.Annotated[ + Maybe[tx.Union[ContrastMethod, str]], + tx.Doc("Contrast method (a known one is a `ContrastMethod`)."), + Scoped(Scope.ACQUISITION), + ConvertTo(MaybeEnumConverter(ContrastMethod)), + ] = None + + +class TransformVocabulary(Vocabulary): + """ + Vocabulary group: what a transformation relates (`file` scope). Not + meant to be instantiated; see [`Metadata`][]. + """ + + moving: tx.Annotated[ + Maybe[str], + tx.Doc("The moving image of a registration (file reference)."), + Scoped(Scope.FILE), + ] = None + + fixed: tx.Annotated[ + Maybe[str], + tx.Doc("The fixed image of a registration (file reference)."), + Scoped(Scope.FILE), + ] = None + + input_space: tx.Annotated[ + Maybe[tx.Union[SpaceEnum, str]], + tx.Doc("Label of the space a transformation maps from."), + Scoped(Scope.FILE), + ConvertTo(MaybeEnumConverter(SpaceEnum)), + ] = None + + output_space: tx.Annotated[ + Maybe[tx.Union[SpaceEnum, str]], + tx.Doc("Label of the space a transformation maps to."), + Scoped(Scope.FILE), + ConvertTo(MaybeEnumConverter(SpaceEnum)), + ] = None + + +GROUPS: tx.Dict[type, tx.Tuple[str, ...]] = { + # `fields` lists exactly the fields of the group: the base class + # declares none. + group: tuple(field.name for field in fields(group)) + for group in ( + ProvenanceVocabulary, + MRIVocabulary, + DiffusionVocabulary, + DisplayVocabulary, + StorageVocabulary, + MicroscopyVocabulary, + TransformVocabulary, + ) +} +"""Each vocabulary group class, and the names of its fields.""" + + +VOCABULARY: tx.Tuple[str, ...] = tuple( + itertools.chain.from_iterable(GROUPS.values()) +) +""" +The names of the common vocabulary fields, group by group. `extra` is not +one (it is the free-form store next to them). `bagof` lists the fields of +a class with several bases in reverse MRO order, so this is the order to +iterate in, never `fields(Metadata)`. +""" + +FIELDS: tx.Tuple[str, ...] = ("extra",) + VOCABULARY +"""The fields `supports=` speaks of, in the order reports list them.""" + + +BIDS_KEYS: tx.Dict[str, str] = { + field.name: field.metadata["bids"] + for group in GROUPS + for field in fields(group) + if "bids" in (field.metadata or {}) +} +"""Vocabulary field -> its BIDS sidecar key (`Bids(...)`), for the fields +BIDS has a key for.""" + + +SCOPES: tx.Dict[str, Scope] = { + field.name: (field.metadata or {}).get("scope", Scope.FILE) + for group in GROUPS + for field in fields(group) +} +"""Vocabulary field -> its propagation scope (`Scoped` or `Along`).""" + + +ALONG: tx.Dict[str, AxisType] = { + field.name: field.metadata["along"] + for group in GROUPS + for field in fields(group) + if "along" in (field.metadata or {}) +} +"""Vocabulary field in the `AXIS` scope -> the type of the axis its +entries run along (`Along`).""" + + +# ---------------------------------------------------------------------- +# PRIVATE +# ---------------------------------------------------------------------- + + +def _select_entries(value: tx.Any, positions: tx.Any) -> tx.Any: + """ + The entries of a field at `positions`. No positions (the axis was + dropped, or the selection is unknown), or a position beyond the + field, clears it. + """ + if value is None or positions is None: + return None + positions = np.asarray(positions) + if positions.ndim != 1 or positions.dtype.kind not in "iu": + return None + try: + size = len(value) + except TypeError: + return None + if positions.size and not (0 <= positions.min() <= positions.max() < size): + return None + return tuple(value[int(i)] for i in positions) diff --git a/src/brainhops/io/__init__.py b/src/brainhops/io/__init__.py index 578c3cf3..deeeddaf 100644 --- a/src/brainhops/io/__init__.py +++ b/src/brainhops/io/__init__.py @@ -16,11 +16,12 @@ "base", "images", "load", + "metadata", "save", "sniff", "transformations", "vectors", ] -from . import base, images, transformations, vectors +from . import base, images, metadata, transformations, vectors from .base import FileBasedObject, WritableFileBasedObject, load, save, sniff diff --git a/src/brainhops/io/base/_metadata_parser.py b/src/brainhops/io/base/_metadata_parser.py new file mode 100644 index 00000000..b1ccced2 --- /dev/null +++ b/src/brainhops/io/base/_metadata_parser.py @@ -0,0 +1,50 @@ +""" +Reading the metadata of a file without its data. + +A file stores its metadata in a raw record (a NIfTI header, the +attributes of a Zarr array, the JSON of an x5 node, ...), which the +metadata class of the format decodes with `from_raw` and encodes with +`to_raw` and `update_raw`. [`MetadataParser`][] adds the file side: its +`from_*` methods read the raw record of a file, and nothing else, then +build the metadata with `from_raw`. The metadata class of a format lists +it first among its bases, as the image class of a format lists its +parser. + +The parsers own no registry: the dispatcher among the formats is +[`FileBasedMetadata`][brainhops.io.metadata.FileBasedMetadata], +whose `load` reads the metadata of a file in any of these formats. +""" + +__all__ = ["MetadataParser"] + +# internals +from .parsers import FileParser + + +class MetadataParser(FileParser): + """ + Reads the metadata of a file of one format. + + A format implements `from_fileobj`, which reads the raw record of an + open file and builds the metadata with `from_raw`, and the sniffers + that recognise its files; a path is opened by `from_filename`, in + binary mode, and handed to `from_fileobj`, and bytes are wrapped in a + stream and handed to it too (`FileParser.from_bytes` does so for a + class that implements `from_fileobj`). A format that reads paths + otherwise (MGH reads its tags lazily from a path) overrides + `from_filename` too. + The parser only reads, as a `FileParser` does: a format whose record + is an object of its own on disk (the attributes of a Zarr array) + defines its own `to_file`, and the record of any other format is + written by the writer of its images or transformations, along with + the data. The parser of a format stored in HDF5 is + [`Hdf5MetadataParser`][brainhops.io.base.hdf5.Hdf5MetadataParser]. + + The parser of a format owns no registry: the class of the format + also derives from + [`FileBasedMetadata`][brainhops.io.metadata.FileBasedMetadata], + and registers into its registry with + [`register_format`][brainhops.io.base.register_format]. + """ + + _READ_MODE = "rb" diff --git a/src/brainhops/io/base/_mgh_metadata.py b/src/brainhops/io/base/_mgh_metadata.py new file mode 100644 index 00000000..64e1aeda --- /dev/null +++ b/src/brainhops/io/base/_mgh_metadata.py @@ -0,0 +1,403 @@ +""" +The metadata of MGH/MGZ files: `MghMetadata`, the metadata of +`MghImage`. + +Its raw record (`raw`) is an `MghRaw`: the `nibabel` header, which +holds the footer of MRI acquisition parameters, and the bytes of the +trailing tags. What the vocabulary covers: + +| Field | Record | Unit conversion | +|---|---|---| +| `repetition_time` | footer `tr` | ms -> s | +| `echo_time` | footer `te` | ms -> s | +| `inversion_time` | footer `ti` | ms -> s | +| `flip_angle` | footer `flip_angle` | rad -> deg | +| `history` | the `TAG_CMDLINE` tags | one command per tag | +| `data_type` | header `type` | uint8, int16, int32, float32 | + +FreeSurfer stores the times in milliseconds and the flip angle in +radians (`mri_info` prints the latter in degrees); a value of zero means +"not recorded", and reads as `None`. The field of view (`fov`) is not in +the vocabulary (it follows from the geometry) and stays in the raw +record. MGH has no free-form store, so `extra` is unsupported. The +writer stores the data as `data_type` when the array's values are of its +kind, or as the nearest type MGH stores (approximated). + +**Why not `nibabel`'s footer.** `nibabel` has no separate footer class: +the footer fields (`tr`, `flip_angle`, `te`, `ti`, `fov`) are part of +`MGHHeader` (its `hf_dtype` is the header and the footer), which is the +first half of `MghRaw`. What `nibabel` does not read, nor write, is the +tag stream after the footer (the command lines, `TAG_CMDLINE`), which +`brainhops.io.base._mgh_tags` parses. + +**Tags.** The command lines are tags after the footer (see +`brainhops.io.base._mgh_tags`). `history` is decoded only when the whole +tag stream parses; otherwise the tags are kept verbatim and `history` is +unknown (and a new value cannot be written: it is reported as lost). +Writing `history` replaces the command-line tags and keeps every other +tag as it was. The tags sit after the whole volume; `history` is +decoded when the metadata is built, so reading the metadata of an MGZ +decompresses it to its end. +""" + +__all__ = ["MghMetadata", "MghRaw"] + +# stdlib +import functools +import math +from io import BytesIO + +# dependencies +import numpy as np +import typing_extensions as tx +from nibabel.freesurfer import mghformat as _mgh + +# internals +from brainhops._core import path +from brainhops._core.numeric import shortest_decimal +from brainhops.datamodel.metadata import ConversionReport +from brainhops.io.base._base import register_format +from brainhops.io.base._metadata_parser import MetadataParser +from brainhops.io.base._mgh_tags import decode_history, encode_history +from brainhops.io.base.parsers import ( + Confidence, + ParserExistsError, + SnifferContentError, +) +from brainhops.io.metadata import FileBasedMetadata + + +class MghRaw: + """ + The raw record of an MGH file: its `nibabel` header (the footer of + MRI parameters included: `nibabel` keeps it in `MGHHeader`) and the + bytes of the trailing tags, which `nibabel` does not read. + + The tags follow the whole volume, so reading them decompresses an + MGZ to its end. A raw record read from a file therefore holds a + `loader` instead, and reads the tags the first time `tags` is used + (only `history` needs them). + """ + + __slots__ = ("header", "_tags", "_loader") + + def __init__( + self, + header: tx.Optional[_mgh.MGHHeader] = None, + tags: tx.Optional[bytes] = b"", + *, + loader: tx.Optional[tx.Callable[[], bytes]] = None, + ) -> None: + """ + Parameters + ---------- + header : nibabel.freesurfer.mghformat.MGHHeader, optional + The header, footer included. By default, an empty header. + tags : bytes or None, optional + The trailing tags, or `None` to read them with `loader`. + loader : callable, optional + A function without arguments that reads the tags, called the + first time they are used. + """ + self.header = _mgh.MGHHeader() if header is None else header + if tags is None and loader is None: + tags = b"" + self._tags = None if tags is None else bytes(tags) + self._loader = None if tags is not None else loader + + @property + def tags(self) -> bytes: + """The raw trailing tags, read on first use when they are lazy.""" + if self._tags is None: + loader, self._loader = self._loader, None + self._tags = bytes(loader() or b"") if loader else b"" + return self._tags + + @tags.setter + def tags(self, value: tx.Optional[bytes]) -> None: + self._tags = bytes(value or b"") + self._loader = None + + @property + def tags_loaded(self) -> bool: + """Whether the tags have been read (or were given).""" + return self._tags is not None + + def __deepcopy__(self, memo: tx.Dict) -> tx.Self: + return type(self)(self.header.copy(), self._tags, loader=self._loader) + + def __getstate__(self) -> tx.Tuple[tx.Any, ...]: + return (self.header, self.tags) + + def __setstate__(self, state: tx.Tuple[tx.Any, ...]) -> None: + self.header, self._tags = state + self._loader = None + + def __eq__(self, other: object) -> bool: + if not isinstance(other, MghRaw): + return NotImplemented + return self.tags == other.tags and bytes( + self.header.binaryblock + ) == bytes(other.header.binaryblock) + + __hash__ = None # type: ignore[assignment] + + def __repr__(self) -> str: + tags = f"{len(self._tags)} bytes" if self._tags is not None else "lazy" + return f"MghRaw(header=..., tags={tags})" + + +@register_format +class MghMetadata( + MetadataParser, + FileBasedMetadata[MghRaw], + on={"format": "mgh"}, + supports=( + "repetition_time", + "echo_time", + "inversion_time", + "flip_angle", + "history", + "data_type", + ), +): + """ + The metadata of an MGH/MGZ file; its raw record (`raw`) is the + `MghRaw` of the file that was read: its `nibabel` header (the footer + of MRI parameters included) and its trailing tags. Geometry is + rewritten from the data model on save. + + `header` and `tags` are the parts of the raw record under their + familiar names. `MghMetadata.load(path)` reads the header and the + footer of a file, and its tags, which follow the voxels, for + `history`. + """ + + EXTENSIONS: tx.ClassVar[tx.Tuple[str, ...]] = (".mgh", ".mgz", ".mgh.gz") + HINTS = ("mgh", "mgz") + + @property + def header(self) -> tx.Optional[_mgh.MGHHeader]: + """The `nibabel` header of the raw record.""" + return None if self.raw is None else self.raw.header + + @property + def tags(self) -> bytes: + """The trailing tags of the raw record.""" + return b"" if self.raw is None else self.raw.tags + + # --- reading the record of a file --------------------------------- + + @classmethod + def sniff_fileobj( + cls, + file: tx.IO, + error: tx.Union[bool, tx.Type[Exception]] = False, + **kwargs: tx.Any, + ) -> float: + """ + Score how confident the class is that an open file holds an MGH + header, gzipped or not. + + Parameters + ---------- + file : file object + A binary stream. + error : bool or type, optional + Raise an error instead of returning 0. + **kwargs + Ignored. + + Returns + ------- + float + The confidence, in `[0, 1]`. + """ + # Not at the top: `brainhops.io.base.mgh` imports this module. + from brainhops.io.base.mgh import is_mgh_stream + + if is_mgh_stream(file): + return Confidence.LIKELY + if error: + raise (SnifferContentError if error is True else error)( + "Content is not a valid MGH/MGZ file" + ) + return Confidence.NO + + @classmethod + def sniff_bytes( + cls, + content: bytes, + error: tx.Union[bool, tx.Type[Exception]] = False, + **kwargs: tx.Any, + ) -> float: + """ + Score how confident the class is that bytes hold an MGH header. + + Parameters + ---------- + content : bytes + The content of a file, gzipped or not. + error : bool or type, optional + Raise an error instead of returning 0. + **kwargs + Ignored. + + Returns + ------- + float + The confidence, in `[0, 1]`. + """ + return cls.sniff_fileobj(BytesIO(content), error=error) + + @classmethod + def from_filename( + cls, filename: path.FilenameLike, **kwargs: tx.Any + ) -> tx.Self: + """ + Read the header and the footer of an MGH file at a path. The tags, + which follow the voxels, are read the first time they are used + (see `MghRaw`). + + Parameters + ---------- + filename : str or path-like + The path, local or remote. + **kwargs + Ignored. + + Returns + ------- + MghMetadata + The metadata of the file, with its `MghRaw` as `raw`. + + Raises + ------ + ParserExistsError + If the path does not exist. + """ + # Not at the top: `brainhops.io.base.mgh` imports this module. + from brainhops.io.base.mgh import read_mgh_raw + + if isinstance(filename, str): + filename = path.Path(filename) + if not path.exists(filename): + raise ParserExistsError(f"No such file: {filename}") + return cls.from_raw(read_mgh_raw(filename)) + + @classmethod + def from_fileobj(cls, file: tx.IO, **kwargs: tx.Any) -> tx.Self: + """ + Read the header, the footer and the tags of an open MGH file, + without its voxels. + + Parameters + ---------- + file : file object + A binary stream, gzipped or not. Its position is restored. + **kwargs + Ignored. + + Returns + ------- + MghMetadata + The metadata of the file, with its `MghRaw` as `raw`. + """ + # Not at the top: `brainhops.io.base.mgh` imports this module. + from brainhops.io.base.mgh import read_mgh_raw + + return cls.from_raw(read_mgh_raw(file)) + + # --- hooks -------------------------------------------------------- + + @classmethod + def _decode_raw( + cls, raw: tx.Optional[MghRaw], *, image: tx.Any = None + ) -> tx.Dict[str, tx.Any]: + if raw is None: + return {} + out: tx.Dict[str, tx.Any] = {} + for name, (slot, factor) in _FOOTER.items(): + stored = float(raw.header[slot]) + if not stored: + continue + # The shortest decimal that is stored as the same number: a + # flip angle of `9.0`, not `9.000000250447817`. + if factor is None: + out[name] = shortest_decimal( + math.degrees(stored), math.radians + ) + else: + out[name] = shortest_decimal( + stored * factor, functools.partial(_divide, factor) + ) + try: + out["data_type"] = raw.header.get_data_dtype() + except Exception: + pass + out["history"] = decode_history(raw.tags) + return out + + def _encode_raw( + self, + raw: MghRaw, + changed: tx.Dict[str, tx.Any], + *, + image: tx.Any = None, + report: ConversionReport, + ) -> MghRaw: + for name, (slot, factor) in _FOOTER.items(): + if name not in changed: + continue + value = changed[name] + if value is None: + stored = 0.0 + elif factor is None: + stored = math.radians(value) + else: + stored = value / factor + raw.header[slot] = stored + if changed.get("data_type") is not None: + # The writer settles it against the data afterwards. + dtype = changed["data_type"] + if dtype in _MGH_DTYPES: + raw.header.set_data_dtype(dtype) + else: + report.approximated["data_type"] = ( + f"MGH cannot store {dtype.name} (it stores uint8, " + f"int16, int32 and float32)" + ) + if "history" in changed: + tags = encode_history(raw.tags, changed["history"]) + if tags is None: + if changed["history"]: + report.lost["history"] = tuple(changed["history"]) + else: + raw.tags = tags + return raw + + +# ---------------------------------------------------------------------- +# PRIVATE +# ---------------------------------------------------------------------- + + +# The voxel types MGH stores. +_MGH_DTYPES = tuple( + np.dtype(t) for t in (np.uint8, np.int16, np.int32, np.float32) +) + + +# Vocabulary field -> (footer slot, factor from the footer unit to the +# vocabulary unit). +_FOOTER = { + "repetition_time": ("tr", 1e-3), + "echo_time": ("te", 1e-3), + "inversion_time": ("ti", 1e-3), + "flip_angle": ("flip_angle", None), # radians -> degrees +} + + +def _divide(factor: float, value: float) -> float: + """A value in the vocabulary unit, in the unit of the footer.""" + return value / factor diff --git a/src/brainhops/io/base/_mgh_tags.py b/src/brainhops/io/base/_mgh_tags.py new file mode 100644 index 00000000..36a58838 --- /dev/null +++ b/src/brainhops/io/base/_mgh_tags.py @@ -0,0 +1,158 @@ +""" +The trailing tags of an MGH/MGZ file, which `nibabel` does not read. + +After the footer, FreeSurfer writes a sequence of tags: a big-endian +`int32` tag id, a length, and the payload. The length is an `int64`, +except for a few legacy ids (`TAG_OLD_MGH_XFORM` has an `int32` length, +and `TAG_OLD_COLORTABLE`, `TAG_OLD_USEREALRAS` and `TAG_OLD_SURF_GEOM` +none). A command line (`TAG_CMDLINE = 3`) is a NUL-terminated string. +""" + +__all__ = ["decode_history", "encode_history", "parse_tags"] + +# stdlib +import struct + +# dependencies +import typing_extensions as tx + +# FreeSurfer tag ids (`utils/tags.h`). +TAG_OLD_COLORTABLE = 1 +TAG_OLD_USEREALRAS = 2 +TAG_CMDLINE = 3 +TAG_OLD_SURF_GEOM = 20 +TAG_OLD_MGH_XFORM = 30 + + +def parse_tags( + tags: bytes, +) -> tx.Optional[tx.List[tx.Tuple[int, bytes]]]: + """ + Split the trailing tags of an MGH file into individual tags. + + Parameters + ---------- + tags : bytes + The bytes that follow the footer. + + Returns + ------- + list of (int, bytes) or None + One `(tag id, chunk)` pair per tag, where `chunk` holds the bytes + of the tag verbatim (id, length and payload), or `None` when the + bytes do not parse as a FreeSurfer tag stream. + """ + out = [] + pos, end = 0, len(tags) + while pos < end: + if end - pos < 4: + return None + start = pos + (tag,) = struct.unpack_from(">i", tags, pos) + pos += 4 + if tag in _NO_LENGTH: + # A legacy tag with no length: only safe when it is the last. + if pos != end: + return None + out.append((tag, tags[start:])) + break + if tag == TAG_OLD_MGH_XFORM: + if end - pos < 4: + return None + (length,) = struct.unpack_from(">i", tags, pos) + pos += 4 + else: + if end - pos < 8: + return None + (length,) = struct.unpack_from(">q", tags, pos) + pos += 8 + if tag <= 0 or length < 0 or pos + length > end: + return None + pos += length + out.append((tag, tags[start:pos])) + return out + + +def decode_history(tags: bytes) -> tx.Optional[tx.Tuple[str, ...]]: + """ + Read the command lines stored in the trailing tags of an MGH file. + + Parameters + ---------- + tags : bytes + The bytes that follow the footer. + + Returns + ------- + tuple of str or None + The command lines, oldest first, or `None` when there is none or + when the tags do not parse. + """ + parsed = parse_tags(tags) + if not parsed: + return None + history = tuple( + _cmdline_payload(chunk) for tag, chunk in parsed if tag == TAG_CMDLINE + ) + return history or None + + +def encode_history( + tags: bytes, history: tx.Optional[tx.Sequence[str]] +) -> tx.Optional[bytes]: + """ + Replace the command lines stored in the trailing tags of an MGH file. + + The other tags are kept in place. The new command lines go where the + first old one was, or at the end when there was none. + + Parameters + ---------- + tags : bytes + The bytes that follow the footer. + history : sequence of str or None + The new command lines. `None` removes them all. + + Returns + ------- + bytes or None + The new trailing tags, or `None` when `tags` does not parse. + """ + parsed = parse_tags(tags) + if parsed is None: + return None + new = b"".join(_cmdline_chunk(c) for c in history or ()) + out, placed = [], False + for tag, chunk in parsed: + if tag == TAG_CMDLINE: + if not placed: + out.append(new) + placed = True + continue + out.append(chunk) + if not placed: + # FreeSurfer writes the command lines before the legacy tags with + # no length, which must stay last. + if out and parsed[-1][0] in _NO_LENGTH: + out.insert(len(out) - 1, new) + else: + out.append(new) + return b"".join(out) + + +# ---------------------------------------------------------------------- +# PRIVATE +# ---------------------------------------------------------------------- + +# Legacy tags with no length field. +_NO_LENGTH = (TAG_OLD_COLORTABLE, TAG_OLD_USEREALRAS, TAG_OLD_SURF_GEOM) + + +def _cmdline_payload(chunk: bytes) -> str: + payload = chunk[12:] + return payload.split(b"\0", 1)[0].decode("utf-8", "replace") + + +def _cmdline_chunk(command: str) -> bytes: + payload = command.encode("utf-8") + b"\0" + return struct.pack(">iq", TAG_CMDLINE, len(payload)) + payload diff --git a/src/brainhops/io/base/_nifti_metadata.py b/src/brainhops/io/base/_nifti_metadata.py new file mode 100644 index 00000000..cc6a6774 --- /dev/null +++ b/src/brainhops/io/base/_nifti_metadata.py @@ -0,0 +1,760 @@ +""" +The metadata of NIfTI files: [`NiftiMetadata`][], shared by `NiftiImage` +and every NIfTI-based transformation (plain fields and affines, FSL +FNIRT, ITK NIfTI fields, NiftyReg, SPM). + +Its raw record (`raw`) is the `nibabel` header. What the vocabulary +covers: + +| Field | Header slots | +|---|---| +| `description` | `descrip` (80 bytes) | +| `display_range` | `cal_min`, `cal_max` | +| `sources` | `aux_file` (one name, 24 bytes) | +| `slice_encoding_direction` | `dim_info` (a voxel axis, no polarity) | +| `phase_encoding_direction` | `dim_info` (a voxel axis, no polarity) | +| `slice_timing` | `slice_code`, `slice_start`, `slice_end`, `slice_duration` | +| `repetition_time` (derived) | `pixdim[4]` (the time step) | +| `intent` (from the axes) | `intent_code` | +| `space` (from the geometry) | `sform_code` / `qform_code` | +| `data_type` | `datatype` (the writer's, see below) | +| `scale_slope`, `scale_intercept` | `scl_slope`, `scl_inter` (the writer's) | + +`repetition_time`, `intent` and `space` are views of geometry that the +writer takes from the data model; a value that disagrees with it is +reported, not written. `repetition_time` is the time step of the image +(the scale of its time axis, `time_step`), which the writer stores as +`pixdim[4]`; only an image whose data model has no time step gets the +field's value there (`check_raw` reports a value the written header does +not hold). `intent` and `space` are checked against the codes the writer +set from the axes and the world space. +NIfTI has no free-form store, so `extra` is unsupported (open question 7 +of the design memo). + +An encoding direction is stored as the voxel axis it is along: its +polarity is dropped (approximated), and one along no voxel axis (an +oblique direction, or one in a world space) is lost. + +The writer keeps, from the raw record of the file that was read, what is +safe to keep: `descrip`, `aux_file`, `cal_*`, `dim_info`, the `slice_*` +fields (when the slice axis kept its length), a non-structural intent +(images only) and the header extensions. Geometry and `xyzt_units` +always come from the data model. The image writer stores the data as +`data_type` when the values of the array are of its kind, and with the +intensity scaling `scale_slope` and `scale_intercept` when the values +fit it, so that a scaled integer file is written back as it was read +(see +[`preferred_storage`][brainhops.datamodel.metadata._dtype.preferred_storage]). +A `dtype=` writer option wins over both. +""" + +__all__ = ["NiftiMetadata"] + +# stdlib +import copy +from io import BytesIO + +# dependencies +import nibabel as nb +import numpy as np +import typing_extensions as tx + +# internals +from brainhops._core.numeric import float32_repr +from brainhops.datamodel.images import Image +from brainhops.datamodel.metadata import ( + ConversionReport, + EncodingDirection, +) +from brainhops.datamodel.metadata._operations import Operation, propagates +from brainhops.datamodel.metadata._terms import AXES +from brainhops.io.base._base import register_format +from brainhops.io.base._metadata_parser import MetadataParser +from brainhops.io.base.parsers import Confidence, SnifferContentError +from brainhops.io.metadata import FileBasedMetadata + + +@register_format +class NiftiMetadata( + MetadataParser, + FileBasedMetadata[nb.Nifti1Header], + on={"format": "nifti"}, + supports=( + "description", + "data_type", + "scale_slope", + "scale_intercept", + "intent", + "space", + "display_range", + "slice_timing", + "slice_encoding_direction", + "phase_encoding_direction", + "sources", + "repetition_time", + ), +): + """ + The metadata of a NIfTI file; its raw record (`raw`) is the `nibabel` + header of the file that was read. Edit it only for what the + vocabulary does not cover; geometry, units and scaling are rewritten + from the data model on save. + + `header` is the raw record under its familiar name. + `NiftiMetadata.load(path)` reads the header of a NIfTI-1 or NIfTI-2 + file, gzipped or not, without its voxels. + """ + + EXTENSIONS: tx.ClassVar[tx.Tuple[str, ...]] = (".nii", ".nii.gz") + HINTS = ("nifti",) + + @property + def header(self) -> tx.Optional[nb.Nifti1Header]: + """The `nibabel` header (the raw record, `raw`).""" + return self.raw + + def check_writable( + self, *, image: tx.Any = None, raw: tx.Any = None + ) -> ConversionReport: + """ + Report what a write of this metadata would lose, without writing + anything. + + By default, the scratch record is the one the writer of `image` + would start from: a copy of `raw` with the shape of the data of + the image, and its time step, whatever the record says. + + Parameters + ---------- + image : object, optional + The image or transformation that would be written. + raw : nibabel.Nifti1Header, optional + The header to encode over, which may be edited. + + Returns + ------- + ConversionReport + What would be lost or approximated. + """ + if raw is None and isinstance(image, Image): + raw = self._raw_or_default() + shape = _data_shape(image) + if shape and shape != _shape(raw): + try: + raw.set_data_shape(shape) + except Exception: + pass + step = time_step(image) + if step is not None and len(_shape(raw)) >= 4: + set_time_step(raw, step) + return super().check_writable(image=image, raw=raw) + + # --- reading the header of a file --------------------------------- + + @classmethod + def sniff_fileobj( + cls, + file: tx.IO, + error: tx.Union[bool, tx.Type[Exception]] = False, + **kwargs: tx.Any, + ) -> float: + """ + Score how confident the class is that an open file holds a NIfTI + header. + + Parameters + ---------- + file : file object + A binary stream, possibly gzipped. + error : bool or type, optional + Raise an error instead of returning 0. + **kwargs + Ignored. + + Returns + ------- + float + The confidence, in `[0, 1]`. + """ + # Not at the top: `brainhops.io.base.nifti` imports this module. + from brainhops.io.base.nifti import is_nifti_stream + + if is_nifti_stream(file): + return Confidence.CERTAIN + if error: + raise (SnifferContentError if error is True else error)( + "Content is not a NIfTI file" + ) + return Confidence.NO + + @classmethod + def sniff_bytes( + cls, + content: bytes, + error: tx.Union[bool, tx.Type[Exception]] = False, + **kwargs: tx.Any, + ) -> float: + """ + Score how confident the class is that bytes hold a NIfTI header. + + Parameters + ---------- + content : bytes + The content of a file, possibly gzipped. + error : bool or type, optional + Raise an error instead of returning 0. + **kwargs + Ignored. + + Returns + ------- + float + The confidence, in `[0, 1]`. + """ + return cls.sniff_fileobj(BytesIO(content), error=error) + + @classmethod + def from_fileobj(cls, file: tx.IO, **kwargs: tx.Any) -> tx.Self: + """ + Read the header of an open NIfTI file, without its voxels. + + Parameters + ---------- + file : file object + A binary stream, possibly gzipped. Its position is restored. + **kwargs + Ignored. + + Returns + ------- + NiftiMetadata + The metadata of the header, with the header as `raw`. + """ + # Not at the top: `brainhops.io.base.nifti` imports this module. + from brainhops.io.base.nifti import _load_nifti_header + + return cls.from_raw(_load_nifti_header(file)) + + # --- hooks -------------------------------------------------------- + + @classmethod + def _decode_raw( + cls, raw: tx.Optional[nb.Nifti1Header], *, image: tx.Any = None + ) -> tx.Dict[str, tx.Any]: + if raw is None: + return {} + h = raw + out: tx.Dict[str, tx.Any] = {} + out["description"] = _bytes_field(h, "descrip") + aux = _bytes_field(h, "aux_file") + out["sources"] = (aux,) if aux else None + + code = int(h["intent_code"]) + out["intent"] = _intent_label(code) if code else None + + scode, qcode = int(h["sform_code"]), int(h["qform_code"]) + out["space"] = _XCODES.get(scode) or _XCODES.get(qcode) + + cal = (float32_repr(h["cal_min"]), float32_repr(h["cal_max"])) + out["display_range"] = cal if any(cal) else None + + try: + out["data_type"] = h.get_data_dtype() + except Exception: + pass + out["scale_slope"], out["scale_intercept"] = _scaling(h, image) + + freq, phase, slice_ = h.get_dim_info() + if phase is not None and phase < len(AXES): + out["phase_encoding_direction"] = AXES[phase] + if slice_ is not None and slice_ < len(AXES): + out["slice_encoding_direction"] = AXES[slice_] + out["slice_timing"] = _decode_slice_timing(h) + + shape = _shape(h) + scale = _time_scale(h) + if len(shape) >= 4 and scale is not None: + step = float32_repr(h["pixdim"][4]) + if step > 0: + out["repetition_time"] = step * scale + return out + + def _encode_raw( + self, + raw: nb.Nifti1Header, + changed: tx.Dict[str, tx.Any], + *, + image: tx.Any = None, + report: ConversionReport, + ) -> nb.Nifti1Header: + h = raw + if "description" in changed: + _encode_bytes( + h, + "descrip", + changed["description"], + _DESCRIP_BYTES, + "description", + report, + ) + if "sources" in changed: + sources = changed["sources"] + first = sources[0] if sources else None + if sources and len(sources) > 1: + report.approximated["sources"] = ( + f"only the first of {len(sources)} kept (aux_file)" + ) + _encode_bytes( + h, "aux_file", first, _AUX_FILE_BYTES, "sources", report + ) + if "display_range" in changed: + lo, hi = changed["display_range"] or (0.0, 0.0) + h["cal_min"], h["cal_max"] = lo, hi + for name, position in ( + ("phase_encoding_direction", 1), + ("slice_encoding_direction", 2), + ): + if name in changed: + _encode_dim_info(h, name, changed[name], position, report) + if "slice_timing" in changed: + _encode_slice_timing(h, changed["slice_timing"], report) + if "repetition_time" in changed and ( + not isinstance(image, Image) or time_step(image) is None + ): + # The time step of an image is the data model's: the writer + # stores it, and `check_raw` reports a value that disagrees. + _encode_repetition_time(h, changed["repetition_time"], report) + if "intent" in changed: + _encode_intent(h, changed["intent"], image, report) + if "space" in changed: + _check_space(h, changed["space"], report) + # `scale_slope` and `scale_intercept` are not encoded here: the + # image writer stores them with the data, when the values fit them + # (see `preferred_storage`), and `nibabel` rewrites `scl_*` from + # the data on save anyway. + if "data_type" in changed and changed["data_type"] is not None: + # The image writer settles it against the data afterwards + # (see `preferred_dtype`). + try: + h.set_data_dtype(changed["data_type"]) + except Exception: + report.lost["data_type"] = changed["data_type"] + return h + + +@propagates(nb.Nifti1Header, Operation) +def _derive_header( + raw: nb.Nifti1Header, + operation: Operation, + *, + name: str, + source: tx.Any, +) -> nb.Nifti1Header: + """The header of derived NIfTI metadata: a copy, whose slice timing + and encoding axes are cleared when the spatial axes move, since they + follow them.""" + raw = copy.deepcopy(raw) + if operation.moves_space: + _clear_slices(raw) + raw.set_dim_info(None, None, None) + return raw + + +# ---------------------------------------------------------------------- +# SHARED WITH THE IMAGE CLASS +# ---------------------------------------------------------------------- + + +def time_step(image: tx.Any) -> tx.Optional[float]: + """ + The time step of an image, in seconds, as its data model gives it. + + It is the spacing that the NIfTI writer stores as `pixdim[4]`: the + scale of the time axis of the image's preferred transformation, from + the voxel space to the world space. A time axis that the + transformation does not map to time -- one that still counts frames, + as the reader leaves a time axis whose spacing is zero, the NIfTI + spelling of a missing repetition time -- gives no time step (see + [`_nifti_geometry`][brainhops.io.base.nifti._nifti_geometry]). The + step is converted from the unit of the time axis of the world space, + and taken to be in seconds when that axis has no time unit. + + Parameters + ---------- + image : Image + The image. + + Returns + ------- + float or None + The time step in seconds, or `None` when the data model gives + none. + """ + # Not at the top: `brainhops.io.base.nifti` imports this module. + from brainhops.io.base.nifti import _geometry_time_step + + try: + transformation = image.transformation + except Exception: + return None + shape = _data_shape(image) + if transformation is None or shape is None: + return None + return _geometry_time_step(transformation, len(shape)) + + +def set_time_step(h: nb.Nifti1Header, seconds: float) -> None: + """ + Store a time step as `pixdim[4]`, in the time unit of the header. + + A header without a time unit is given seconds. + + Parameters + ---------- + h : nibabel.Nifti1Header + The header to edit, in place. + seconds : float + The time step, in seconds. + """ + space, time = h.get_xyzt_units() + if time not in _TIME_UNITS: + h.set_xyzt_units(space, "sec") + time = "sec" + h["pixdim"][4] = float(seconds) / _TIME_UNITS[time] + + +def copy_record( + target: nb.Nifti1Header, + record: tx.Optional[nb.Nifti1Header], + *, + intent: bool = True, +) -> None: + """ + Copy what is safe to keep from the record of the file that was read + onto a header that the writer has just built. + + `descrip`, `aux_file`, `cal_min` and `cal_max` are always kept, and + so are the header extensions, unless the writer added its own. + `dim_info` is kept when its axes still exist, and the `slice_*` slots + are kept when the slice axis kept its length. An intent that does not + retype the axes is kept when the writer set none. The geometry, + `xyzt_units`, the data type and the intensity scaling are never + touched. + + Parameters + ---------- + target : nibabel.Nifti1Header + The header that the writer built, edited in place. + record : nibabel.Nifti1Header or None + The header of the file that was read. Nothing is copied when it + is `None`. + intent : bool, optional + Whether to keep the intent of the record. + """ + if record is None: + return + for slot in ("descrip", "aux_file", "cal_min", "cal_max"): + try: + target[slot] = record[slot] + except (KeyError, ValueError): + pass + + shape, old_shape = _shape(target), _shape(record) + dims = record.get_dim_info() + if all(d is None or d < len(shape) for d in dims): + target.set_dim_info(*dims) + slice_dim = dims[2] + if ( + slice_dim is not None + and slice_dim < len(old_shape) + and shape[slice_dim] == old_shape[slice_dim] + ): + for slot in ( + "slice_code", + "slice_start", + "slice_end", + "slice_duration", + ): + target[slot] = record[slot] + + code = int(record["intent_code"]) + if ( + intent + and code + and code not in _STRUCTURAL_INTENTS + and not int(target["intent_code"]) + ): + for slot in ( + "intent_code", + "intent_name", + "intent_p1", + "intent_p2", + "intent_p3", + ): + target[slot] = record[slot] + + # The extensions are kept unless the writer added its own (NiftyReg + # writes structural ones, which the record holds too). + extensions = getattr(record, "extensions", None) + if extensions and not getattr(target, "extensions", True): + for extension in extensions: + target.extensions.append(extension) + + +# ---------------------------------------------------------------------- +# PRIVATE +# ---------------------------------------------------------------------- + + +# NIfTI xform codes and their names; see `brainhops.io.base.nifti`. +_XCODES = { + 1: "scanner", + 2: "aligned", + 3: "talairach", + 4: "mni", + 5: "template", +} + + +# Intent codes that retype the axes of the data (see `nifti.py`); a +# writer never takes them from a record, only from the data model. +_STRUCTURAL_INTENTS = frozenset( + {1004, 1006, 1008, 1009, 2001, 2002, 2003, 2004, 2005} + | {2006, 2007, 2008, 2009} +) + + +# Seconds per NIfTI time unit. +_TIME_UNITS = {"sec": 1.0, "msec": 1e-3, "usec": 1e-6} + + +_DESCRIP_BYTES = 80 + + +_AUX_FILE_BYTES = 24 + + +# ---------------------------------------------------------------------- +# PRIVATE: DECODING +# ---------------------------------------------------------------------- + + +def _bytes_field(header: nb.Nifti1Header, name: str) -> tx.Optional[str]: + value = np.asarray(header[name]).item() + if isinstance(value, bytes): + value = value.split(b"\0", 1)[0].decode("utf-8", "replace") + value = str(value).strip() + return value or None + + +def _scaling( + header: nb.Nifti1Header, image: tx.Any +) -> tx.Tuple[tx.Optional[float], tx.Optional[float]]: + """ + The intensity scaling of a file, `(slope, intercept)`, or `None` for + each part that is absent. + + A header read on its own holds `scl_slope` and `scl_inter`. The + header of an image loaded by `nibabel` does not (`nibabel` moves the + scaling into the array proxy and resets the header's to NaN), so the + proxy of the image is read then. A slope of 0 or NaN means no + scaling, as the NIfTI standard says, and so does the identity. + """ + slope, intercept = float(header["scl_slope"]), float(header["scl_inter"]) + if not slope or not np.isfinite(slope): + proxy = getattr(getattr(image, "image", None), "dataobj", None) + slope = float(getattr(proxy, "slope", 0.0) or 0.0) + intercept = float(getattr(proxy, "inter", 0.0) or 0.0) + if not slope or not np.isfinite(slope): + return None, None + if not np.isfinite(intercept): + intercept = 0.0 + if slope == 1.0 and intercept == 0.0: + return None, None + return float32_repr(slope), float32_repr(intercept) or None + + +def _time_scale(header: nb.Nifti1Header) -> tx.Optional[float]: + """Seconds per unit of the header's time axis, `None` if unknown.""" + try: + return _TIME_UNITS.get(header.get_xyzt_units()[1]) + except Exception: + return None + + +def _intent_label(code: int) -> tx.Optional[str]: + try: + return str(nb.nifti1.intent_codes.label[code]) + except KeyError: + return None + + +def _shape(header: nb.Nifti1Header) -> tx.Tuple[int, ...]: + return tuple(int(d) for d in header.get_data_shape()) + + +def _decode_slice_timing( + h: nb.Nifti1Header, +) -> tx.Optional[tx.Tuple[float, ...]]: + scale = _time_scale(h) or 1.0 + try: + if not int(h["slice_code"]) or not float(h["slice_duration"]): + return None + times = h.get_slice_times() + except Exception: + return None + if any(t is None for t in times): + # Padding slices have no time: BIDS has no way to say so, and + # the record keeps them. + return None + # Times are multiples of the (single-precision) slice duration: + # recompute them in double precision from the duration as written. + stored = float(h["slice_duration"]) + duration = float32_repr(stored) * scale + return tuple(round(round(float(t) / stored) * duration, 9) for t in times) + + +# ---------------------------------------------------------------------- +# PRIVATE: ENCODING +# ---------------------------------------------------------------------- + + +def _encode_bytes( + h: nb.Nifti1Header, + slot: str, + value: tx.Optional[str], + size: int, + name: str, + report: ConversionReport, +) -> None: + data = (value or "").encode("utf-8") + if len(data) > size: + report.approximated[name] = f"truncated to {size} bytes ({slot})" + data = data[:size] + h[slot] = data + + +def _clear_slices(h: nb.Nifti1Header) -> None: + h["slice_code"] = 0 + h["slice_start"] = 0 + h["slice_end"] = 0 + h["slice_duration"] = 0 + + +def _encode_dim_info( + h: nb.Nifti1Header, + name: str, + value: tx.Optional[EncodingDirection], + position: int, + report: ConversionReport, +) -> None: + dims = list(h.get_dim_info()) + if value is None: + dims[position] = None + else: + bids = value.to_bids() + if bids is None: + # Along no voxel axis: `dim_info` stores an axis. + report.lost[name] = value + return + if bids.endswith("-"): + report.approximated[name] = "polarity dropped (dim_info)" + dims[position] = AXES.index(bids[0]) + h.set_dim_info(*dims) + + +def _encode_slice_timing( + h: nb.Nifti1Header, + value: tx.Optional[tx.Sequence[float]], + report: ConversionReport, +) -> None: + if value is None: + _clear_slices(h) + return + scale = _time_scale(h) or 1.0 + times = [float(t) / scale for t in value] + slice_dim = h.get_dim_info()[2] + try: + if slice_dim is None: + raise ValueError("no slice axis (dim_info)") + h.set_slice_times(times) + except Exception: + # No slice axis, not one time per slice, or not one of the NIfTI + # slice orders (sequential, alternating, their reverses). The + # value replaced the record's, so the record's is cleared too. + _clear_slices(h) + report.lost["slice_timing"] = tuple(value) + + +def _encode_repetition_time( + h: nb.Nifti1Header, value: tx.Optional[float], report: ConversionReport +) -> None: + """Write a repetition time the data model says nothing about (an + image with no physical time axis) as the time step, `pixdim[4]`.""" + if value is None: + return + if len(_shape(h)) < 4: + report.lost["repetition_time"] = value + return + set_time_step(h, value) + + +def _intent_code(name: str) -> tx.Optional[int]: + try: + return int(nb.nifti1.intent_codes.code[name]) + except KeyError: + return None + + +def _encode_intent( + h: nb.Nifti1Header, + value: tx.Optional[str], + image: tx.Any, + report: ConversionReport, +) -> None: + if value is None: + return + current = int(h["intent_code"]) + if current: + if _intent_label(current) != value: + report.approximated["intent"] = ( + f"derived from the axes ({_intent_label(current)})" + ) + return + code = _intent_code(value) + if code is None: + report.lost["intent"] = value + elif code in _STRUCTURAL_INTENTS: + report.approximated["intent"] = ( + "derived from the axes (none): this intent retypes the axes" + ) + else: + h.set_intent(code) + + +def _check_space( + h: nb.Nifti1Header, value: tx.Optional[str], report: ConversionReport +) -> None: + if value is None: + return + scode, qcode = int(h["sform_code"]), int(h["qform_code"]) + written = _XCODES.get(scode) or _XCODES.get(qcode) + if written != value: + report.approximated["space"] = ( + f"derived from the world space name ({written})" + ) + + +def _data_shape(image: tx.Any) -> tx.Optional[tx.Tuple[int, ...]]: + """The shape of the data of an image, without reading the data.""" + data = image.__dict__.get("_data") + if data is None: + nib = getattr(image, "image", None) + if nib is not None and hasattr(nib, "shape"): + return tuple(int(d) for d in nib.shape) + try: + data = image.data + except Exception: + return None + shape = getattr(data, "shape", None) + return None if shape is None else tuple(int(d) for d in shape) diff --git a/src/brainhops/io/base/hdf5.py b/src/brainhops/io/base/hdf5.py index ee607ab8..fdbfc51b 100644 --- a/src/brainhops/io/base/hdf5.py +++ b/src/brainhops/io/base/hdf5.py @@ -15,6 +15,8 @@ `from_h5`. - [`Hdf5ParserWriter`][] does the same for writing, through the format's `to_h5`. +- [`Hdf5MetadataParser`][] is the [`Hdf5Parser`][] of the metadata of + a format: it reads the metadata of a file without its data. - [`DelayedH5Array`][] is a dataset that can still be read after its file was closed, by reopening it on demand. @@ -25,6 +27,7 @@ __all__ = [ "DelayedH5Array", "H5Like", + "Hdf5MetadataParser", "Hdf5Parser", "Hdf5ParserWriter", "delayed_dataset", @@ -47,6 +50,7 @@ from brainhops._core.typing import ArrayProtocol # io +from brainhops.io.base._metadata_parser import MetadataParser from brainhops.io.base.parsers import ( BinaryFileParser, BinaryFileParserWriter, @@ -301,6 +305,21 @@ def to_bytes(self, **kwargs) -> bytes: return buffer.getvalue() +class Hdf5MetadataParser(Hdf5Parser, MetadataParser): + """ + Reads the metadata of a format stored in an HDF5 file, without its + data. + + The metadata class of the format lists it first among its bases, and + implements `sniff_h5(h5file, error=False)` and `from_h5(h5file, + **kwargs)`, as the formats of [`Hdf5Parser`][] do, which routes + paths, streams and bytes to them. `from_h5` reads the raw record + alone, and builds the metadata with `from_raw`; it is also given the + `keep_open` and `load` options of `Hdf5Parser`, which a raw record + read at once does not need. + """ + + # ---------------------------------------------------------------------- # HELPERS # ---------------------------------------------------------------------- diff --git a/src/brainhops/io/base/mgh.py b/src/brainhops/io/base/mgh.py index 2e67ba08..720a102e 100644 --- a/src/brainhops/io/base/mgh.py +++ b/src/brainhops/io/base/mgh.py @@ -35,6 +35,7 @@ __all__ = ["MghParser", "MGH_HEADER_SIZE", "MGH_FOOTER_SIZE"] # stdlib +import functools import gzip import struct from io import BytesIO @@ -52,6 +53,7 @@ from brainhops.datamodel.axes import Axis from brainhops.datamodel.base import DataModelBase from brainhops.datamodel.systems import CoordinateSystem +from brainhops.io.base._mgh_metadata import MghRaw from brainhops.io.base.freesurfer import ( FS_DEFAULT_XRAS, FS_DEFAULT_YRAS, @@ -516,6 +518,68 @@ def sniff_bytes( return cls.sniff_fileobj(BytesIO(data), error=error, **kwargs) +def read_mgh_raw(file: tx.Union[path.FilenameLike, tx.BinaryIO]) -> MghRaw: + """ + Read the raw record of an MGH or MGZ file, without its voxels. + + The header (footer included) is read right away. The trailing tags + follow the voxels, so reading them decompresses an MGZ file to its + end: from a local path, they are read lazily, the first time they are + used (see `MghRaw`); from a stream or a remote path, they are read now. + + Parameters + ---------- + file : path-like or file object + A path, or a binary stream, gzipped or not. The position of a + stream is restored. + + Returns + ------- + MghRaw + The raw record. + """ + if isinstance(file, (str, path.PathLike)): + if _is_local(file): + name = str(path.Path(file)) + with open(name, "rb") as f: + header = _mgh.MGHHeader.from_fileobj(open_compressed(f)) + return MghRaw( + header, + None, + loader=functools.partial(_read_tags_file, name, header), + ) + with path.Path(file).open("rb") as f: + return read_mgh_raw(BytesIO(f.read())) + with preserve_position(file): + stream = open_compressed(file) + start = stream.tell() + header = _mgh.MGHHeader.from_fileobj(stream) + stream.seek(start) + return MghRaw(header, _read_tags(stream, header)) + + +def is_mgh_stream(fileobj: tx.BinaryIO) -> bool: + """ + Whether an open stream, gzipped or not, starts with an MGH header, + judged from its leading fields. + + Parameters + ---------- + fileobj : file object + A binary stream. Its position is restored. + + Returns + ------- + bool + Whether the stream starts with an MGH header. + """ + try: + with preserve_position(fileobj): + return _valid_prefix(_read_prefix(open_compressed(fileobj))) + except Exception: + return False + + def _good_ras(prefix: tx.Optional[tuple]) -> tx.Optional[bool]: """Whether the `goodRASFlag` of the leading header fields is positive, or `None` when they could not be read.""" @@ -562,6 +626,12 @@ def _seekable(fileobj: tx.IO) -> bool: return False +def _read_tags_file(filename: str, header: _mgh.MGHHeader) -> bytes: + """Read the raw bytes after the footer of an MGH or MGZ file.""" + with open(filename, "rb") as f: + return _read_tags(open_compressed(f), header) + + def _read_tags(stream: tx.BinaryIO, header: _mgh.MGHHeader) -> bytes: """Read the raw bytes after the footer of a decompressed stream. diff --git a/src/brainhops/io/base/nifti.py b/src/brainhops/io/base/nifti.py index b3010419..a9d49fb6 100644 --- a/src/brainhops/io/base/nifti.py +++ b/src/brainhops/io/base/nifti.py @@ -41,6 +41,7 @@ declared_axes, embed_affine, ) +from brainhops.io.base._nifti_metadata import _TIME_UNITS from brainhops.io.base._nifti_units import nifti_unit_meters, unit_to_nifti from brainhops.io.base.parsers import ( BinaryFileParserWriter, @@ -844,15 +845,63 @@ def _load_nifti( def _load_nifti_header( - file: path.FilenameLike, + file: tx.Union[path.FilenameLike, tx.BinaryIO], ) -> tx.Union[nb.Nifti1Header, nb.Nifti2Header]: - """Read the header of a NIfTI file at a path, local or remote, - without reading its voxels.""" - if _is_local(file): - return _load_nifti(file).header - with path.Path(file).open("rb") as f: - header_class = _NIFTI_HEADERS[_nifti_version(f)] - return header_class.from_fileobj(open_compressed(f)) + """ + Read the header of a NIfTI file, without reading its voxels. + + The header is read as it is stored, intensity scaling included + (`nibabel` moves the scaling of a loaded image into its array proxy, + and leaves NaN in the header of the image). This is the reader of + `NiftiMetadata`, and of the `like=` templates of the writers. + + Parameters + ---------- + file : path-like or file object + A path, local or remote, or a binary stream, possibly gzipped. + The position of a stream is restored. + + Returns + ------- + nibabel.Nifti1Header or nibabel.Nifti2Header + The header. + """ + if isinstance(file, (str, path.PathLike)): + with path.Path(file).open("rb") as f: + return _read_nifti_header(f) + with preserve_position(file): + return _read_nifti_header(file) + + +def _read_nifti_header( + fileobj: tx.BinaryIO, +) -> tx.Union[nb.Nifti1Header, nb.Nifti2Header]: + """Read the header of an open, possibly gzipped, NIfTI stream.""" + header_class = _NIFTI_HEADERS[_nifti_version(fileobj)] + return header_class.from_fileobj(open_compressed(fileobj)) + + +def is_nifti_stream(fileobj: tx.BinaryIO) -> bool: + """ + Whether an open, possibly gzipped, stream holds a NIfTI-1 or NIfTI-2 + header, judged from its size and magic string. + + Parameters + ---------- + fileobj : file object + A binary stream. Its position is restored. + + Returns + ------- + bool + Whether the stream starts with a NIfTI header. + """ + try: + with preserve_position(fileobj): + head = open_compressed(fileobj).read(_NIFTI_HEADER_SIZES[2]) + except Exception: + return False + return any(_has_nifti_magic(head, version) for version in (1, 2)) def _save_nifti( @@ -1072,6 +1121,34 @@ def _nifti_geometry( return arranged.matrix, others, timed, arranged.layout +def _geometry_time_step( + transformation: Transformation, ndim: int +) -> tx.Optional[float]: + """ + The time step, in seconds, that the writer stores for data of `ndim` + axes placed by `transformation` (`pixdim[4]`), or `None` when it + stores none. + + It is the scale of the time axis (see [`_nifti_geometry`][]), in the + unit of the time axis of the world space, taken to be seconds when + that axis has no time unit. A time axis that is not mapped to time, + whose repetition time is written as missing, gives `None`, and so + does a transformation that has no NIfTI form. + """ + voxel_axes = _declared_axes(getattr(transformation, "input", None), ndim) + try: + _, others, timed, _ = _nifti_geometry(transformation, voxel_axes) + except Exception: + return None + if not timed or not others or others[0] is None: + return None + scale = float(others[0][0]) + if not (np.isfinite(scale) and scale > 0): + return None + _, time = _xyzt_labels(getattr(transformation, "output", None)) + return scale * _TIME_UNITS.get(time, 1.0) + + def _voxel_to_ras_and_others( xform: Transformation, voxel_axes: tx.Optional[tx.List[Axis]] = None ) -> tx.Tuple[np.ndarray, tx.List[tx.Optional[tx.Tuple[float, float]]], bool]: diff --git a/src/brainhops/io/images/freesurfer/__init__.py b/src/brainhops/io/images/freesurfer/__init__.py index 14e4b02a..7e99b5e3 100644 --- a/src/brainhops/io/images/freesurfer/__init__.py +++ b/src/brainhops/io/images/freesurfer/__init__.py @@ -1,6 +1,6 @@ """Readers and writers for FreeSurfer image formats.""" -__all__ = ["mgh", "MghImage"] +__all__ = ["mgh", "MghImage", "MghMetadata"] from . import mgh -from .mgh import MghImage +from .mgh import MghImage, MghMetadata diff --git a/src/brainhops/io/images/freesurfer/mgh/__init__.py b/src/brainhops/io/images/freesurfer/mgh/__init__.py index 60ea6940..138eb408 100644 --- a/src/brainhops/io/images/freesurfer/mgh/__init__.py +++ b/src/brainhops/io/images/freesurfer/mgh/__init__.py @@ -40,6 +40,8 @@ do not survive a conversion to another format. """ -__all__ = ["MghImage"] +__all__ = ["MghImage", "MghMetadata"] + +from brainhops.io.base._mgh_metadata import MghMetadata from ._image import MghImage diff --git a/src/brainhops/io/images/nifti/__init__.py b/src/brainhops/io/images/nifti/__init__.py index 08c67c32..38a417af 100644 --- a/src/brainhops/io/images/nifti/__init__.py +++ b/src/brainhops/io/images/nifti/__init__.py @@ -1,5 +1,7 @@ """Readers and writers for images stored in NIfTI files.""" -__all__ = ["NiftiImage"] +__all__ = ["NiftiImage", "NiftiMetadata"] + +from brainhops.io.base._nifti_metadata import NiftiMetadata from ._image import NiftiImage diff --git a/src/brainhops/io/images/zarr/__init__.py b/src/brainhops/io/images/zarr/__init__.py index f4caca5a..67742048 100644 --- a/src/brainhops/io/images/zarr/__init__.py +++ b/src/brainhops/io/images/zarr/__init__.py @@ -1,7 +1,14 @@ """Readers and writers for Zarr and OME-Zarr images.""" -__all__ = ["ZarrImage", "OmeZarrImage", "OmeImageError"] +__all__ = [ + "ZarrImage", + "ZarrMetadata", + "OmeZarrImage", + "OmeZarrMetadata", + "OmeImageError", +] from ._image import ZarrImage +from ._metadata import OmeZarrMetadata, ZarrMetadata from ._multiscale import OmeZarrImage from ._ome import OmeImageError diff --git a/src/brainhops/io/images/zarr/_metadata.py b/src/brainhops/io/images/zarr/_metadata.py new file mode 100644 index 00000000..0f35651f --- /dev/null +++ b/src/brainhops/io/images/zarr/_metadata.py @@ -0,0 +1,1038 @@ +""" +The metadata of Zarr images: `ZarrMetadata` for a plain array, and +`OmeZarrMetadata` for an OME-Zarr multiscale pyramid. + +**Plain Zarr.** A plain array has no metadata convention, only free-form +attributes. The vocabulary is stored as a BIDS-style sidecar (the keys of +[`to_bids`][brainhops.io.metadata.bids.to_bids]) under +the array attribute `"brainhops"`, and `extra` maps to the other +attributes of the array. Everything but the diffusion fields (which are +not sidecar keys) is supported. The raw record (`raw`) is a `ZarrRaw`: +the array's attributes, and the node they were read from. + +**OME-Zarr.** The raw record is an `OmeZarrRaw`: the typed `abczarr` +multiscale (normalised to 0.6), the `omero` block as JSON, the group +attributes that are not OME metadata, and the group they were read +from. What the vocabulary covers: + +| Field | Record | +|---|---| +| `name` | multiscale `name` | +| `channels` | `omero.channels`: `label`, `color`, `window.start`/`end` | +| `display_range` | `omero.channels[*].window.start`/`end` | +| `extra` | the other group attributes | +| `data_type` (from the data) | the data type of the arrays | + +An omero color (`RRGGBB`) reads as an RGBA string (`RRGGBBFF`). The +display range is read when every channel shares it, and written to every +channel. A window is required for each channel: when nothing gives one, +it is the range of the data type (0 to 1 for floats). OME-Zarr has no +unit for the values, so `data_unit` is unsupported, and the unit of a +channel is dropped (approximated). + +In both, `data_type` is derived: it is the data type of the array that +was read, and a writer stores the array as it is, so a `data_type` that +disagrees with it is reported as approximated (by `check_raw`). + +Both raw records are rebuilt from the store on each read, so each +remembers the node it was read from (`node`): a parser given metadata +read from its own node keeps it as it is. The node is a handle, not +state: a deep copy or a pickle of the record drops it. + +There is one metadata object per pyramid; each level holds a derived copy +(see `OmeZarrImage`). +""" + +__all__ = [ + "OmeZarrMetadata", + "OmeZarrRaw", + "ZarrMetadata", + "ZarrRaw", + "node_attributes", + "write_attributes", +] + +# stdlib +import copy + +# dependencies +import numpy as np +import typing_extensions as tx +from abczarr import ZarrArray, ZarrGroup +from abczarr import open as open_node + +# internals +from brainhops.datamodel.metadata import ( + Channel, + ConversionReport, +) +from brainhops.datamodel.metadata._vocabulary import ( + DisplayVocabulary, + MicroscopyVocabulary, + MRIVocabulary, + ProvenanceVocabulary, + TransformVocabulary, +) +from brainhops.io.base._base import register_format +from brainhops.io.base._metadata_parser import MetadataParser +from brainhops.io.base.parsers import ( + Confidence, + ParserExistsError, + ParserNotImplementedError, + ParserTypeError, + SnifferContentError, +) +from brainhops.io.metadata import FileBasedMetadata +from brainhops.io.metadata._json import ( + decode_object, + encode_changes, + encode_extra, +) + +# locals +from ._ome import looks_like_multiscale, read_multiscale + +# The array attribute that holds the vocabulary of a plain Zarr image. +BRAINHOPS_KEY = "brainhops" + + +# Group attributes that belong to the OME metadata (0.5 and later nest it +# under "ome"; 0.4 writes it at the top level). +OME_KEYS = frozenset( + { + "ome", + "multiscales", + "omero", + "image-label", + "labels", + "plate", + "well", + } +) + + +# ---------------------------------------------------------------------- +# PLAIN ZARR +# ---------------------------------------------------------------------- + + +class ZarrRaw: + """ + The raw record of a plain Zarr array: its attributes, as JSON, and + the node they were read from (`None` for a record built in memory). + + The node is a handle, not state: a deep copy or a pickle of the + record drops it, and it takes no part in `==`. + """ + + __slots__ = ("attrs", "node") + + def __init__( + self, + attrs: tx.Optional[tx.Mapping[str, tx.Any]] = None, + node: tx.Any = None, + ) -> None: + """ + Parameters + ---------- + attrs : mapping, optional + The attributes of the array, as JSON. + node : object, optional + The array they were read from. + """ + self.attrs = dict(attrs or {}) + self.node = node + + def __deepcopy__(self, memo: tx.Dict) -> tx.Self: + return type(self)(copy.deepcopy(self.attrs, memo)) + + def __getstate__(self) -> tx.Dict[str, tx.Any]: + return self.attrs + + def __setstate__(self, state: tx.Dict[str, tx.Any]) -> None: + self.attrs, self.node = state, None + + def __eq__(self, other: object) -> bool: + if not isinstance(other, ZarrRaw): + return NotImplemented + return self.attrs == other.attrs + + __hash__ = None # type: ignore[assignment] + + def __repr__(self) -> str: + return f"ZarrRaw(attrs={sorted(self.attrs)})" + + +# Above the two formats, which derive from it. +class _ZarrMetadataParser(MetadataParser): + """ + The metadata parser of a Zarr store: a store is a directory, read + from its path, never from a stream. A format implements + `sniff_node(node)` and `from_node(node)` on the opened node, as + `ZarrImage` does. + """ + + @classmethod + def sniff_node(cls, node: tx.Any) -> float: + """ + Score how confident the class is that an opened Zarr node is one + of its nodes. + + Parameters + ---------- + node : ZarrArray or ZarrGroup + The opened node. + + Returns + ------- + float + The confidence, in `[0, 1]`. By default, 0. + """ + return Confidence.NO + + @classmethod + def from_node(cls, node: tx.Any, **kwargs: tx.Any) -> tx.Any: + """ + Read the metadata of an opened Zarr node, without its arrays. + + Parameters + ---------- + node : ZarrArray or ZarrGroup + The opened node. + **kwargs + Ignored. + + Returns + ------- + Metadata + The metadata of the node, with its raw record. + + Raises + ------ + ParserNotImplementedError + By default: a format implements this method. + """ + raise ParserNotImplementedError( + f"{cls.__name__} cannot read the metadata of a Zarr node." + ) + + @classmethod + def sniff_file( + cls, + file: tx.Any, + error: tx.Union[bool, tx.Type[Exception]] = False, + **kwargs: tx.Any, + ) -> float: + """ + Score how confident the class is that a store is one of its + stores. + + Parameters + ---------- + file : str or path-like + The location of the store. + error : bool or type, optional + Raise an error instead of returning 0. + **kwargs + Ignored. + + Returns + ------- + float + The confidence, in `[0, 1]`. + """ + score = 0.0 + try: + node = _open(file, "r") + score = cls.sniff_node(node) + except Exception: + pass + if not score and error: + raise (SnifferContentError if error is True else error)( + f"Not a {cls.__name__} store: {file}" + ) + return score + + sniff_filename = sniff_file + + @classmethod + def sniff_fileobj( + cls, + file: tx.Any, + error: tx.Union[bool, tx.Type[Exception]] = False, + **kwargs: tx.Any, + ) -> float: + """ + Refuse an open file: a Zarr store is a directory. + + Parameters + ---------- + file : file object + An open file. + error : bool or type, optional + Raise an error instead of returning 0. + **kwargs + Ignored. + + Returns + ------- + float + 0. + """ + if error: + raise (ParserTypeError if error is True else error)( + "A Zarr store is a directory, not a file object." + ) + return Confidence.NO + + sniff_bytes = sniff_fileobj + + @classmethod + def from_file(cls, file: tx.Any, **kwargs: tx.Any) -> tx.Any: + """ + Read the metadata of a store, without its arrays. + + Parameters + ---------- + file : str or path-like + The location of the store. + **kwargs + Options of `from_node`. + + Returns + ------- + Metadata + The metadata of the store, with its raw record. + + Raises + ------ + ParserTypeError + If `file` is an open file: a Zarr store is a directory. + ParserExistsError + If there is no store at the location. + """ + if hasattr(file, "read"): + return cls.from_fileobj(file, **kwargs) + return cls.from_node(_open(file, "r"), **kwargs) + + @classmethod + def from_fileobj(cls, file: tx.IO, **kwargs: tx.Any) -> tx.Any: + """ + Refuse an open file: a Zarr store is a directory. + + Parameters + ---------- + file : file object + An open file. + **kwargs + Ignored. + + Returns + ------- + Metadata + Never: the method always raises. + + Raises + ------ + ParserTypeError + Always. + """ + raise ParserTypeError( + "A Zarr store is read from a store path, not from a file object." + ) + + +@register_format +class ZarrMetadata( + _ZarrMetadataParser, + FileBasedMetadata[ZarrRaw], + on={"format": "zarr"}, + # Not the diffusion fields: they are not sidecar keys. + supports=( + ProvenanceVocabulary, + MRIVocabulary, + DisplayVocabulary, + MicroscopyVocabulary, + TransformVocabulary, + "data_type", + "extra", + ), +): + """ + The metadata of a plain Zarr array: the vocabulary as a sidecar under + the attribute `"brainhops"`, and `extra` as the other attributes. + Its raw record (`raw`) is a `ZarrRaw`, the attributes of the array + that was read, as JSON (and the array they were read from). + `data_type` is the data type of the array. + + `attributes` is the raw record's attributes under their familiar + name. `ZarrMetadata.load(store)` reads the attributes of an array + without its data, and `metadata.save(store)` writes them back into + the array. + """ + + EXTENSIONS: tx.ClassVar[tx.Tuple[str, ...]] = (".zarr",) + HINTS = ("zarr",) + + @property + def attributes(self) -> tx.Dict[str, tx.Any]: + """The attributes of the raw record, as JSON.""" + return {} if self.raw is None else self.raw.attrs + + # --- reading and writing the attributes of a store -------------- + + @classmethod + def sniff_node(cls, node: tx.Any) -> float: + """ + Score how confident the class is that an opened node is a plain + Zarr array. + + Parameters + ---------- + node : ZarrArray or ZarrGroup + The opened node. + + Returns + ------- + float + The confidence, in `[0, 1]`. + """ + return Confidence.LIKELY if isinstance(node, ZarrArray) else 0.0 + + @classmethod + def from_node(cls, node: tx.Any, **kwargs: tx.Any) -> tx.Self: + """ + Read the attributes of an opened array, without its data. + + Parameters + ---------- + node : ZarrArray + The opened array. + **kwargs + Ignored. + + Returns + ------- + ZarrMetadata + The metadata of the array, with its `ZarrRaw` as `raw`. + """ + return cls.from_raw(ZarrRaw(node_attributes(node), node)) + + def to_file(self, file: tx.Any, **kwargs: tx.Any) -> None: + """ + Write the metadata into the attributes of an existing array. + + The fields are encoded with `to_raw`, over a copy of the + attributes this metadata was read from, and the attributes that + changed are written into the array. + + Parameters + ---------- + file : str or path-like + The location of the array. + **kwargs + Options of `to_raw` (`image=`, `on_loss=`). + + Raises + ------ + ParserExistsError + If there is no store at the location. + MetadataLossError + If something is lost under the `"raise"` policy. + """ + raw = self.to_raw(**kwargs) + node = _open(file, "r+") + write_attributes(node, raw.attrs, self.attributes) + + # --- hooks -------------------------------------------------------- + + @classmethod + def _decode_raw( + cls, raw: tx.Optional[ZarrRaw], *, image: tx.Any = None + ) -> tx.Dict[str, tx.Any]: + dtype = _array_dtype(image) if image is not None else _node_dtype(raw) + out: tx.Dict[str, tx.Any] = {"data_type": dtype} + attrs = None if raw is None else raw.attrs + if not attrs: + return out + block = attrs.get(BRAINHOPS_KEY) + if isinstance(block, tx.Mapping): + # `data_type` is the array's; other keys are not ours. + names = cls.supported_fields - {"extra", "data_type"} + values, _ = decode_object(block, names) + out.update(values) + extra = {k: v for k, v in attrs.items() if k != BRAINHOPS_KEY} + if extra: + out["extra"] = extra + return out + + def _encode_raw( + self, + raw: ZarrRaw, + changed: tx.Dict[str, tx.Any], + *, + image: tx.Any = None, + report: ConversionReport, + ) -> ZarrRaw: + attrs = raw.attrs + block = dict(attrs.get(BRAINHOPS_KEY) or {}) + # `data_type` is the array's: nothing to store. + fields = { + name: value + for name, value in changed.items() + if name not in ("extra", "data_type") + } + encode_changes(block, fields, report=report) + if "extra" in changed: + encode_extra( + attrs, + changed["extra"], + reserved=(BRAINHOPS_KEY,), + report=report, + ) + if block: + attrs[BRAINHOPS_KEY] = block + else: + attrs.pop(BRAINHOPS_KEY, None) + return raw + + +# ---------------------------------------------------------------------- +# OME-ZARR +# ---------------------------------------------------------------------- + + +class OmeZarrRaw: + """ + The raw record of an OME-Zarr pyramid: its typed `abczarr` multiscale + (normalised to 0.6), its `omero` block as JSON, and the group + attributes that are not OME metadata. + """ + + __slots__ = ("multiscale", "omero", "attrs", "node") + + def __init__( + self, + multiscale: tx.Any = None, + omero: tx.Optional[tx.Dict[str, tx.Any]] = None, + attrs: tx.Optional[tx.Dict[str, tx.Any]] = None, + node: tx.Any = None, + ) -> None: + """ + Parameters + ---------- + multiscale : object, optional + The typed `abczarr` multiscale, normalised to OME-NGFF 0.6. + omero : dict, optional + The `omero` block, as JSON. + attrs : dict, optional + The group attributes that are not OME metadata. + node : object, optional + The group they were read from. + """ + self.multiscale = multiscale + self.omero = omero + self.attrs = dict(attrs or {}) + self.node = node + + def __deepcopy__(self, memo: tx.Dict) -> tx.Self: + # The typed multiscale is immutable: it is shared. The node is a + # handle, not state: it is dropped. + return type(self)( + self.multiscale, + copy.deepcopy(self.omero, memo), + copy.deepcopy(self.attrs, memo), + ) + + def __getstate__(self) -> tx.Tuple[tx.Any, ...]: + return (self.multiscale, self.omero, self.attrs) + + def __setstate__(self, state: tx.Tuple[tx.Any, ...]) -> None: + self.multiscale, self.omero, self.attrs = state + self.node = None + + def __eq__(self, other: object) -> bool: + if not isinstance(other, OmeZarrRaw): + return NotImplemented + return ( + _json(self.multiscale) == _json(other.multiscale) + and self.omero == other.omero + and self.attrs == other.attrs + ) + + __hash__ = None # type: ignore[assignment] + + def __repr__(self) -> str: + name = getattr(self.multiscale, "name", None) + return ( + f"OmeZarrRaw(multiscale={name!r}, " + f"omero={self.omero is not None}, attrs={sorted(self.attrs)})" + ) + + @classmethod + def from_attributes( + cls, + multiscale: tx.Any, + attrs: tx.Mapping[str, tx.Any], + node: tx.Any = None, + ) -> tx.Self: + """ + Build the raw record of an OME-Zarr group. + + Parameters + ---------- + multiscale : object + The typed `abczarr` multiscale of the group, normalised to + OME-NGFF 0.6. + attrs : mapping + The attributes of the group, as JSON. + node : object, optional + The group the attributes were read from. + + Returns + ------- + OmeZarrRaw + The raw record. + """ + attrs = dict(attrs) + block = attrs.get("ome") + holder = block if isinstance(block, tx.Mapping) else attrs + omero = holder.get("omero") + return cls( + multiscale, + copy.deepcopy(dict(omero)) + if isinstance(omero, tx.Mapping) + else None, + {k: v for k, v in attrs.items() if k not in OME_KEYS}, + node, + ) + + +@register_format +class OmeZarrMetadata( + _ZarrMetadataParser, + FileBasedMetadata[OmeZarrRaw], + on={"format": "ome-zarr"}, + supports=("name", "channels", "display_range", "extra", "data_type"), +): + """ + The metadata of an OME-Zarr multiscale pyramid; its raw record + (`raw`) is the `OmeZarrRaw` of the pyramid that was read: its typed + multiscale (normalised to OME-NGFF 0.6), its `omero` block and its + other group attributes. The levels and the coordinate + transformations are rewritten from the data model on save. + `data_type` is the data type of the arrays. + + `multiscale` and `omero` are the parts of the raw record under their + familiar names. `OmeZarrMetadata.load(store)` reads the metadata of a + pyramid without its arrays. + """ + + EXTENSIONS: tx.ClassVar[tx.Tuple[str, ...]] = (".zarr", ".ome.zarr") + HINTS = ("ome-zarr", "ome") + + @property + def multiscale(self) -> tx.Any: + """The typed `abczarr` multiscale of the raw record.""" + return None if self.raw is None else self.raw.multiscale + + @property + def omero(self) -> tx.Optional[tx.Dict[str, tx.Any]]: + """The `omero` block of the raw record, as JSON.""" + return None if self.raw is None else self.raw.omero + + # --- reading the attributes of a store --------------------------- + + @classmethod + def sniff_node(cls, node: tx.Any) -> float: + """ + Score how confident the class is that an opened node is an + OME-Zarr pyramid. + + Parameters + ---------- + node : ZarrArray or ZarrGroup + The opened node. + + Returns + ------- + float + The confidence, in `[0, 1]`. + """ + if isinstance(node, ZarrGroup) and looks_like_multiscale(node): + return Confidence.CERTAIN + return 0.0 + + @classmethod + def from_node(cls, node: tx.Any, **kwargs: tx.Any) -> tx.Self: + """ + Read the metadata of an opened pyramid, without its arrays. + + Parameters + ---------- + node : ZarrGroup + The opened group. + **kwargs + Ignored. + + Returns + ------- + OmeZarrMetadata + The metadata of the pyramid, with its `OmeZarrRaw` as `raw`. + """ + multiscale, _ = read_multiscale(node) + return cls.from_raw( + OmeZarrRaw.from_attributes(multiscale, node_attributes(node), node) + ) + + # --- hooks -------------------------------------------------------- + + @classmethod + def _decode_raw( + cls, raw: tx.Optional[OmeZarrRaw], *, image: tx.Any = None + ) -> tx.Dict[str, tx.Any]: + dtype = _array_dtype(image) if image is not None else _node_dtype(raw) + out: tx.Dict[str, tx.Any] = {"data_type": dtype} + if raw is None: + return out + name = getattr(raw.multiscale, "name", None) + out["name"] = name if isinstance(name, str) and name else None + entries = (raw.omero or {}).get("channels") or [] + channels = [] + for entry in entries: + window = entry.get("window") or {} + start, end = window.get("start"), window.get("end") + channels.append( + Channel( + name=entry.get("label"), + color=_color_in(entry.get("color")), + display_range=( + (float(start), float(end)) + if start is not None and end is not None + else None + ), + ) + ) + if channels: + out["channels"] = tuple(channels) + ranges = {c.display_range for c in channels} + if len(ranges) == 1: + out["display_range"] = ranges.pop() + if raw.attrs: + out["extra"] = dict(raw.attrs) + return out + + def _encode_raw( + self, + raw: OmeZarrRaw, + changed: tx.Dict[str, tx.Any], + *, + image: tx.Any = None, + report: ConversionReport, + ) -> OmeZarrRaw: + if "name" in changed and raw.multiscale is not None: + block = raw.multiscale.to_json() + if changed["name"] is None: + block.pop("name", None) + else: + block["name"] = changed["name"] + raw.multiscale = type(raw.multiscale).from_json(block) + if "channels" in changed or "display_range" in changed: + self._encode_omero(raw, changed, image, report) + if "extra" in changed: + encode_extra( + raw.attrs, changed["extra"], reserved=OME_KEYS, report=report + ) + return raw + + def _encode_omero( + self, + raw: OmeZarrRaw, + changed: tx.Dict[str, tx.Any], + image: tx.Any, + report: ConversionReport, + ) -> None: + """Write the channels (`channels`) and the window of every channel + (`display_range`) into `omero`, which needs one per channel.""" + omero = dict(raw.omero or {}) + entries = [dict(e) for e in omero.get("channels") or []] + common = self.display_range or None + if "channels" in changed: + channels = changed["channels"] + if channels is None: + omero.pop("channels", None) + raw.omero = omero if omero else None + return + entries = [ + _channel_entry( + entries[index] if index < len(entries) else {}, + channel, + common, + image, + report, + ) + for index, channel in enumerate(channels) + ] + elif changed["display_range"] is None: + if entries: + report.approximated["display_range"] = ( + "kept: every omero channel needs a window" + ) + return + else: + if not entries: + entries = [ + {"color": "FFFFFF"} for _ in range(_channel_count(image)) + ] + for entry in entries: + entry["window"] = _window( + entry.get("window") or {}, common, image, report + ) + omero["channels"] = entries + raw.omero = omero + + +# ---------------------------------------------------------------------- +# HELPERS +# ---------------------------------------------------------------------- + + +def node_attributes(node: tx.Any) -> tx.Dict[str, tx.Any]: + """ + Read the attributes of a Zarr node, as plain JSON. + + Parameters + ---------- + node : ZarrArray or ZarrGroup + The node. + + Returns + ------- + dict + Its attributes, or an empty dict when they cannot be read. + """ + try: + attrs = node.attrs + return {key: attrs[key] for key in attrs} + except Exception: + return {} + + +def write_attributes( + node: tx.Any, + attrs: tx.Mapping[str, tx.Any], + before: tx.Optional[tx.Mapping[str, tx.Any]] = None, +) -> None: + """ + Write attributes onto a Zarr node. + + Only the keys whose value changed are written, and the keys of + `before` (the raw record that was read) that are no longer in `attrs` + are removed. + + Parameters + ---------- + node : ZarrArray or ZarrGroup + The node, open for writing. + attrs : mapping + The attributes to write. + before : mapping, optional + The attributes the record was read with. + """ + current = node_attributes(node) + for key in before or {}: + if key not in attrs and key in current: + del node.attrs[key] + for key, value in attrs.items(): + if current.get(key) != value: + node.attrs[key] = value + + +# ---------------------------------------------------------------------- +# PRIVATE +# ---------------------------------------------------------------------- + + +def _open(location: tx.Any, mode: str) -> tx.Any: + """Open the node of a store, or raise if there is none.""" + node = open_node(location, mode) + if node is None: + raise ParserExistsError(f"No Zarr store at {location}") + return node + + +def _node_dtype(raw: tx.Any) -> tx.Optional[np.dtype]: + """The data type of the array a raw record was read from (of the + first level of a pyramid), without reading its data.""" + node = getattr(raw, "node", None) + if node is None: + return None + multiscale = getattr(raw, "multiscale", None) + try: + if multiscale is not None: + node = node[str(list(multiscale.datasets)[0].path)] + dtype = getattr(node, "dtype", None) + return None if dtype is None else np.dtype(dtype) + except Exception: + return None + + +def _array_dtype(image: tx.Any) -> tx.Optional[np.dtype]: + """The data type of the array of an image (of the first level of a + pyramid), read from its node when it has one (without reading the + data), `None` when there is none.""" + if image is None: + return None + objects = [image] + try: + images = getattr(image, "images", None) + if images: + objects.append(images[0]) + except Exception: + pass + for obj in objects: + dtype = getattr(getattr(obj, "node", None), "dtype", None) + if dtype is not None: + return np.dtype(dtype) + for obj in objects: + if getattr(obj, "node", None) is not None: + continue + try: + dtype = getattr(getattr(obj, "data", None), "dtype", None) + except Exception: + dtype = None + if dtype is not None: + return np.dtype(dtype) + return None + + +def _json(obj: tx.Any) -> tx.Any: + return None if obj is None else obj.to_json() + + +def _color_in(color: tx.Any) -> tx.Optional[str]: + """An omero color (`RRGGBB`) as an RGBA hex string.""" + if not isinstance(color, str): + return None + color = color.lstrip("#").upper() + if len(color) == 6: + return color + "FF" + return color or None + + +def _color_out( + color: tx.Optional[str], default: str, report: ConversionReport +) -> str: + """An RGBA hex string as an omero color (`RRGGBB`).""" + if not color: + return default + color = color.lstrip("#").upper() + if len(color) == 8: + if color[6:] != "FF": + report.approximated["channels"] = "alpha of the colors dropped" + color = color[:6] + return color + + +def _channel_entry( + base: tx.Mapping[str, tx.Any], + channel: Channel, + common: tx.Optional[tx.Tuple[float, float]], + image: tx.Any, + report: ConversionReport, +) -> tx.Dict[str, tx.Any]: + """The omero entry of a channel, over the entry it had (`base`): its + label, its color, and its window (its own display range, or the + common one).""" + entry = dict(base) + if channel.name is None: + entry.pop("label", None) + else: + entry["label"] = channel.name + entry["color"] = _color_out( + channel.color, base.get("color", "FFFFFF"), report + ) + entry["window"] = _window( + base.get("window") or {}, + channel.display_range or common, + image, + report, + ) + if channel.unit is not None: + report.approximated["channels"] = ( + "channel units dropped (omero has no unit)" + ) + return entry + + +def _channel_count(image: tx.Any) -> int: + """The size of the channel axis of a pyramid (1 without one).""" + try: + shape = image.images[0].data.shape + axes = image._write_axes(len(shape)) + for axis, size in zip(axes, shape): + if getattr(axis, "type", None) == "channel": + return int(size) + except Exception: + pass + return 1 + + +def _window( + base: tx.Mapping[str, tx.Any], + display_range: tx.Optional[tx.Tuple[float, float]], + image: tx.Any, + report: ConversionReport, +) -> tx.Dict[str, float]: + if base: + window = dict(base) + else: + # A display range gives the window: only invented without one. + invented = report if display_range is None else None + window = _default_window(image, invented) + if display_range is not None: + start, end = (float(v) for v in display_range) + window["start"], window["end"] = start, end + window["min"] = min(float(window.get("min", start)), start) + window["max"] = max(float(window.get("max", end)), end) + return window + + +def _default_window( + image: tx.Any, report: tx.Optional[ConversionReport] = None +) -> tx.Dict[str, float]: + """ + The display window of a channel nothing says anything about. + + OME-Zarr requires one. The window (`start`, `end`) is the range of + the values of the smallest level (cheap to read), and the allowed + range (`min`, `max`) that of the data type (of the values, for + floats); with no data, both are the range of the data type (0..1 + for floats). It is reported as approximated in `report`, under + `"channels"`: nothing in the metadata gave it. + """ + dtype = data = None + try: + dtype = np.dtype(image.images[0].data.dtype) + data = np.asarray(image.images[-1].data) + except Exception: + pass + if dtype is not None and dtype.kind in "iub": + info = np.iinfo(np.uint8 if dtype.kind == "b" else dtype) + lo, hi = float(info.min), float(info.max) + else: + lo, hi = 0.0, 1.0 + start, end, where = lo, hi, "the range of the data type" + if data is not None and data.size: + values = data[np.isfinite(data)] if data.dtype.kind == "f" else data + if values.size: + start, end = float(values.min()), float(values.max()) + where = "the range of the values of the smallest level" + if dtype is None or dtype.kind not in "iub": + lo, hi = start, end + if report is not None: + report.approximated["channels"] = ( + f"display window not given, written as {where} " + f"({start:g}..{end:g})" + ) + return {"min": lo, "max": hi, "start": start, "end": end} diff --git a/src/brainhops/io/metadata/__init__.py b/src/brainhops/io/metadata/__init__.py new file mode 100644 index 00000000..9d2d040e --- /dev/null +++ b/src/brainhops/io/metadata/__init__.py @@ -0,0 +1,17 @@ +"""The metadata of file formats, and codecs between format-agnostic +metadata and files that only hold metadata, such as BIDS JSON sidecars. + +The format-agnostic model itself lives in +[`brainhops.datamodel.metadata`][]. This package holds the base of the +metadata of every file format, +[`FileBasedMetadata`][brainhops.io.metadata.FileBasedMetadata], as +[`brainhops.io.images`][] holds `FileBasedImage`, and +[`OpaqueMetadata`][brainhops.io.metadata.OpaqueMetadata], for the formats +that store none. The metadata class of each format lives next to that +format's parser. +""" + +__all__ = ["FileBasedMetadata", "OpaqueMetadata", "from_bids", "to_bids"] + +from ._base import FileBasedMetadata, OpaqueMetadata +from .bids import from_bids, to_bids diff --git a/src/brainhops/io/metadata/_base.py b/src/brainhops/io/metadata/_base.py new file mode 100644 index 00000000..dd2c8ef4 --- /dev/null +++ b/src/brainhops/io/metadata/_base.py @@ -0,0 +1,722 @@ +""" +`FileBasedMetadata`: the metadata of a file format, its hooks, and the +dispatcher among the formats whose files hold metadata. + +It lives in `brainhops.io`, next to `FileBasedImage`, because it derives +from the dispatcher of the formats; the format-agnostic `Metadata` it +extends lives in `brainhops.datamodel.metadata`, which never imports it. +""" + +__all__ = ["FileBasedMetadata", "OpaqueMetadata"] + +# stdlib +import copy +import math + +# externals +import typing_extensions as tx +from bagof.magic import HideIfDefault, NoEq, NoRepr + +# internals +from brainhops._core.compare import differs +from brainhops.datamodel.metadata._base import ( + FIELDS, + Metadata, + _convert_from, + _format_name, + _History, +) +from brainhops.datamodel.metadata._operations import ( + Operation, + propagate_raw, +) +from brainhops.datamodel.metadata._report import ( + ConversionReport, + OnLoss, + apply_loss_policy, + short, +) +from brainhops.datamodel.metadata._sentinel import UNSUPPORTED +from brainhops.io.base._base import FormatDispatcher, format_registry +from brainhops.io.base.parsers import ParserNotImplementedError + +RawT = tx.TypeVar("RawT") +"""The type of the raw record of a format (see `FileBasedMetadata`).""" + + +@format_registry +class FileBasedMetadata(FormatDispatcher, Metadata, tx.Generic[RawT]): + """ + The metadata of a file format: the common vocabulary, decoded from and + encoded into the format's own raw record. + + This is the base of every `Metadata`, as `FileBasedImage` is the + base of the image class of every format. A format declares what it + can store with `supports=`, declares the type of its raw record as + the type argument of its base (`FileBasedMetadata[nb.Nifti1Header]`; + `FileBasedMetadata[None]` for a format without one), and decodes and + encodes the record with the hooks described in the format author's + guide (`docs/dev/metadata-formats.md`). The record itself and the + read-time snapshot are fields of + [`Metadata`][brainhops.datamodel.metadata.Metadata], so that generic + metadata carries them through a conversion; reading and writing them + is what this class adds. + + A new, empty record is the type called without arguments, so the + type of a record must build one that way. + + The class is also the dispatcher of the formats whose files hold + metadata: `FileBasedMetadata.load(path)` picks the registered format + that best matches the file. The class of such a format lists its + parser (a + [`MetadataParser`][brainhops.io.base._metadata_parser.MetadataParser]) + first among its bases, and registers with + [`register_format`][brainhops.io.base.register_format]. The registry + is separate from that of + [`FileBasedObject`][brainhops.io.base.FileBasedObject], which this + class is not, so that `brainhops.io.load` never returns metadata + where an image or a transformation was asked for. + """ + + # Declared again to hide it from `repr` on the class of a format, + # whose name says the format: `HideIfDefault` is bound again on each + # class, against the value the class pins (`on={"format": ...}`), so + # it hides that value. Generic `Metadata` keeps showing its format. + format: tx.Annotated[ + str, + tx.Doc( + "The format this metadata belongs to; selects the subclass. " + "Hidden from `repr` when it is the format of the class." + ), + HideIfDefault(), + ] = "generic" + + raw: tx.Annotated[ + tx.Optional[RawT], + tx.Doc( + """ + The raw record of the file the metadata was read from, of the + type the format declares, or `None` for metadata built in + memory. Edit it only for what the vocabulary does not cover: + on write, a field left untouched keeps the value of the + record, and a field that was set wins over it. + """ + ), + NoRepr(), + NoEq(), + ] = None + + # No record, until a format declares the type of its own, as the + # type argument of this class (which the metaclass reads): a format + # never keeps the record of another format (see `_accepts_raw` in + # `brainhops.datamodel.metadata._base`). + _raw_class: tx.ClassVar[tx.Optional[type]] = type(None) + + # --- reading ------------------------------------------------------ + + @classmethod + def load(cls, file: tx.Any, **kwargs: tx.Any) -> "Metadata": + """ + Read the metadata of a file, without reading its data. + + On `FileBasedMetadata` itself, the dispatcher, the format is found + among the registered formats whose files hold metadata (NIfTI, + MGH, plain Zarr and OME-Zarr, x5, ITK `.h5`, and BIDS JSON + sidecars), by the name of the file and by its content, as + `brainhops.io.load` finds the format of an image; `hint=` + restricts the candidates. On the class of a format, the file is + read as a file of that format, by its parser (`MetadataParser`), + which comes first among its bases; the class of a format whose + files hold no metadata (FLIRT, ITK `.tfm` and `.mat`) refuses. + Only the raw record is read: a NIfTI header, the footer and the + tags of an MGH file, the attributes of a Zarr node, the JSON of + an x5 node. + + Parameters + ---------- + file : str, path-like, file object or bytes + The file, or the Zarr store. + **kwargs + Options of the reader of the format, and, on the dispatcher, + `hint=` (a format name such as `"nifti"`, or several). + + Returns + ------- + Metadata + The metadata of the file, with its raw record: the metadata + of its format, or generic metadata for a BIDS sidecar. + Convert it with `to(Metadata)` for generic metadata (which + keeps the record). + + Raises + ------ + ParserContentError + On the dispatcher, if no registered format reads the file. + ParserNotImplementedError + On the class of a format whose files hold no metadata. + ParserExistsError + If the file does not exist. + + Examples + -------- + ```python + meta = FileBasedMetadata.load("sub-01_bold.nii.gz") + meta.repetition_time # read from the header alone + FileBasedMetadata.load("sub-01_bold.json").extra["TaskName"] + FileBasedMetadata.load("scan.mgz", hint="mgh") + NiftiMetadata.load("sub-01_bold.nii.gz") # as a NIfTI file + ``` + """ + return super().load(file, **kwargs) + + @classmethod + def from_raw( + cls, raw: tx.Any, *, image: tx.Any = None, **values: tx.Any + ) -> tx.Self: + """ + Build the metadata of a raw record that was just read. + + The raw record is decoded into the common fields by the format's + `_decode_raw` hook. The decoded values are kept as the read-time + snapshot, so that a field the user leaves untouched keeps the + value of the record when the record is written back. + + Parameters + ---------- + raw : object + The raw record, of the type the format declares for `raw`. + image : object, optional + The image or transformation the record was read with, for the + fields that the format decodes from the data model. + **values + Fields to set over the decoded values. They count as changes + on write. + + Returns + ------- + FileBasedMetadata + The metadata, with `raw` set to the record. + + Raises + ------ + TypeError + If `_decode_raw` returned a value for a field that this class does + not support. The raw record belongs to the format, so this + error never comes from the data: it reveals a format class + whose `_decode_raw` disagrees with its `supports=` declaration, + and which would otherwise drop the value without a report. + """ + decoded = cls._checked_decode_raw(raw, image) + obj = cls(raw=raw, **decoded) + # Snapshot the *converted* values, so that a decoded list held as + # a tuple does not count as a change. + obj._snapshot = { + key: copy.deepcopy(getattr(obj, key)) for key in decoded + } + for key, value in values.items(): + setattr(obj, key, value) + return obj + + def update_from_raw(self, raw: tx.Any, *, image: tx.Any = None) -> tx.Self: + """ + Build the metadata of another raw record, keeping the changes made + to this metadata. + + The new record is decoded as `from_raw` decodes it. Every field + that changed in this object since it was read is then set over the + decoded values, and counts as a change. For an object built in + memory, every field that is set counts as changed. The keys of + `extra` are merged one by one. This is how an image that is given + a new raw record (`replace(image, header=...)`) keeps the metadata + edits it carried. + + Parameters + ---------- + raw : object + The new raw record. + image : object, optional + The image or transformation the record belongs to. + + Returns + ------- + FileBasedMetadata + A new metadata object of the same class. + """ + changed = { + key: value + for key, value in self._changed_fields().items() + if key in type(self).supported_fields + } + extra = changed.pop("extra", None) + obj = type(self).from_raw(raw, image=image) + for key, value in changed.items(): + setattr(obj, key, value) + if extra: + obj.extra = _apply_diff(obj.extra or {}, extra) + return obj + + # --- the change-detecting write ----------------------------------- + + def _changed_fields(self) -> tx.Dict[str, tx.Any]: + """ + The common fields that differ from the read-time snapshot. + + An object read from a file and left untouched has none. An object + built in memory has no snapshot, so every field that is not + `None` is a change. A field set to `None` after a read is a change + to `None` (the record's slot is cleared on write). `extra` is + compared key by key: its entry is the per-key diff, where `None` + removes a key. + """ + snapshot = self._snapshot + changed: tx.Dict[str, tx.Any] = {} + for name in FIELDS: + value = getattr(self, name, None) + if value is UNSUPPORTED: + continue + before = snapshot.get(name) + if name == "extra": + diff = _extra_diff(before, value) + if diff: + changed["extra"] = diff + elif differs(value, before): + changed[name] = value + return changed + + def to_raw( + self, + *, + image: tx.Any = None, + on_loss: tx.Optional[OnLoss] = None, + ) -> tx.Any: + """ + Encode the common fields into a new raw record, and return it. + + This is the counterpart of `from_raw`: the record is a copy of + `raw` (or a default record, for metadata built in memory), with + the fields that changed since the read encoded over it. It is what + a metadata writer writes (see `ZarrMetadata.to_file`). A writer + that builds its own fresh record calls `update_raw` instead. + + Parameters + ---------- + image : object, optional + The image or transformation the metadata belongs to. + on_loss : {"ignore", "warn", "raise"} or ConversionReport, optional + What to do with the losses. By default, the policy in effect. + + Returns + ------- + object + The encoded record. `raw` itself is left unchanged. + + Raises + ------ + MetadataLossError + If something is lost under the `"raise"` policy. + """ + return self.update_raw(None, image=image, on_loss=on_loss) + + def update_raw( + self, + raw: tx.Any = None, + *, + image: tx.Any = None, + on_loss: tx.Optional[OnLoss] = None, + force: tx.Collection[str] = (), + ) -> tx.Any: + """ + Encode the common fields over a raw record, and return the record. + + Only the fields that changed since the read are encoded, so that a + field the user did not touch keeps the value that the record holds. + The fields named in `force` are encoded whether they changed or + not: a writer uses `force` for a keyword argument that must win + over the record, such as the `tr=` option of the MGH writer, and a + forced `None` clears the slot. + + A field that the format does not support, but that was assigned + after construction, is reported as lost, as are the + value-dependent losses found by `_encode_raw`. What the writer + then changes in the record (a slot that the data model owns, such + as the time step of a NIfTI image) is found by `check_raw`, which + the writer calls once the record is finished. + + Parameters + ---------- + raw : object, optional + The record to encode over. A writer passes its own fresh + record, already filled with what it keeps from `self.raw`. + By default, a copy of `self.raw` is used, or a default record + when there is none. + image : object, optional + The image or transformation being written. + on_loss : {"ignore", "warn", "raise"} or ConversionReport, optional + What to do with the losses. By default, the policy in effect + (see + [`metadata_loss_policy`][brainhops.datamodel.metadata.metadata_loss_policy]). + A writer passes the report of its whole write, which it then + acts on once. + force : collection of str, optional + The fields to encode even when they did not change. + + Returns + ------- + object + The encoded record. + + Raises + ------ + MetadataLossError + If something is lost under the `"raise"` policy. + """ + sink = isinstance(on_loss, ConversionReport) + if sink: + report = on_loss + else: + report = ConversionReport(source=self.format, target=self.format) + unsupported = type(self).unsupported_fields + for name in unsupported: + value = getattr(self, name, None) + if value is not None and value is not UNSUPPORTED: + report.lost[name] = value + if raw is None: + raw = self._raw_or_default() + changed = { + key: value + for key, value in self._changed_fields().items() + if key not in unsupported + } + for name in force: + if name in FIELDS and name != "extra": + if name not in unsupported: + changed[name] = getattr(self, name) + raw = self._encode_raw(raw, changed, image=image, report=report) + if not sink: + apply_loss_policy(report, on_loss, stacklevel=2) + return raw + + def check_raw( + self, + raw: tx.Any, + *, + image: tx.Any = None, + on_loss: tx.Optional[OnLoss] = None, + ) -> ConversionReport: + """ + Check that a finished record holds the fields that changed. + + A writer encodes the fields with `update_raw`, then may set slots + of the record from the data model, whatever the metadata says: the + time step of a NIfTI image, the data type of the stored values. + Once the record is finished, it calls this method, which decodes + the record as a reader would (`from_raw`) and reports as + approximated every changed field whose value the record does not + hold. A field already reported (lost or approximated), an + unsupported field and `extra` are not checked again. + + Parameters + ---------- + raw : object + The finished record. + image : object, optional + The image or transformation being written, as a reader would + be given it. + on_loss : {"ignore", "warn", "raise"} or ConversionReport, optional + What to do with the losses. By default, the policy in effect. + A writer passes the report of its whole write. + + Returns + ------- + ConversionReport + The report, `on_loss` itself when it is one. + + Raises + ------ + MetadataLossError + If something is approximated under the `"raise"` policy. + """ + sink = isinstance(on_loss, ConversionReport) + if sink: + report = on_loss + else: + report = ConversionReport(source=self.format, target=self.format) + decoded = type(self).from_raw(raw, image=image) + unsupported = type(self).unsupported_fields + for name, value in self._changed_fields().items(): + if ( + name == "extra" + or name in unsupported + or name in report.lost + or name in report.approximated + ): + continue + held = getattr(decoded, name) + if not _agrees(held, value): + report.approximated[name] = f"the record holds {short(held)}" + if not sink: + apply_loss_policy(report, on_loss, stacklevel=2) + return report + + @classmethod + def writable( + cls, metadata: Metadata + ) -> tx.Tuple["FileBasedMetadata", ConversionReport]: + """ + Prepare metadata for a writer of this format. + + Metadata that is already of this class is returned as it is. + Metadata of another class is converted into this class, and the + losses of the conversion seed the report. The loss policy is not + applied here: the writer encodes the fields into the same report + (`update_raw(..., on_loss=report)`) and applies the policy once, so + that one write gives one report and at most one warning. + + Parameters + ---------- + metadata : Metadata + The metadata of the object being written. + + Returns + ------- + metadata : FileBasedMetadata + The metadata to write, of this class. + report : ConversionReport + The report of the write so far. + """ + if isinstance(metadata, cls): + target = _format_name(cls) + return metadata, ConversionReport( + source=metadata.format, target=target + ) + return _convert_from(cls, metadata) + + def check_writable( + self, *, image: tx.Any = None, raw: tx.Any = None + ) -> ConversionReport: + """ + Report what a write of this metadata would lose, without writing + anything. + + The class declarations only give a lower bound of what is lost. + This method runs the encoder on a scratch record, so that the + losses that depend on the values, such as an over-long description + or an irregular slice timing, are included as well. A format whose + writer starts from another record than a copy of `raw` overrides + this method, builds that record, and calls `super()` with it. + + Parameters + ---------- + image : object, optional + The image or transformation that would be written, for the + fields that depend on the data model. + raw : object, optional + The record to encode over, which may be edited. By default, a + copy of `raw`, or a new, empty record when there is none. + + Returns + ------- + ConversionReport + What would be lost or approximated. + """ + report = ConversionReport(source=self.format, target=self.format) + raw = self.update_raw(raw, image=image, on_loss=report) + self.check_raw(raw, image=image, on_loss=report) + return report + + # --- per-format hooks --------------------------------------------- + + @classmethod + def _decode_raw(cls, raw: tx.Any, *, image: tx.Any = None) -> tx.Dict: + """ + Decode a raw record into common fields. + + A format overrides this hook. The default decodes nothing. + + Parameters + ---------- + raw : object + The raw record. + image : object, optional + The image or transformation the record was read with. + + Returns + ------- + dict + Field name to decoded value. A value that is `None` or + `UNSUPPORTED` is ignored. + """ + return {} + + def _encode_raw( + self, + raw: tx.Any, + changed: tx.Dict[str, tx.Any], + *, + image: tx.Any = None, + report: ConversionReport, + ) -> tx.Any: + """ + Encode changed common fields into a raw record. + + A format overrides this hook. The default leaves the record + unchanged. + + Parameters + ---------- + raw : object + The record to encode into, which may be edited in place. + changed : dict + Field name to new value, for the fields to encode. A `None` + value clears the slot of the field. + image : object, optional + The image or transformation being written. + report : ConversionReport + The report to fill with what cannot be stored exactly. + + Returns + ------- + object + The encoded record. + """ + return raw + + # --- internals ---------------------------------------------------- + + @classmethod + def _checked_decode_raw( + cls, raw: tx.Any, image: tx.Any + ) -> tx.Dict[str, tx.Any]: + """`_decode_raw`, without its absent values, and checked against the + declarations of the class (see `from_raw`).""" + decoded: tx.Dict[str, tx.Any] = {} + for key, value in cls._decode_raw(raw, image=image).items(): + if value is None or value is UNSUPPORTED: + continue + if key not in cls.supported_fields: + raise TypeError( + f"{cls.__name__}._decode_raw returned {key}={value!r}, " + f"but {cls.__name__} does not support {key!r}." + ) + decoded[key] = value + return decoded + + def _raw_or_default(self) -> tx.Any: + """A copy of `raw`, or a new, empty record when there is none.""" + if self.raw is not None: + return copy.deepcopy(self.raw) + cls = type(self)._raw_class + return None if cls is None else cls() + + def _derive_values( + self, + *, + operation: tx.Optional[Operation], + history: _History, + ) -> tx.Dict[str, tx.Any]: + # A copy of the snapshot is kept, so that a field the operation + # cleared is cleared in the record on write. The record is + # propagated by the handler of its type, which scrubs what else + # the operation invalidates, on a copy (see `propagate_raw`). + values = super()._derive_values(operation=operation, history=history) + values["snapshot"] = copy.copy(self._snapshot) + values["raw"] = propagate_raw(self.raw, operation, source=self) + return values + + +class OpaqueMetadata( + FileBasedMetadata[None], on={"format": "opaque"}, supports=() +): + """ + The metadata of a format that stores none, not even in memory: every + field is unsupported, and there is no raw record. + + Base class of the formats that store a bare matrix and are given + nothing else (matrix text, ITK `.tfm`/`.mat`). A read-then-save + loses nothing, because nothing is there. A format that keeps + anything, if only in memory (FLIRT `moving`/`fixed`), is a + `FileBasedMetadata` with a `supports=` list instead. Its `raw` is + always `None`. + """ + + @classmethod + def load(cls, file: tx.Any, **kwargs: tx.Any) -> "Metadata": + """ + Refuse to read the metadata of a file: the files of the format + hold none. + + Parameters + ---------- + file : str, path-like or file object + The file. + **kwargs + Ignored. + + Returns + ------- + Metadata + Never: the method always raises. + + Raises + ------ + ParserNotImplementedError + Always. + """ + raise ParserNotImplementedError( + f"{cls.__name__} stores no metadata that can be read on its own." + ) + + +# ---------------------------------------------------------------------- +# PRIVATE +# ---------------------------------------------------------------------- + + +def _extra_diff( + before: tx.Optional[tx.Mapping], after: tx.Any +) -> tx.Dict[str, tx.Any]: + before = dict(before or {}) + after = dict(after or {}) + diff = { + key: value + for key, value in after.items() + if key not in before or differs(value, before[key]) + } + for key in before: + if key not in after: + diff[key] = None + return diff + + +def _apply_diff( + mapping: tx.Mapping[str, tx.Any], diff: tx.Mapping[str, tx.Any] +) -> tx.Dict[str, tx.Any]: + """A copy of `mapping` with an `extra` diff applied (`None` removes + a key).""" + out = dict(mapping) + for key, value in diff.items(): + if value is None: + out.pop(key, None) + else: + out[key] = value + return out + + +def _agrees(value: tx.Any, given: tx.Any) -> bool: + """Whether a value agrees with what a record holds: numbers, and + sequences of numbers element by element, within single-precision + rounding.""" + if _is_number(value) and _is_number(given): + return math.isclose(value, given, rel_tol=1e-6, abs_tol=1e-9) + if ( + isinstance(value, (tuple, list)) + and isinstance(given, (tuple, list)) + and len(value) == len(given) + and all(_is_number(v) for v in (*value, *given)) + ): + return all(_agrees(v, g) for v, g in zip(value, given)) + return not differs(value, given) + + +def _is_number(value: tx.Any) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) diff --git a/src/brainhops/io/metadata/_json.py b/src/brainhops/io/metadata/_json.py new file mode 100644 index 00000000..078966b1 --- /dev/null +++ b/src/brainhops/io/metadata/_json.py @@ -0,0 +1,355 @@ +""" +The vocabulary as a flat JSON object: the codec that BIDS sidecars +(`bids`) and the JSON stores of formats (x5 node `Metadata`, Zarr +attributes) share. + +A vocabulary field is stored under its *sidecar key*: its BIDS key +(`"RepetitionTime"`), or, when BIDS has none, its name in `CamelCase` +(`display_range` as `"DisplayRange"`). Every other key of the object is +`extra`. `GeneratedBy` is the BIDS list of objects (`Name`, `Version`, +`Description`, `CodeURL`), and `channels` a list of objects with the +`CamelCase` names of the `Channel` fields. Times are ISO 8601 strings. +An encoding direction is its BIDS string (`"j-"`), or, when it has none, +an object (`Vector`, `Space`). A known term (a `SpaceEnum`, an `IntentEnum`, +...) is its string, a `data_unit` its unit symbol (`"ms"`, `"a.u."`), a +`data_type` its `numpy` name (`"int16"`). +""" + +__all__ = [ + "decode_object", + "encode_changes", + "encode_extra", + "from_json", + "jsonable", + "sidecar_key", + "to_json", +] + +# stdlib +import datetime +import enum + +# externals +import numpy as np +import typing_extensions as tx + +from brainhops.datamodel.metadata import ( + Channel, + ConversionReport, + EncodingDirection, + GeneratedBy, +) + +# internals +from brainhops.datamodel.metadata._vocabulary import BIDS_KEYS +from brainhops.datamodel.systems import CoordinateSystem +from brainhops.datamodel.units import Unit + + +def sidecar_key(name: str) -> str: + """ + The key of a vocabulary field in a JSON object. + + Parameters + ---------- + name : str + The name of the vocabulary field. + + Returns + ------- + str + The BIDS key of the field, or its name in `CamelCase` when BIDS + has no key for it. + """ + return BIDS_KEYS.get(name) or _camel(name) + + +def decode_object( + obj: tx.Mapping[str, tx.Any], names: tx.Iterable[str] +) -> tx.Tuple[tx.Dict[str, tx.Any], tx.Dict[str, tx.Any]]: + """ + Split a JSON object into vocabulary values and other keys. + + Parameters + ---------- + obj : mapping + The JSON object. + names : iterable of str + The vocabulary fields to read, each from its key (see + `sidecar_key`). + + Returns + ------- + values : dict + Field name to decoded value, for the fields found in `obj`. + others : dict + The other keys of `obj`, with their values as they are. + """ + fields = {sidecar_key(name): name for name in names} + values: tx.Dict[str, tx.Any] = {} + others: tx.Dict[str, tx.Any] = {} + for key, value in obj.items(): + name = fields.get(key) + if name is None: + others[key] = value + else: + values[name] = from_json(name, value) + return values, others + + +def encode_changes( + obj: tx.Dict[str, tx.Any], + changed: tx.Mapping[str, tx.Any], + *, + report: ConversionReport, +) -> None: + """ + Write changed vocabulary values into a JSON object, in place. + + A value that JSON cannot hold (an encoding direction in a coordinate + system without a name) is not written: its key is removed, so that + the object does not keep a stale value, and the value is reported as + lost. + + Parameters + ---------- + obj : dict + The JSON object to edit. + changed : mapping + Field name to new value. Each value is written under the key of + its field (see `sidecar_key`), and a `None` value removes the key. + report : ConversionReport + The report to fill with the values that were not written. + """ + for name, value in changed.items(): + key = sidecar_key(name) + if value is None: + obj.pop(key, None) + continue + try: + obj[key] = to_json(name, value) + except ValueError: + obj.pop(key, None) + report.lost[name] = value + + +def encode_extra( + obj: tx.Dict[str, tx.Any], + diff: tx.Mapping[str, tx.Any], + *, + report: ConversionReport, + reserved: tx.Collection[str] = (), +) -> None: + """ + Apply a diff of `extra` to a JSON object, in place. + + The diff is the one that `FileBasedMetadata._changed_fields` computes: + a `None` value removes the key, and any other value is written as + JSON. A key that the format keeps for its own use is not written, and + is reported as lost. + + Parameters + ---------- + obj : dict + The JSON object to edit. + diff : mapping + Key to new value, or to `None` for a removed key. + report : ConversionReport + The report to fill with the reserved keys that were refused. + reserved : collection of str, optional + The keys the format keeps for its own use. + """ + # Not `FileBasedMetadata`'s diff helper: this one also writes JSON + # and refuses the keys a format reserves, two things the data model + # knows nothing of. + for key, value in diff.items(): + if key in reserved: + report.lost[f"extra[{key!r}]"] = value + elif value is None: + obj.pop(key, None) + else: + obj[key] = to_json("extra", value) + + +def from_json(name: str, value: tx.Any) -> tx.Any: + """ + Decode the JSON form of a vocabulary value. + + Parameters + ---------- + name : str + The name of the vocabulary field. + value : object + The value, as JSON holds it. + + Returns + ------- + object + The value, ready to be assigned to the field, which converts it + further. + """ + if value is None: + return None + if name == "generated_by": + entries = value if isinstance(value, list) else [value] + return tuple( + GeneratedBy( + **{ + _GENERATED_BY_KEYS[k]: v + for k, v in entry.items() + if k in _GENERATED_BY_KEYS + } + ) + for entry in entries + ) + if name == "channels": + return tuple( + Channel( + **{ + _CHANNEL_KEYS[k]: v + for k, v in entry.items() + if k in _CHANNEL_KEYS + } + ) + for entry in value + ) + if name in _TIMES and isinstance(value, str): + try: + return datetime.datetime.fromisoformat(value) + except ValueError: + return value + if name in ("history", "sources") and isinstance(value, str): + return (value,) + return value + + +def to_json(name: str, value: tx.Any) -> tx.Any: + """ + Encode a vocabulary value, or a value of `extra`, as JSON. + + `GeneratedBy` and `Channel` entries become objects with keys in the + BIDS style. An encoding direction becomes its BIDS string, or, when + BIDS cannot write it, an object with the keys `Vector` and `Space`; + a space that is a `CoordinateSystem` is written as its name. + + Parameters + ---------- + name : str + The name of the vocabulary field, or `"extra"`. + value : object + The value to encode. + + Returns + ------- + object + A value that can be serialised to JSON. + + Raises + ------ + ValueError + If `value` is an encoding direction in a coordinate system + without a name, which JSON cannot hold. + """ + if name == "generated_by": + return [ + { + key: getattr(entry, attr) + for key, attr in _GENERATED_BY_KEYS.items() + if getattr(entry, attr) is not None + } + for entry in value + ] + if name == "channels": + return [ + { + key: jsonable(getattr(entry, attr)) + for key, attr in _CHANNEL_KEYS.items() + if getattr(entry, attr) is not None + } + for entry in value + ] + if isinstance(value, EncodingDirection): + bids = value.to_bids() + if bids is not None: + return bids + out = {"Vector": list(value.vector)} + if value.space is not None: + out["Space"] = _space_label(value.space) + return out + return jsonable(value) + + +def jsonable(value: tx.Any) -> tx.Any: + """ + Make a value serialisable to JSON. + + Times become ISO 8601 strings, known terms become their strings, a + data type becomes its name, a unit becomes its symbol, and tuples and + arrays become lists. + + Parameters + ---------- + value : object + The value to convert. + + Returns + ------- + object + A value that can be serialised to JSON. + """ + if isinstance(value, (datetime.datetime, datetime.date, datetime.time)): + return value.isoformat() + if isinstance(value, enum.Enum): + return jsonable(value.value) + if isinstance(value, np.dtype): + return value.name + if isinstance(value, Unit): + # The symbol (`"a.u."`, `"mm / s"`) parses back to the same unit. + return value.symbol + if isinstance(value, tuple): + return [jsonable(v) for v in value] + if isinstance(value, list): + return [jsonable(v) for v in value] + if isinstance(value, dict): + return {k: jsonable(v) for k, v in value.items()} + if hasattr(value, "tolist"): + return value.tolist() + return value + + +# ---------------------------------------------------------------------- +# PRIVATE +# ---------------------------------------------------------------------- + +_GENERATED_BY_KEYS = { + "Name": "name", + "Version": "version", + "Description": "description", + "CodeURL": "code_url", +} + +_CHANNEL_KEYS = { + "Name": "name", + "Color": "color", + "DisplayRange": "display_range", + "Unit": "unit", +} + +_TIMES = ("creation_time", "acquisition_time") + + +def _camel(name: str) -> str: + return "".join(part.capitalize() for part in name.split("_")) + + +def _space_label(space: tx.Any) -> str: + """The string a JSON object holds for the space of a direction: a + label as it is, a coordinate system by its name.""" + if not isinstance(space, CoordinateSystem): + return str(space) + if not space.name: + raise ValueError( + "A direction in a coordinate system without a name cannot be " + "written: JSON names the space of a direction." + ) + return space.name diff --git a/src/brainhops/io/metadata/_sync.py b/src/brainhops/io/metadata/_sync.py new file mode 100644 index 00000000..658191b9 --- /dev/null +++ b/src/brainhops/io/metadata/_sync.py @@ -0,0 +1,115 @@ +"""Keeping the metadata of a parser in step with its raw record.""" + +__all__ = ["parent_post_init", "sync_metadata"] + +# stdlib +from inspect import Parameter, signature + +# dependencies +import typing_extensions as tx + +# internals +from ._base import FileBasedMetadata + + +def sync_metadata( + obj: tx.Any, + cls: tx.Type[FileBasedMetadata], + raw: tx.Any = None, + *, + same: tx.Optional[tx.Callable[[tx.Any], bool]] = None, + force: bool = False, + image: tx.Any = None, +) -> bool: + """ + Give a parser the metadata of its raw record, unless the parser + already holds it. + + A parser calls this function from its `__post_init__`. When the + metadata that the parser holds already has the record as its `raw`, + the metadata is kept as it is. Otherwise the record is decoded, and + the fields that changed in the metadata the parser holds (because the + metadata was given explicitly, or carried over by `replace()`) are + set over the decoded values, as changes (see + `FileBasedMetadata.update_from_raw`). Metadata with no record is + decoded again every time, since nothing tells whether it was read + already. The `metadata` field converts what it is given, so the + metadata is already of class `cls` here. + + Parameters + ---------- + obj : object + The parser, which has a `metadata` field. + cls : type + The metadata class of the format. + raw : object or callable + The raw record, or a function without arguments that builds it. + A function suits a record that is rebuilt on each read (Zarr) or + costly to build (MGH): it is called only when needed. + same : callable, optional + Whether the record of the metadata held, passed as the argument, + is the parser's record. By default, `held is raw`. + force : bool, optional + Decode the record afresh, and drop the changes. + image : object, optional + The object passed to the decoder (usually `obj`). + + Returns + ------- + bool + Whether the metadata was read again. + """ + metadata = obj.metadata + held = getattr(metadata, "raw", None) + if not force and held is not None: + if held is raw if same is None else same(held): + return False + if callable(raw): + raw = raw() + if force or metadata is None: + obj.metadata = cls.from_raw(raw, image=image) + else: + obj.metadata = metadata.update_from_raw(raw, image=image) + return True + + +def parent_post_init(parent: tx.Any, arguments: tx.Any = None) -> None: + """ + Run the `__post_init__` that a parser's own one overrides. + + A parser's hook takes the constructor's arguments (`bagof` hands them + to a hook that declares a parameter), and passes them on to the hook + it overrides when that one declares a parameter too -- as the hook of + a concrete transformation does -- and nothing otherwise. + + Parameters + ---------- + parent : super + `super()`, from the parser's hook. + arguments : object, optional + The arguments the parser's hook was given. + """ + hook = getattr(parent, "__post_init__", None) + if hook is None: + return + try: + parameters = signature(hook).parameters.values() + except (TypeError, ValueError): + hook(arguments) + return + if any(parameter.kind in _POSITIONAL for parameter in parameters): + hook(arguments) + else: + hook() + + +# ---------------------------------------------------------------------- +# PRIVATE +# ---------------------------------------------------------------------- + + +_POSITIONAL = ( + Parameter.POSITIONAL_ONLY, + Parameter.POSITIONAL_OR_KEYWORD, + Parameter.VAR_POSITIONAL, +) diff --git a/src/brainhops/io/metadata/bids.py b/src/brainhops/io/metadata/bids.py new file mode 100644 index 00000000..70b8993a --- /dev/null +++ b/src/brainhops/io/metadata/bids.py @@ -0,0 +1,240 @@ +""" +The BIDS JSON sidecar codec. + +A sidecar is the vocabulary as a flat JSON object, in the form that +`brainhops.io.metadata._json` writes and reads (BIDS keys, or a field's +name in `CamelCase` where BIDS has none; every other key is `extra`). +The units already match (seconds, degrees, tesla). The fields of the +[`DiffusionVocabulary`][brainhops.datamodel.metadata._vocabulary.DiffusionVocabulary] +group are not sidecar keys (BIDS keeps them in `.bval`/`.bvec` files, in +voxel axes), and an encoding direction along no voxel axis has no BIDS +string, so `to_bids` reports them as lost. +""" + +__all__ = ["BidsSidecar", "from_bids", "to_bids"] + +# stdlib +import json +import os +from io import BytesIO + +# externals +import typing_extensions as tx + +# internals +from brainhops._core import path +from brainhops._core.streams import preserve_position +from brainhops.datamodel.metadata import ( + UNSUPPORTED, + ConversionReport, + EncodingDirection, + Metadata, +) +from brainhops.datamodel.metadata._report import OnLoss, apply_loss_policy +from brainhops.datamodel.metadata._vocabulary import ( + GROUPS, + VOCABULARY, + DiffusionVocabulary, +) +from brainhops.io.base._metadata_parser import MetadataParser +from brainhops.io.base.parsers import Confidence, SnifferContentError + +from ._base import FileBasedMetadata +from ._json import decode_object, jsonable, sidecar_key, to_json + + +def from_bids(source: tx.Any) -> Metadata: + """ + Read a BIDS JSON sidecar into + [`Metadata`][brainhops.datamodel.metadata.Metadata]. + + Parameters + ---------- + source : mapping | str | PathLike | file + The sidecar, as a decoded JSON object, a JSON string, a path, or + an open file. + + Returns + ------- + Metadata + The vocabulary fields the sidecar names, and its other keys in + `extra`. + """ + values, extra = decode_object(_read(source), _SIDECAR_FIELDS) + return Metadata(extra=extra, **values) + + +def to_bids( + metadata: Metadata, + *, + on_loss: tx.Optional[OnLoss] = None, +) -> tx.Dict[str, tx.Any]: + """ + Write metadata as a BIDS JSON sidecar. + + Parameters + ---------- + metadata : Metadata + The metadata to write; only its vocabulary and `extra` are used. + on_loss : {"ignore", "warn", "raise"} or ConversionReport, optional + What to do with the fields a sidecar cannot hold (the diffusion + fields, an encoding direction along no voxel axis). Defaults to + the policy in effect; a `ConversionReport` is filled instead. + + Returns + ------- + dict + A JSON-serialisable sidecar. A key of `extra` that collides with + a vocabulary key is overridden by the vocabulary value. + """ + report = ConversionReport(source=metadata.format, target="bids") + sidecar: tx.Dict[str, tx.Any] = {} + extra = metadata.extra + if extra and extra is not UNSUPPORTED: + sidecar.update(jsonable(dict(extra))) + for name in VOCABULARY: + value = getattr(metadata, name, None) + if value is None or value is UNSUPPORTED: + continue + if name not in _SIDECAR_FIELDS or ( + isinstance(value, EncodingDirection) and value.to_bids() is None + ): + report.lost[name] = value + continue + sidecar[sidecar_key(name)] = to_json(name, value) + apply_loss_policy(report, on_loss, stacklevel=3) + return sidecar + + +class BidsSidecar(MetadataParser): + """ + The reader of BIDS JSON sidecars, for + [`FileBasedMetadata.load`][brainhops.io.metadata.FileBasedMetadata.load]. + + A sidecar is a file of metadata only, with no format class of its + own: it reads as generic `Metadata` (see [`from_bids`][]). Any JSON + object is accepted, and its keys that are not BIDS keys of the + vocabulary land in `extra`. Not being a `FileBasedMetadata`, it is + added to the registry of that dispatcher by hand. + """ + + EXTENSIONS: tx.ClassVar[tx.Tuple[str, ...]] = (".json",) + HINTS = ("bids", "json") + + @classmethod + def sniff_fileobj( + cls, + file: tx.IO, + error: tx.Union[bool, tx.Type[Exception]] = False, + **kwargs: tx.Any, + ) -> float: + """ + Score how confident the reader is that an open file holds a + sidecar: a JSON object. + + Parameters + ---------- + file : file object + A binary stream. Its position is restored. + error : bool or type, optional + Raise an error instead of returning 0. + **kwargs + Ignored. + + Returns + ------- + float + `MAYBE` for a JSON object (many JSON files are not sidecars), + else 0. + """ + start = file.tell() + try: + is_object = isinstance(json.load(file), dict) + except Exception: + is_object = False + finally: + file.seek(start) + if is_object: + return Confidence.MAYBE + if error: + raise (SnifferContentError if error is True else error)( + "Content is not a JSON object" + ) + return Confidence.NO + + @classmethod + def sniff_bytes( + cls, + content: bytes, + error: tx.Union[bool, tx.Type[Exception]] = False, + **kwargs: tx.Any, + ) -> float: + """ + Score how confident the reader is that bytes hold a sidecar. + + Parameters + ---------- + content : bytes + The content of a file. + error : bool or type, optional + Raise an error instead of returning 0. + **kwargs + Ignored. + + Returns + ------- + float + The confidence, in `[0, 1]`. + """ + return cls.sniff_fileobj(BytesIO(content), error=error) + + @classmethod + def from_fileobj(cls, file: tx.IO, **kwargs: tx.Any) -> Metadata: + """ + Read an open sidecar. + + Parameters + ---------- + file : file object + The sidecar, open for reading. Its position is restored. + **kwargs + Ignored. + + Returns + ------- + Metadata + Generic metadata (see [`from_bids`][]). + """ + with preserve_position(file): + return from_bids(_read(file)) + + +# `register_format` registers a class into the registries of its +# ancestors, and the sidecar reader is not a `FileBasedMetadata`. +FileBasedMetadata._REGISTRY.add(BidsSidecar) + + +# ---------------------------------------------------------------------- +# PRIVATE +# ---------------------------------------------------------------------- + +# Not the diffusion fields: BIDS stores them as `.bval`/`.bvec` files. +_SIDECAR_FIELDS = tuple( + name for name in VOCABULARY if name not in GROUPS[DiffusionVocabulary] +) + + +def _read(source: tx.Any) -> tx.Dict[str, tx.Any]: + if isinstance(source, tx.Mapping): + return dict(source) + if isinstance(source, str) and source.lstrip().startswith("{"): + return json.loads(source) + if isinstance(source, (str, os.PathLike, path.PathLike)): + with path.Path(source).open("r") as f: + return json.load(f) + if hasattr(source, "read"): + return json.load(source) + raise TypeError( + f"A BIDS sidecar is a mapping, a JSON string, a path or an open " + f"file, not {type(source).__name__}." + ) diff --git a/src/brainhops/io/transformations/fsl/flirt/__init__.py b/src/brainhops/io/transformations/fsl/flirt/__init__.py index 0fbc631c..d714ae4a 100644 --- a/src/brainhops/io/transformations/fsl/flirt/__init__.py +++ b/src/brainhops/io/transformations/fsl/flirt/__init__.py @@ -4,7 +4,8 @@ scaled-mm coordinates to reference-image scaled-mm coordinates. """ -__all__ = ["FlirtMatrixParser", "FlirtTransform"] +__all__ = ["FlirtMatrixParser", "FlirtMetadata", "FlirtTransform"] +from ._metadata import FlirtMetadata from ._parser import FlirtMatrixParser from ._xform import FlirtTransform diff --git a/src/brainhops/io/transformations/fsl/flirt/_metadata.py b/src/brainhops/io/transformations/fsl/flirt/_metadata.py new file mode 100644 index 00000000..220a1b28 --- /dev/null +++ b/src/brainhops/io/transformations/fsl/flirt/_metadata.py @@ -0,0 +1,94 @@ +""" +The metadata of FLIRT `.mat` files: [`FlirtMetadata`][]. + +A FLIRT matrix is a bare `(4, 4)` matrix: it stores no metadata, and +there is no raw record. The reader needs the moving and reference +images, though, and when they were read from files their paths are +`moving` and `fixed`. So the metadata is not opaque: it holds these two +fields, in memory only. A `.mat` file has no place for them, so a write +reports them as lost. +""" + +__all__ = ["FlirtMetadata"] + +# dependencies +import typing_extensions as tx + +# internals +from brainhops.datamodel.metadata import ConversionReport +from brainhops.io.base.parsers import ParserNotImplementedError +from brainhops.io.metadata import FileBasedMetadata + + +class FlirtMetadata( + FileBasedMetadata[None], + on={"format": "flirt"}, + supports=("moving", "fixed"), +): + """ + The metadata of a FLIRT `.mat` file: only `moving` and `fixed`, the + paths of the images the reader was given, kept in memory (a read and + a copy keep them; a write reports them as lost). There is no raw + record: `raw` is always `None`. + """ + + @classmethod + def load(cls, file: tx.Any, **kwargs: tx.Any) -> "FlirtMetadata": + """ + Refuse to read the metadata of a file: a `.mat` file holds none. + + Parameters + ---------- + file : str, path-like or file object + The file. + **kwargs + Ignored. + + Returns + ------- + FlirtMetadata + Never: the method always raises. + + Raises + ------ + ParserNotImplementedError + Always. + """ + raise ParserNotImplementedError( + f"{cls.__name__} stores no metadata that can be read on its own." + ) + + @classmethod + def _decode_raw( + cls, raw: tx.Any, *, image: tx.Any = None + ) -> tx.Dict[str, tx.Any]: + return { + "moving": _filename(getattr(image, "moving", None)), + "fixed": _filename(getattr(image, "reference", None)), + } + + def _encode_raw( + self, + raw: tx.Any, + changed: tx.Dict[str, tx.Any], + *, + image: tx.Any = None, + report: ConversionReport, + ) -> tx.Any: + # Nothing is stored: the images are given again at read time. + for name in ("moving", "fixed"): + value = getattr(self, name) + if value is not None: + report.lost[name] = value + return raw + + +def _filename(image: tx.Any) -> tx.Optional[str]: + """The path a (`nibabel` or brainhops NIfTI) image was read from.""" + for obj in (image, getattr(image, "image", None)): + getter = getattr(obj, "get_filename", None) + if callable(getter): + name = getter() + if name: + return str(name) + return None diff --git a/src/brainhops/io/transformations/itk/__init__.py b/src/brainhops/io/transformations/itk/__init__.py index d42b48a3..57b56a37 100644 --- a/src/brainhops/io/transformations/itk/__init__.py +++ b/src/brainhops/io/transformations/itk/__init__.py @@ -45,6 +45,7 @@ "ItkAffineBase", "ItkBlockBase", "ItkDisplacementBase", + "ItkMetadata", "ItkPrecision", "ItkStruct", "ItkTransform", @@ -62,6 +63,7 @@ ItkStruct, ItkTransformClass, ) +from ._metadata import ItkMetadata from ._xform import ItkTransform # The h5 reader needs h5py, which is optional. It is imported only when @@ -69,8 +71,9 @@ # its own optional-dependency submodules. try: from . import h5 + from .h5 import ItkH5Metadata - __all__ += ["h5"] + __all__ += ["h5", "ItkH5Metadata"] except ImportError: # h5py is optional pass diff --git a/src/brainhops/io/transformations/itk/_metadata.py b/src/brainhops/io/transformations/itk/_metadata.py new file mode 100644 index 00000000..8c11647b --- /dev/null +++ b/src/brainhops/io/transformations/itk/_metadata.py @@ -0,0 +1,75 @@ +""" +The metadata of ITK transformation files. + +- `.tfm` and `.mat` store a bare chain of parameters, and no metadata: + [`ItkMetadata`][] is an `OpaqueMetadata`, and every field is + unsupported. +- `.h5` records the version of ITK that wrote it (`/ITKVersion`): + [`ItkH5Metadata`][brainhops.io.transformations.itk.h5.ItkH5Metadata], + next to the parser of `.h5` files (which needs `h5py`), reads it as + `generated_by`, with the small root header ([`H5Header`][], defined + here) as its raw record. + +ITK's `precision` (`float`/`double`) is the element type of the stored +parameters; it stays in the raw record (`ItkStruct.precision`) for now, +rather than `data_type`. + +The blocks of a chain (or of a `CompositeTransform`) are data model +transformations with no metadata of their own: composition does not +merge, so the metadata of the file is on the transformation read from +it, not on its blocks. +""" + +__all__ = ["H5Header", "ItkMetadata"] + +# dependencies +import typing_extensions as tx +from bagof.magic import HIDE_IF_NONE, Magic + +# internals +from brainhops.io.metadata import OpaqueMetadata + + +class H5Header( + Magic, + convert=True, + repr=HIDE_IF_NONE, +): + """ + The root header of an ITK `.h5` file: the versions of the libraries + and of the system that wrote it, as the root datasets of the file + record them. It is the raw record of + [`ItkH5Metadata`][brainhops.io.transformations.itk.h5.ItkH5Metadata]. + """ + + HDFVersion: tx.Optional[str] = None + """ + A string describing the version of the HDF5 library used. + Ex: "HDF5 library version: 1.10.4" + """ + + ITKVersion: tx.Optional[str] = None + """ + A string describing the version of the ITK library used. + Ex: "5.1.0" + """ + + OSName: tx.Optional[str] = None + """ + A string describing the operating system name. + Ex: "Linux" + """ + + OSVersion: tx.Optional[str] = None + """ + A string describing the operating system version. + Ex: "6.1.0-1007-oem" + """ + + +class ItkMetadata(OpaqueMetadata, on={"format": "itk"}): + """The metadata of an ITK `.tfm` or `.mat` file: none.""" + + # Declared again: the field of a subclass of a pinned format is + # narrowed to the parent's value (`'opaque'`), which `'itk'` is not. + format: tx.Annotated[tx.Literal["itk"], tx.Doc("Always `'itk'`.")] = "itk" diff --git a/src/brainhops/io/transformations/itk/h5/__init__.py b/src/brainhops/io/transformations/itk/h5/__init__.py index 01e335b7..3c3ac4ec 100644 --- a/src/brainhops/io/transformations/itk/h5/__init__.py +++ b/src/brainhops/io/transformations/itk/h5/__init__.py @@ -129,7 +129,14 @@ properly spelled version cannot be indexed. """ -__all__ = ["H5Transform", "H5Header", "H5TransformParser", "DelayedH5Array"] - +__all__ = [ + "H5Transform", + "H5Header", + "H5TransformParser", + "DelayedH5Array", + "ItkH5Metadata", +] + +from ._metadata import ItkH5Metadata from ._parser import DelayedH5Array, H5Header, H5TransformParser from ._xform import H5Transform diff --git a/src/brainhops/io/transformations/itk/h5/_metadata.py b/src/brainhops/io/transformations/itk/h5/_metadata.py new file mode 100644 index 00000000..b4ae20bc --- /dev/null +++ b/src/brainhops/io/transformations/itk/h5/_metadata.py @@ -0,0 +1,151 @@ +""" +The metadata of ITK `.h5` files: [`ItkH5Metadata`][], next to the parser +of the files, since reading them needs `h5py`, which is optional. +""" + +__all__ = ["ItkH5Metadata", "read_h5_header"] + +# dependencies +import h5py +import typing_extensions as tx +from bagof.magic import replace + +# internals +from brainhops.datamodel.metadata import ConversionReport, GeneratedBy +from brainhops.io.base._base import register_format +from brainhops.io.base.hdf5 import Hdf5MetadataParser, read_string +from brainhops.io.base.parsers import Confidence, SnifferContentError +from brainhops.io.metadata import FileBasedMetadata + +# locals +from .._metadata import H5Header + + +@register_format +class ItkH5Metadata( + Hdf5MetadataParser, + FileBasedMetadata[H5Header], + on={"format": "itk-h5"}, + supports=("generated_by",), +): + """ + The metadata of an ITK `.h5` file: the version of ITK that wrote + it, as `generated_by`. Its raw record (`raw`) is the root header of + the file (an `H5Header`: `/ITKVersion`, ...). + `ItkH5Metadata.load(path)` reads the root header alone. + """ + + EXTENSIONS: tx.ClassVar[tx.Tuple[str, ...]] = (".h5",) + HINTS = ("itk", "h5") + + @classmethod + def sniff_h5( + cls, + h5file: h5py.File, + error: tx.Union[bool, tx.Type[Exception]] = False, + ) -> float: + """ + Score how confident the class is that an open HDF5 file is an ITK + transform file: one that records the ITK version at its root. + + Parameters + ---------- + h5file : h5py.File + The open file. + error : bool or type, optional + Raise an error (this one, or `SnifferContentError` for `True`) + instead of returning 0. + + Returns + ------- + float + The confidence, in `[0, 1]`. + """ + if "ITKVersion" in h5file.keys(): + return Confidence.CERTAIN + if error: + raise (SnifferContentError if error is True else error)( + "HDF5 file is not an ITK transform file" + ) + return Confidence.NO + + @classmethod + def from_h5(cls, h5file: h5py.File, **kwargs: tx.Any) -> tx.Self: + """ + Read the root header of an open ITK `.h5` file, without its + transforms. + + Parameters + ---------- + h5file : h5py.File + The open file. + **kwargs + Ignored. + + Returns + ------- + ItkH5Metadata + The metadata of the file, with its root header as `raw`. + """ + return cls.from_raw(read_h5_header(h5file)) + + @property + def header(self) -> tx.Optional[H5Header]: + """The root header (the raw record, `raw`).""" + return self.raw + + @classmethod + def _decode_raw( + cls, raw: tx.Optional[H5Header], *, image: tx.Any = None + ) -> tx.Dict[str, tx.Any]: + version = getattr(raw, "ITKVersion", None) + if not version: + return {} + return {"generated_by": (GeneratedBy(name=_ITK, version=version),)} + + def _encode_raw( + self, + raw: H5Header, + changed: tx.Dict[str, tx.Any], + *, + image: tx.Any = None, + report: ConversionReport, + ) -> H5Header: + if "generated_by" not in changed: + return raw + entries = tuple(changed["generated_by"] or ()) + itk = [g for g in entries if g.name == _ITK] + others = tuple(g for g in entries if g.name != _ITK) + if others: + report.lost["generated_by"] = others + version = itk[0].version if itk else None + return replace(raw, ITKVersion=version) + + +def read_h5_header(h5file: h5py.File) -> H5Header: + """ + Read the root header of an open ITK `.h5` file. + + Parameters + ---------- + h5file : h5py.File + The open file. + + Returns + ------- + H5Header + The versions recorded at the root of the file. + """ + header = H5Header() + for name in ("HDFVersion", "ITKVersion", "OSName", "OSVersion"): + if f"/{name}" in h5file: + setattr(header, name, read_string(h5file[f"/{name}"])) + return header + + +# ---------------------------------------------------------------------- +# PRIVATE +# ---------------------------------------------------------------------- + + +_ITK = "ITK" diff --git a/src/brainhops/io/transformations/itk/h5/_parser.py b/src/brainhops/io/transformations/itk/h5/_parser.py index 51ed0c7f..73ae037f 100644 --- a/src/brainhops/io/transformations/itk/h5/_parser.py +++ b/src/brainhops/io/transformations/itk/h5/_parser.py @@ -17,40 +17,10 @@ # locals from .._common import ItkStruct, ItkTransformClass, _application_order +from .._metadata import H5Header +from ._metadata import read_h5_header -__all__ = ["DelayedH5Array", "H5Header", "H5TransformParser"] - - -class H5Header( - Magic, - convert=True, - repr=HIDE_IF_NONE, -): - """Header of a ITK H5 file.""" - - HDFVersion: tx.Optional[str] = None - """ - A string describing the version of the HDF5 library used. - Ex: "HDF5 library version: 1.10.4" - """ - - ITKVersion: tx.Optional[str] = None - """ - A string describing the version of the ITK library used. - Ex: "5.1.0" - """ - - OSName: tx.Optional[str] = None - """ - A string describing the operating system name. - Ex: "Linux" - """ - - OSVersion: tx.Optional[str] = None - """ - A string describing the operating system version. - Ex: "6.1.0-1007-oem" - """ +__all__ = ["DelayedH5Array", "H5Header", "H5TransformParser", "read_h5_header"] class H5TransformParser( @@ -129,15 +99,7 @@ def from_h5( obj The parsed object. """ - header = H5Header() - if "/HDFVersion" in h5file: - header.HDFVersion = _readstr(h5file["/HDFVersion"]) - if "/ITKVersion" in h5file: - header.ITKVersion = _readstr(h5file["/ITKVersion"]) - if "/OSName" in h5file: - header.OSName = _readstr(h5file["/OSName"]) - if "/OSVersion" in h5file: - header.OSVersion = _readstr(h5file["/OSVersion"]) + header = read_h5_header(h5file) obj = cls(header=header, file=h5file if keep_open else None) nodes = h5file.get("/TransformGroup", {}) diff --git a/src/brainhops/io/transformations/nifti/__init__.py b/src/brainhops/io/transformations/nifti/__init__.py index 1120f93d..a1c6db6a 100644 --- a/src/brainhops/io/transformations/nifti/__init__.py +++ b/src/brainhops/io/transformations/nifti/__init__.py @@ -38,12 +38,15 @@ __all__ = [ "NiftiBasedTransformation", + "NiftiMetadata", "NiftiRASCoordinatesField", "NiftiRASDisplacementField", "NiftiRASToVoxel", "NiftiVoxelToRAS", ] +from brainhops.io.base._nifti_metadata import NiftiMetadata + from .affines import NiftiRASToVoxel, NiftiVoxelToRAS from .base import NiftiBasedTransformation from .fields import NiftiRASCoordinatesField, NiftiRASDisplacementField diff --git a/src/brainhops/io/transformations/x5/__init__.py b/src/brainhops/io/transformations/x5/__init__.py index c8933532..84780904 100644 --- a/src/brainhops/io/transformations/x5/__init__.py +++ b/src/brainhops/io/transformations/x5/__init__.py @@ -173,11 +173,14 @@ "X5DisplacementField", "X5Domain", "X5Header", + "X5Metadata", + "X5Raw", "X5Node", "X5Transform", "X5TransformParser", ] from ._blocks import X5BSplineField, X5CoordinatesField, X5DisplacementField +from ._metadata import X5Metadata, X5Raw from ._struct import X5Domain, X5Header, X5Node from ._xform import X5Transform, X5TransformParser diff --git a/src/brainhops/io/transformations/x5/_metadata.py b/src/brainhops/io/transformations/x5/_metadata.py new file mode 100644 index 00000000..15021da8 --- /dev/null +++ b/src/brainhops/io/transformations/x5/_metadata.py @@ -0,0 +1,299 @@ +""" +The metadata of X5 files: [`X5Metadata`][]. + +Its raw record (`raw`) is an `X5Raw`, which holds the root of the file +(`X5Header`) and the node the transformation was read from (`X5Node`). +Every vocabulary field is stored in the node's JSON `Metadata` +attribute, under its BIDS sidecar key (`Description`, `GeneratedBy`, +...) or, when BIDS has none, under its name in `CamelCase` (`History`, +`Moving`, `Fixed`, `InputSpace`, `OutputSpace`). The other keys of the +JSON object are `extra`. A JSON object stores anything, so nothing is +unsupported. + +`Domain.Coordinates` is not decoded: nitransforms writes the kind of +coordinates there (`"cartesian"`), not the label of a space. +""" + +__all__ = ["X5Metadata", "X5Raw", "metadata_index"] + +# dependencies +import typing_extensions as tx +from bagof.magic import replace + +from brainhops.datamodel.metadata import ConversionReport + +# internals +from brainhops.datamodel.metadata._sentinel import ALL +from brainhops.datamodel.metadata._vocabulary import VOCABULARY +from brainhops.io.base._base import register_format +from brainhops.io.base.hdf5 import Hdf5MetadataParser +from brainhops.io.base.parsers import ( + Confidence, + ParserContentError, + SnifferContentError, +) +from brainhops.io.metadata import FileBasedMetadata +from brainhops.io.metadata._json import ( + decode_object, + encode_changes, + encode_extra, +) + +# locals +from ._struct import X5Header, X5Node, is_x5, read_x5 + +# The default of `X5Raw(node=...)`: a new, empty node. `None` means a +# chain of several nodes, so it cannot be the default. Above `X5Raw`, +# whose signature evaluates it. +_NEW = object() + + +class X5Raw: + """ + The raw record of an X5 transformation: the root of the file and the + node that the transformation was read from. + + A class of its own, rather than a pair, so that the type of the + record names its format: a conversion gives a record back to the + format whose class declares its type, and no other format declares + this one. + """ + + __slots__ = ("header", "node") + + def __init__( + self, + header: tx.Optional[X5Header] = None, + node: tx.Any = _NEW, + ) -> None: + """ + Parameters + ---------- + header : X5Header, optional + The root of the file. By default, an empty header. + node : X5Node or None, optional + The node of the transformation, or `None` for a chain of + several nodes, whose own metadata stays in their nodes. By + default, an empty node. + """ + self.header = X5Header() if header is None else header + self.node = X5Node() if node is _NEW else node + + def is_record_of(self, header: tx.Any, node: tx.Any) -> bool: + """ + Whether this record holds exactly the given header and node. + + Parameters + ---------- + header : X5Header + A root of a file. + node : X5Node or None + A node. + + Returns + ------- + bool + Whether both are the objects this record holds. + """ + return self.header is header and self.node is node + + def __eq__(self, other: object) -> bool: + if not isinstance(other, X5Raw): + return NotImplemented + return self.header == other.header and self.node == other.node + + __hash__ = None # type: ignore[assignment] + + def __repr__(self) -> str: + return f"X5Raw(header=..., node={self.node is not None})" + + +@register_format +class X5Metadata( + Hdf5MetadataParser, + FileBasedMetadata[X5Raw], + on={"format": "x5"}, + supports=ALL, +): + """ + The metadata of an X5 transform node, stored in its JSON `Metadata`. + + Its raw record (`raw`) is an `X5Raw`: the root of the file and the + node the transformation was read from (no node for a chain of + several nodes, whose own metadata stays in their nodes). `node` and + `header` are its two halves. + `X5Metadata.load(path)` reads the metadata of the transformation that + `X5Transform.load(path)` would read (`chain=` and `position=` select + another one), without reading its arrays. + """ + + EXTENSIONS: tx.ClassVar[tx.Tuple[str, ...]] = (".x5",) + HINTS = ("x5",) + + # --- reading the node of a file ----------------------------------- + + @classmethod + def sniff_h5( + cls, + h5file: tx.Any, + error: tx.Union[bool, tx.Type[Exception]] = False, + ) -> float: + """ + Score how confident the class is that an open HDF5 file is an X5 + file. + + Parameters + ---------- + h5file : h5py.File + The open file. + error : bool or type, optional + Raise an error (this one, or `SnifferContentError` for `True`) + instead of returning 0. + + Returns + ------- + float + The confidence, in `[0, 1]`. + """ + if is_x5(h5file): + return Confidence.CERTAIN + if error: + raise (SnifferContentError if error is True else error)( + "HDF5 file is not an X5 file: no Format='X5'." + ) + return Confidence.NO + + @classmethod + def from_h5( + cls, + h5file: tx.Any, + chain: tx.Optional[int] = None, + position: tx.Optional[int] = None, + **kwargs: tx.Any, + ) -> tx.Self: + """ + Read the metadata of the transformation of an open X5 file that + `X5Transform` would read, without its arrays. + + Parameters + ---------- + h5file : h5py.File + The open file. + chain : int, optional + The chain of `/TransformChain` to read. + position : int, optional + The single transform of `/TransformGroup` to read. + **kwargs + Ignored. + + Returns + ------- + X5Metadata + The metadata of the node, with the root and the node as `raw`. + + Raises + ------ + ParserContentError + If the file has no such transform. + """ + header, nodes = read_x5(h5file, load=False) + try: + index = metadata_index(header, chain, position) + node = None if index is None else nodes[index] + except IndexError: + raise ParserContentError( + f"This X5 file has no transform for chain={chain}, " + f"position={position}." + ) from None + return cls.from_raw(X5Raw(header, node)) + + @property + def header(self) -> tx.Optional[X5Header]: + """The root of the file (first half of the raw record).""" + return None if self.raw is None else self.raw.header + + @property + def node(self) -> tx.Optional[X5Node]: + """The node (second half of the raw record).""" + return None if self.raw is None else self.raw.node + + # --- hooks -------------------------------------------------------- + + @classmethod + def _decode_raw( + cls, raw: tx.Any, *, image: tx.Any = None + ) -> tx.Dict[str, tx.Any]: + node = None if raw is None else raw.node + json = getattr(node, "metadata", None) + if not isinstance(json, dict): + return {} + values, extra = decode_object(json, VOCABULARY) + return {**values, "extra": extra or None} + + def _encode_raw( + self, + raw: X5Raw, + changed: tx.Dict[str, tx.Any], + *, + image: tx.Any = None, + report: ConversionReport, + ) -> X5Raw: + header, node = raw.header, raw.node + if not changed: + return raw + if node is None: + # A chain of several nodes: no node to write into. + for name, value in changed.items(): + report.lost[name] = value + return raw + json = node.metadata + if isinstance(json, str) and json: + report.approximated["extra"] = ( + "the node's Metadata was not JSON, and is replaced" + ) + json = dict(json) if isinstance(json, dict) else {} + encode_changes( + json, + {k: v for k, v in changed.items() if k != "extra"}, + report=report, + ) + if "extra" in changed: + encode_extra(json, changed["extra"], report=report) + if not json and node.metadata is None: + return raw + return X5Raw(header, replace(node, metadata=json)) + + +def metadata_index( + header: X5Header, + chain: tx.Optional[int] = None, + position: tx.Optional[int] = None, +) -> tx.Optional[int]: + """ + The node whose metadata is the metadata of a transformation. + + Parameters + ---------- + header : X5Header + The root of the file. + chain : int, optional + The chain of `/TransformChain` the transformation is. + position : int, optional + The single transform of `/TransformGroup` the transformation is. + + Returns + ------- + int or None + The index of the node: the single node read, or `None` for a + chain of several nodes, which has no metadata of its own + (composition does not merge). + """ + if position is not None: + return int(position) + if chain is not None: + nodes = header.chains[chain] + elif header.chains: + nodes = header.chains[0] + else: + return 0 + return nodes[0] if len(nodes) == 1 else None diff --git a/tests/test_datamodel_metadata.py b/tests/test_datamodel_metadata.py new file mode 100644 index 00000000..be7ac892 --- /dev/null +++ b/tests/test_datamodel_metadata.py @@ -0,0 +1,1484 @@ +""" +Tests for the format-agnostic metadata framework +(`brainhops.datamodel.metadata`), on synthetic formats. + +What is checked: the `UNSUPPORTED` sentinel; the class hierarchy and the +vocabulary groups; the `supports=` class keyword; the +typed terms (enums, units, data types, encoding directions); the read-time +snapshot and the change-detecting write (cases 1-4 of section 6 of the +design memo); `to()`, conversion loss reports and the loss policies; +`derive` and the propagation hooks; the BIDS sidecar codec; and the +`metadata` field of the data model roots. +""" + +import copy +import json +import pickle +import subprocess +import sys +import warnings + +import numpy as np +import pytest +import typing_extensions as tx +from bagof.magic import Factory, Magic, fields, replace + +import brainhops.datamodel.metadata +from brainhops.datamodel.axes import Axis +from brainhops.datamodel.base import DataModelBase +from brainhops.datamodel.enums import ( + AxisType, + ContrastMethod, + IlluminationType, + IntentEnum, + Manufacturer, + SpaceEnum, +) +from brainhops.datamodel.geometry import Geometry +from brainhops.datamodel.metadata import ( + UNSUPPORTED, + Channel, + ConversionReport, + DiffusionVocabulary, + DisplayVocabulary, + EncodingDirection, + GeneratedBy, + Indexed, + Metadata, + MetadataLossError, + MetadataLossWarning, + MicroscopyVocabulary, + MRIVocabulary, + Operation, + ProvenanceVocabulary, + Resampled, + Scope, + StorageVocabulary, + TransformVocabulary, + Vocabulary, + metadata_loss_policy, +) +from brainhops.datamodel.metadata._dtype import preferred_dtype, stored_values +from brainhops.datamodel.metadata._field import MetadataField +from brainhops.datamodel.metadata._operations import propagate, propagates +from brainhops.datamodel.metadata._report import ( + apply_loss_policy, + collect_loss_reports, +) +from brainhops.datamodel.metadata._sentinel import ALL, Unsupported +from brainhops.datamodel.metadata._vocabulary import ( + GROUPS, + VOCABULARY, + Along, + Scoped, +) +from brainhops.datamodel.systems import CoordinateSystem +from brainhops.datamodel.transformations import Affine, CartesianField +from brainhops.io.metadata import ( + FileBasedMetadata, + OpaqueMetadata, + from_bids, + to_bids, +) + + +def _to(source, target, **kwargs): # noqa: ANN001, ANN003, ANN202 + """`source.to(target, ...)`, and the report it filled.""" + report = ConversionReport() + return source.to(target, on_loss=report, **kwargs), report + + +# ---------------------------------------------------------------------- +# SYNTHETIC FORMATS +# ---------------------------------------------------------------------- + + +class LiteMetadata( + FileBasedMetadata, + on={"format": "test-lite"}, + supports=("description", "slice_timing", "history", "extra"), +): + """A format that stores three fields and free-form keys.""" + + +class DictRecord(dict): + """The raw record of `DictMetadata`: a dict of a type of its own, so + that a conversion knows whose record it is.""" + + +class DictMetadata( + FileBasedMetadata[DictRecord], + on={"format": "test-dict"}, + supports=("description", "display_range", "slice_timing", "extra"), +): + """A format whose record is a dict: `desc`, `cal`, `slices`, and any + other key is free-form.""" + + _KNOWN = {"desc": "description", "cal": "display_range"} + + @classmethod + def _decode_raw(cls, raw, *, image=None) -> dict: # noqa: ANN001 + out = {name: raw.get(key) for key, name in cls._KNOWN.items()} + out["slice_timing"] = raw.get("slices") + out["extra"] = { + k: v + for k, v in raw.items() + if k not in cls._KNOWN and k != "slices" + } + return out + + def _encode_raw(self, raw, changed, *, image=None, report) -> dict: # noqa: ANN001 + for key, name in self._KNOWN.items(): + if name in changed: + if changed[name] is None: + raw.pop(key, None) + else: + raw[key] = changed[name] + if "slice_timing" in changed: + times = changed["slice_timing"] + if times is None: + raw.pop("slices", None) + elif image is not None and len(times) != image: + # `image` stands for the number of slices here. + report.lost["slice_timing"] = times + else: + raw["slices"] = times + for key, value in (changed.get("extra") or {}).items(): + if value is None: + raw.pop(key, None) + else: + raw[key] = value + if "description" in changed and changed["description"]: + if len(changed["description"]) > 8: + report.approximated["description"] = "truncated to 8" + raw["desc"] = changed["description"][:8] + return raw + + +@propagates(DictRecord, Operation) +def _derive_dict_record(raw, operation, *, name, source) -> DictRecord: # noqa: ANN001 + """The record of derived `DictMetadata`: a copy, without its + `slice_hint` when the spatial axes move.""" + raw = copy.deepcopy(raw) + if operation.moves_space: + raw.pop("slice_hint", None) + return raw + + +class DialectMetadata( + FileBasedMetadata, + on={"format": "test-dialect"}, + supports=("description", "channels"), +): + """A format whose `channels` support depends on the instance.""" + + dialect: str = "rich" + + def __post_init__(self) -> None: + super().__post_init__() + if self.dialect == "plain" and self.channels is None: + object.__setattr__(self, "channels", UNSUPPORTED) + + +def _rich() -> Metadata: + return Metadata( + description="a scan", + echo_time=0.03, + slice_timing=(0.0, 0.5, 1.0), + history=("acquired",), + extra={"Custom": 1}, + ) + + +# ---------------------------------------------------------------------- +# SENTINEL +# ---------------------------------------------------------------------- + + +def test_unsupported_is_a_falsy_singleton() -> None: + assert Unsupported() is UNSUPPORTED + assert not UNSUPPORTED + assert UNSUPPORTED is not None + assert UNSUPPORTED != None # noqa: E711 + assert repr(UNSUPPORTED) == "UNSUPPORTED" + + +def test_unsupported_survives_copy_and_pickle() -> None: + assert copy.copy(UNSUPPORTED) is UNSUPPORTED + assert copy.deepcopy(UNSUPPORTED) is UNSUPPORTED + assert pickle.loads(pickle.dumps(UNSUPPORTED)) is UNSUPPORTED + + +def test_maybe_fields_convert_values_and_keep_the_sentinel() -> None: + meta = Metadata(repetition_time="2", slice_timing=[0, 1]) + assert meta.repetition_time == 2.0 + assert meta.slice_timing == (0.0, 1.0) + meta = Metadata(echo_time=UNSUPPORTED) + assert meta.echo_time is UNSUPPORTED + + +def test_an_unsupported_field_is_refused_at_construction() -> None: + with pytest.raises(ValueError, match="echo_time"): + LiteMetadata(echo_time=0.03) + with pytest.raises(ValueError, match="echo_time"): + LiteMetadata.from_dict({"echo_time": 0.03}) + # None and UNSUPPORTED both read as "nothing here". + assert LiteMetadata(echo_time=None).echo_time is UNSUPPORTED + assert LiteMetadata(echo_time=UNSUPPORTED).echo_time is UNSUPPORTED + + +def test_repr_hides_none_and_unsupported() -> None: + # The class of a format hides `format`: its name says it. + assert repr(LiteMetadata(description="x")) == ( + "LiteMetadata(description='x')" + ) + # `format` first, then `extra` (only when not empty), then the + # vocabulary in its declared order. + assert repr(Metadata(echo_time=0.03, description="x", extra={})) == ( + "Metadata(format='generic', description='x', echo_time=0.03)" + ) + assert repr(Metadata(name="n", extra={"k": 1})) == ( + "Metadata(format='generic', extra={'k': 1}, name='n')" + ) + assert repr(OpaqueMetadata()) == "OpaqueMetadata()" + # Generic metadata shows any format, including an unknown one. + assert repr(Metadata(format="unknown")) == "Metadata(format='unknown')" + assert "format=" not in repr(Metadata(format="test-lite")) + + +# ---------------------------------------------------------------------- +# supports= +# ---------------------------------------------------------------------- + + +def test_supports_lists_what_a_format_stores() -> None: + vocabulary = set(VOCABULARY) | {"extra"} + supported = {"description", "slice_timing", "history", "extra"} + assert LiteMetadata.supported_fields == supported + # The complement is derived from the declaration. + assert LiteMetadata.unsupported_fields == vocabulary - supported + assert Metadata.supported_fields == vocabulary + assert Metadata.unsupported_fields == frozenset() + assert FileBasedMetadata.supported_fields == vocabulary + assert OpaqueMetadata.supported_fields == frozenset() + assert OpaqueMetadata.unsupported_fields == vocabulary + for name in LiteMetadata.unsupported_fields: + field = next(f for f in fields(LiteMetadata) if f.name == name) + assert field.default is UNSUPPORTED + + +def test_supports_reads_the_class_declaration() -> None: + assert LiteMetadata.supports("description") + assert not LiteMetadata.supports("echo_time") + assert LiteMetadata.supports("extra") + with pytest.raises(KeyError): + LiteMetadata.supports("not_a_field") + + +def test_per_instance_capability() -> None: + assert DialectMetadata.supports("channels") + assert DialectMetadata().channels is not UNSUPPORTED + plain = DialectMetadata(dialect="plain") + assert plain.channels is UNSUPPORTED + # `replace` runs `__post_init__` again, so the rule holds. + assert replace(DialectMetadata(), dialect="plain").channels is UNSUPPORTED + + +def test_a_subclass_inherits_and_may_change_its_capabilities() -> None: + class Inherits(LiteMetadata): + pass + + class Widens(LiteMetadata, supports=("description", "echo_time")): + pass + + assert Inherits.unsupported_fields == LiteMetadata.unsupported_fields + assert Widens.supports("echo_time") + assert not Widens.supports("slice_timing") + assert Widens(echo_time=0.03).echo_time == 0.03 + assert Widens().echo_time is None + + +def test_a_field_may_be_declared_unsupported_by_hand() -> None: + class ByHand(Metadata): + echo_time: tx.Optional[float] = UNSUPPORTED + + assert ByHand.unsupported_fields == {"echo_time"} + + +def test_supports_all() -> None: + class Everything(FileBasedMetadata, supports=ALL): + pass + + assert Everything.unsupported_fields == frozenset() + + +def test_wrong_declarations_are_refused() -> None: + with pytest.raises(TypeError, match="not vocabulary"): + + class Typo(FileBasedMetadata, supports=("descr",)): + pass + + with pytest.raises(TypeError, match="derived"): + + class Derived(FileBasedMetadata, derived=("description",)): + pass + + with pytest.raises(TypeError, match="typo_kw"): + + class Misspelt(FileBasedMetadata, typo_kw=1): + pass + + +def test_polymorphic_construction_on_format() -> None: + meta = Metadata(format="test-lite", description="x") + assert type(meta) is LiteMetadata + assert type(Metadata(description="x")) is Metadata + assert type(Metadata(format="opaque")) is OpaqueMetadata + # An unknown format falls back to the root, the generic metadata. + assert type(Metadata(format="unknown")) is Metadata + with pytest.raises(ValueError): + LiteMetadata(format="test-dict") + + +def test_bids_and_scope_annotations() -> None: + by_name = {f.name: f.metadata for f in fields(Metadata)} + assert by_name["repetition_time"]["bids"] == "RepetitionTime" + assert by_name["repetition_time"]["scope"] is Scope.ACQUISITION + assert by_name["slice_timing"]["scope"] is Scope.SPATIAL + assert by_name["channels"]["scope"] is Scope.AXIS + assert by_name["channels"]["along"] is AxisType.channel + assert by_name["bvalues"]["along"] is AxisType.time + assert by_name["display_range"]["scope"] is Scope.FILE + with pytest.raises(ValueError): + Scoped(Scope.AXIS) + with pytest.raises(ValueError): + Along("space") + assert "bids" not in by_name["display_range"] + + +# ---------------------------------------------------------------------- +# SNAPSHOT AND CHANGE-DETECTING WRITE (memo, section 6) +# ---------------------------------------------------------------------- + + +def _read(record=None) -> DictMetadata: # noqa: ANN001 + if record is None: + record = {"desc": "short", "cal": (0.0, 1.0), "Key": "v"} + return DictMetadata.from_raw(record) + + +def test_reading_decodes_and_snapshots() -> None: + meta = _read() + assert meta.description == "short" + assert meta.display_range == (0.0, 1.0) + assert meta.extra == {"Key": "v"} + assert meta._changed_fields() == {} + + +def test_case1_untouched_writes_the_record_as_read() -> None: + meta = _read() + report = ConversionReport() + record = meta.update_raw(on_loss=report) + assert record == {"desc": "short", "cal": (0.0, 1.0), "Key": "v"} + assert record is not meta.raw # written over a copy + assert not report.lossy + + +def test_case2_a_record_edit_survives() -> None: + meta = _read() + meta.raw["desc"] = "edited" + assert meta.update_raw()["desc"] == "edited" + + +def test_case3_a_common_field_set_wins() -> None: + meta = _read() + meta.raw["desc"] = "edited" + meta.description = "mine" + assert meta._changed_fields() == {"description": "mine"} + assert meta.update_raw()["desc"] == "mine" + + +def test_case4_none_clears_the_slot() -> None: + meta = _read() + meta.description = None + meta.display_range = None + assert meta._changed_fields() == { + "description": None, + "display_range": None, + } + record = meta.update_raw() + assert "desc" not in record and "cal" not in record + + +def test_extra_is_compared_key_by_key() -> None: + meta = _read({"desc": "d", "Kept": 1, "Gone": 2, "Edited": 3}) + meta.extra = {"Kept": 1, "Edited": 4, "New": 5} + assert meta._changed_fields() == { + "extra": {"Edited": 4, "New": 5, "Gone": None} + } + assert meta.update_raw() == {"desc": "d", "Kept": 1, "Edited": 4, "New": 5} + + +def test_a_snapshot_holds_converted_values() -> None: + # A list decoded into a tuple field is not a change. + meta = DictMetadata.from_raw({"slices": [0, 1, 2]}) + assert meta.slice_timing == (0.0, 1.0, 2.0) + assert meta._changed_fields() == {} + + +def test_an_object_built_in_memory_has_everything_changed() -> None: + meta = DictMetadata(description="d", display_range=(1, 2)) + assert meta._changed_fields() == { + "description": "d", + "display_range": (1.0, 2.0), + } + assert meta.update_raw() == {"desc": "d", "cal": (1.0, 2.0)} + + +def test_the_snapshot_survives_replace_copy_and_pickle() -> None: + meta = _read() + for other in ( + replace(meta, description="x"), + copy.deepcopy(meta), + pickle.loads(pickle.dumps(meta)), + ): + assert other._snapshot == meta._snapshot + assert replace(meta, description="x")._changed_fields() == { + "description": "x" + } + + +def test_value_dependent_loss_is_reported_by_the_encoder() -> None: + meta = DictMetadata(description="much too long", slice_timing=(0, 1)) + report = meta.check_writable(image=3) + assert report.lost == {"slice_timing": (0.0, 1.0)} + assert report.approximated == {"description": "truncated to 8"} + assert not meta.check_writable(image=2).lost + + +def test_an_unsupported_field_assigned_later_is_reported_at_write() -> None: + meta = LiteMetadata() + meta.echo_time = 0.03 # attribute assignment is not validated + report = meta.check_writable() + assert report.lost == {"echo_time": 0.03} + + +# ---------------------------------------------------------------------- +# CONVERSION AND LOSS REPORTS (memo, section 7) +# ---------------------------------------------------------------------- + + +def test_conversion_reports_what_the_target_cannot_hold() -> None: + target, report = _to(_rich(), LiteMetadata) + assert type(target) is LiteMetadata + assert target.description == "a scan" + assert target.slice_timing == (0.0, 0.5, 1.0) + assert target.extra == {"Custom": 1} + assert report.lost == {"echo_time": 0.03} + assert (report.source, report.target) == ("generic", "test-lite") + assert "echo_time" in str(report) + + +def test_conversion_accepts_a_format_name() -> None: + target, _ = _to(_rich(), "test-lite") + assert type(target) is LiteMetadata + + +def test_conversion_through_the_hub_loses_what_a_direct_one_does() -> None: + lite = LiteMetadata(description="d", history=("h",), extra={"k": 1}) + hub, report = _to(lite, Metadata) + assert not report.lossy + # UNSUPPORTED on the source side reads as None. + assert hub.echo_time is None + _, direct = _to(lite, DictMetadata) + _, via_hub = _to(hub, DictMetadata) + assert direct.lost == via_hub.lost == {"history": ("h",)} + + +def test_the_record_travels_only_within_a_format() -> None: + meta = _read() + other, _ = _to(meta, Metadata) + assert type(other) is Metadata + # The hub carries the record and the snapshot... + assert other.raw is meta.raw and other._snapshot == meta._snapshot + # ... back to the format whose type of record it is, not to another. + assert _to(other, DictMetadata)[0].raw is meta.raw + assert _to(other, LiteMetadata)[0].raw is None + same = DictMetadata.from_other(meta) + assert same.raw is meta.raw and same._snapshot == meta._snapshot + # A copy keeps the most specific class. + assert type(FileBasedMetadata.from_other(meta)) is DictMetadata + + +def test_extra_is_lost_where_the_target_has_no_store() -> None: + _, report = _to(Metadata(extra={"Key": 1}), OpaqueMetadata) + assert report.lost == {"extra": {"Key": 1}} + + +def test_explicit_values_win_over_the_source() -> None: + target, _ = _to(_rich(), Metadata, description="other") + assert target.description == "other" + + +# ---------------------------------------------------------------------- +# POLICIES +# ---------------------------------------------------------------------- + + +def test_the_default_policy_warns_once_per_conversion() -> None: + source = Metadata( + echo_time=0.03, flip_angle=90.0, magnetic_field_strength=3.0 + ) + with pytest.warns(MetadataLossWarning) as record: + LiteMetadata.from_other(source) + assert len(record) == 1 + report = record[0].message.report + assert set(report.lost) == { + "echo_time", + "flip_angle", + "magnetic_field_strength", + } + + +def test_ignore_is_silent_and_raise_raises() -> None: + with warnings.catch_warnings(): + warnings.simplefilter("error") + _rich().to(LiteMetadata, on_loss="ignore") + # Nothing lost: silent whatever the policy. + Metadata(description="d").to(LiteMetadata, on_loss="raise") + with pytest.raises(MetadataLossError) as info: + _rich().to(LiteMetadata, on_loss="raise") + assert info.value.report.lost == {"echo_time": 0.03} + + +def test_the_policy_context_manager_nests_and_restores() -> None: + with metadata_loss_policy("raise"): + with metadata_loss_policy("ignore"): + LiteMetadata.from_other(_rich()) + with pytest.raises(MetadataLossError): + LiteMetadata.from_other(_rich()) + with pytest.warns(MetadataLossWarning): + LiteMetadata.from_other(_rich()) + with pytest.raises(ValueError): + with metadata_loss_policy("loud"): + pass + + +def test_the_policy_governs_implicit_field_conversions() -> None: + class Holder(DataModelBase): + meta: tx.Optional[LiteMetadata] = None + + with metadata_loss_policy("raise"): + with pytest.raises(MetadataLossError): + Holder(meta=_rich()) + with pytest.warns(MetadataLossWarning): + held = Holder(meta=_rich()) + assert type(held.meta) is LiteMetadata + + +def test_a_report_raises_and_merges() -> None: + report = ConversionReport(source="a", target="b") + report.raise_if_lossy() + other = ConversionReport(lost={"x": 1}) + report.merge(other) + assert report.lost == {"x": 1} + with pytest.raises(MetadataLossError): + report.raise_if_lossy() + with pytest.raises(MetadataLossError): + apply_loss_policy(report, "raise") + + +# ---------------------------------------------------------------------- +# DERIVE (memo, section 9) +# ---------------------------------------------------------------------- + + +def _scoped() -> Metadata: + import datetime + + return Metadata( + description="d", + creation_time=datetime.datetime(2020, 1, 1), + history=("acquired",), + echo_time=0.03, + slice_timing=(0.0, 0.5), + phase_encoding_direction="j-", + channels=(Channel(name="a"), Channel(name="b"), Channel(name="c")), + bvalues=(0, 1000, 2000), + display_range=(0, 1), + extra={"Key": 1}, + ) + + +def test_derive_records_provenance() -> None: + meta = _scoped() + derived = meta.derive(history="smooth") + assert derived is not meta + assert derived.description == "d" + assert derived.creation_time is None + assert derived.history == ("acquired", "smooth") + assert derived.echo_time == 0.03 + assert derived.slice_timing == (0.0, 0.5) + assert derived.phase_encoding_direction == EncodingDirection("j-") + assert derived.bvalues == (0.0, 1000.0, 2000.0) + assert derived.extra == {"Key": 1} + assert derived.extra is not meta.extra + assert [g.name for g in derived.generated_by] == ["brainhops"] + # The brainhops entry is added once. + assert len(derived.derive().generated_by) == 1 + # A string is one entry; a sequence, several; `None`, none. + assert meta.derive().history == ("acquired",) + assert meta.derive(history=["a", "b"]).history == ("acquired", "a", "b") + + +def _dwi_system() -> CoordinateSystem: + """The voxel axes of a 4-D diffusion series: three spatial axes and a + time axis.""" + return CoordinateSystem( + axes=[ + Axis("i", "space"), + Axis("j", "space"), + Axis("k", "space"), + Axis("t", "time"), + ] + ) + + +def _indexed(*index) -> Indexed: # noqa: ANN002 + """`image[index]`, for a diffusion series of shape (4, 5, 6, 3).""" + return Indexed(index, (4, 5, 6, 3), _dwi_system()) + + +class _Mapped(Operation): + """An operation the handlers do not know, which moves the spatial axes + through a given voxel map.""" + + matrix: tx.Any = None + + @property + def voxel_map(self) -> tx.Optional[np.ndarray]: + if self.matrix is None: + return None + return np.asarray(self.matrix, dtype=float) + + +def test_indexed_expands_its_index() -> None: + op = _indexed(Ellipsis, 1) + assert op.expanded == (slice(None),) * 3 + (1,) + assert op.axes == (AxisType.space,) * 3 + (AxisType.time,) + assert Indexed(0, (4, 5)).index == (0,) + assert Indexed((0,), (4, 5)).axes == (None, None) + + +def test_indexed_positions_along_an_axis() -> None: + assert list(_indexed(Ellipsis, [2, 0]).positions("time")) == [2, 0] + assert list(_indexed(Ellipsis, slice(1, 3)).positions(AxisType.time)) == [ + 1, + 2, + ] + # Untouched: the full range. + assert list(_indexed(slice(None)).positions("time")) == [0, 1, 2] + # A boolean mask keeps the positions it selects. + mask = np.array([True, False, True]) + assert list(_indexed(Ellipsis, mask).positions("time")) == [0, 2] + # A dropped axis, a selection out of range or not 1-D, or an axis the + # image does not have: no positions. + assert _indexed(Ellipsis, 1).positions("time") is None + assert _indexed(Ellipsis, [5]).positions("time") is None + assert _indexed(Ellipsis, np.array([[0, 1]])).positions("time") is None + assert _indexed(Ellipsis).positions("channel") is None + + +def test_indexed_moves_space() -> None: + assert not _indexed(Ellipsis, slice(0, 2)).moves_space + assert not _indexed(Ellipsis, 1).moves_space + assert not _indexed(slice(None), slice(0, 5)).moves_space + assert _indexed(slice(None), slice(None, None, -1)).moves_space + assert _indexed(None).moves_space + assert _indexed(2).moves_space + # An axis of unknown type is taken to be spatial. + assert Indexed((Ellipsis, 1), (4, 5, 6, 3)).moves_space + + +def test_indexed_voxel_map() -> None: + flip = _indexed(slice(None), slice(None, None, -1)).voxel_map + assert np.allclose(flip, np.diag([1.0, -1.0, 1.0, 1.0])) + # A dropped axis has a zero column. + dropped = _indexed(slice(None), 2).voxel_map + assert dropped.shape == (3, 4) + assert np.allclose(dropped[:, 1], 0) + # An index other than integers, slices and `None` has no map. + assert _indexed(Ellipsis, [0, 2]).voxel_map is None + + +def test_derive_indexed_selects_the_entries_along_the_axis() -> None: + meta = _scoped() + selected = meta.derive(_indexed(Ellipsis, [2, 0]), history="getitem") + assert selected.bvalues == (2000.0, 0.0) + assert meta.derive(_indexed(Ellipsis, slice(1, 3))).bvalues == ( + 1000.0, + 2000.0, + ) + assert meta.derive(_indexed(Ellipsis, slice(0, 0))).bvalues == () + # A dropped axis, or a position beyond the field, clears it. + assert meta.derive(_indexed(Ellipsis, 1)).bvalues is None + short = Indexed((Ellipsis, [5]), (4, 5, 6, 8), _dwi_system()) + assert meta.derive(short).bvalues is None + # The other axes, and the other fields, are untouched. + assert [c.name for c in selected.channels] == ["a", "b", "c"] + assert selected.slice_timing == (0.0, 0.5) + assert selected.phase_encoding_direction == EncodingDirection("j-") + assert selected.display_range == (0.0, 1.0) + assert selected.history == ("acquired", "getitem") + assert selected.creation_time is None + channels = CoordinateSystem( + axes=[Axis("i", "space"), Axis("c", "channel")] + ) + channel = meta.derive(Indexed((Ellipsis, [2, 0]), (4, 3), channels)) + assert [c.name for c in channel.channels] == ["c", "a"] + assert channel.bvalues == (0.0, 1000.0, 2000.0) + # An image without an axis of that type keeps the field. + assert meta.derive(Indexed((0,), (4, 3))).bvalues == meta.bvalues + + +def test_derive_maps_directions_through_the_voxel_map() -> None: + meta = Metadata( + phase_encoding_direction="j-", + slice_encoding_direction=EncodingDirection((0, 0, 1), space="mni"), + slice_timing=(0.0, 0.5), + echo_time=0.03, + bvalues=(0, 1000), + ) + swap = np.array([[0, 1, 0], [1, 0, 0], [0, 0, 1]]) + derived = meta.derive(_Mapped(swap), history="reslice") + assert derived.phase_encoding_direction == EncodingDirection("i-") + # A direction in a world space does not move with the voxels. + assert derived.slice_encoding_direction == EncodingDirection( + (0, 0, 1), space="mni" + ) + # An operation the handlers do not know falls back on the scope + # defaults: the slice timing is cleared, the rest is kept. + assert derived.slice_timing is None + assert derived.echo_time == 0.03 + assert derived.bvalues == (0.0, 1000.0) + assert derived.history == ("reslice",) + flip = np.diag([1.0, -1.0, 1.0]) + assert meta.derive(_Mapped(flip)).phase_encoding_direction == ( + EncodingDirection("j") + ) + # The map of a 4-D image: the direction lies in its first three axes. + flip4 = np.diag([1.0, -1.0, 1.0, 1.0]) + assert meta.derive(_Mapped(flip4)).phase_encoding_direction == ( + EncodingDirection("j") + ) + onto_time = np.eye(4)[[0, 3, 2, 1]] + assert meta.derive(_Mapped(onto_time)).phase_encoding_direction is None + rotated = meta.derive( + _Mapped([[1, 0, 0], [0, 2**-0.5, -(2**-0.5)], [0, 2**-0.5, 2**-0.5]]) + ) + assert rotated.phase_encoding_direction.to_bids() is None + # Without a map, a change of the spatial axes clears it. + cleared = meta.derive(_Mapped()) + assert cleared.phase_encoding_direction is None + assert cleared.slice_encoding_direction.space == "mni" + assert cleared.slice_timing is None + # Through an index of the voxels. + indexed = meta.derive(_indexed(slice(None), slice(None, None, -1))) + assert indexed.phase_encoding_direction == EncodingDirection("j") + assert meta.derive(_indexed(slice(None), 2)).phase_encoding_direction is ( + None + ) + + +def test_derive_resampled() -> None: + meta = Metadata( + phase_encoding_direction="j-", + slice_encoding_direction=EncodingDirection((0, 0, 1), space="mni"), + slice_timing=(0.0, 0.5), + bvalues=(0, 1000), + ) + # The transformation goes from the new voxels to the old ones. + swap = Affine(np.eye(4)[[1, 0, 2, 3]][:-1]) + resampled = Resampled(swap, Geometry()) + assert resampled.moves_space + assert np.allclose(resampled.voxel_map, np.eye(3)[[1, 0, 2]]) + derived = meta.derive(resampled) + assert derived.phase_encoding_direction == EncodingDirection("i-") + assert derived.slice_timing is None + assert derived.bvalues == (0.0, 1000.0) + # Without an affine form, a direction in voxel axes is cleared, and + # one in a world space is kept. + warped = Resampled(CartesianField()) + assert warped.geometry is None + assert warped.voxel_map is None + derived = meta.derive(warped) + assert derived.phase_encoding_direction is None + assert derived.slice_encoding_direction.space == "mni" + + +def test_derive_without_an_operation_keeps_every_field() -> None: + meta = _scoped() + assert meta.derive() == meta.derive(None) + assert meta.derive().slice_timing == meta.slice_timing + assert meta.derive().phase_encoding_direction == ( + meta.phase_encoding_direction + ) + + +def test_the_handler_of_the_record_scrubs_it() -> None: + meta = DictMetadata.from_raw( + {"desc": "d", "slices": [0, 1], "slice_hint": "x"} + ) + derived = meta.derive(_Mapped()) + assert type(derived) is DictMetadata + assert derived._snapshot == meta._snapshot + assert "slice_hint" not in derived.raw and "slice_hint" in meta.raw + # The cleared field differs from the snapshot: it is cleared on write. + assert derived._changed_fields()["slice_timing"] is None + assert "slices" not in derived.update_raw() + # Unsupported fields stay unsupported. + assert derived.echo_time is UNSUPPORTED + # An index along time keeps the spatial content of the record. + kept = meta.derive(_indexed(Ellipsis, 0)) + assert kept.raw["slice_hint"] == "x" and kept.raw is not meta.raw + + +class _Marked(str): + """A value type with handlers of its own, for the lookup order.""" + + +@propagates(_Marked, Operation) +def _marked_by_operation(value, operation, *, name, source): # noqa: ANN001, ANN202 + return _Marked("operation") + + +@propagates(_Marked, Indexed) +def _marked_by_index(value, operation, *, name, source): # noqa: ANN001, ANN202 + return _Marked("indexed") + + +def test_the_lookup_order() -> None: + meta = _scoped() + + def through(value, operation, scope=Scope.SPATIAL): # noqa: ANN001, ANN202 + return propagate( + value, operation, name="slice_timing", scope=scope, source=meta + ) + + # The type of the value beats the scope, which would clear it. + assert through(_Marked("x"), _Mapped()) == "operation" + assert through("x", _Mapped()) is None + # The most specific operation first. + assert through(_Marked("x"), _indexed(Ellipsis)) == "indexed" + # No handler: the value is kept; `UNSUPPORTED` always is. + assert through("x", _Mapped(), scope="no scope") == "x" + assert through(UNSUPPORTED, _Mapped()) is UNSUPPORTED + with pytest.raises(TypeError): + propagates(object, Operation) + with pytest.raises(TypeError): + propagates(_Marked, int) + + +def test_the_metadata_imports_before_the_transformations() -> None: + # The operations import the transformations lazily: the + # transformations import the metadata. + code = ( + "import brainhops.datamodel.metadata as m, numpy as np\n" + "from brainhops.datamodel.transformations import Affine\n" + "from brainhops.datamodel.geometry import Geometry\n" + "r = m.Resampled(Affine(np.eye(3)[:2]), Geometry())\n" + "assert r.voxel_map.shape == (2, 2)\n" + ) + subprocess.run([sys.executable, "-c", code], check=True) + + +def test_derive_copies_the_record_by_default() -> None: + meta = LiteMetadata.from_raw({"key": [1]}) + derived = meta.derive() + assert derived.raw == meta.raw and derived.raw is not meta.raw + + +# ---------------------------------------------------------------------- +# BIDS SIDECAR +# ---------------------------------------------------------------------- + + +_SIDECAR = { + "RepetitionTime": 2.0, + "EchoTime": 0.03, + "SliceTiming": [0.0, 0.5, 1.0, 1.5], + "PhaseEncodingDirection": "j-", + "Description": "a bold run", + "Sources": ["bids:raw:sub-01_T1w.nii.gz"], + "GeneratedBy": [{"Name": "fMRIPrep", "Version": "23.0"}], + "TaskName": "rest", + "DisplayRange": [0, 255], +} + + +def test_a_sidecar_round_trips() -> None: + meta = from_bids(_SIDECAR) + assert meta.repetition_time == 2.0 + assert meta.slice_timing == (0.0, 0.5, 1.0, 1.5) + assert meta.generated_by == (GeneratedBy(name="fMRIPrep", version="23.0"),) + assert meta.display_range == (0.0, 255.0) + # A key that names no vocabulary field lands in `extra`. + assert meta.extra == {"TaskName": "rest"} + sidecar = to_bids(meta) + assert sidecar == { + **_SIDECAR, + "DisplayRange": [0.0, 255.0], + } + json.dumps(sidecar) + + +def test_a_sidecar_is_read_from_a_path_or_a_string(tmp_path) -> None: # noqa: ANN001 + path = tmp_path / "sub-01_bold.json" + path.write_text(json.dumps(_SIDECAR)) + assert from_bids(path) == from_bids(_SIDECAR) + assert from_bids(str(path)) == from_bids(_SIDECAR) + assert from_bids(json.dumps(_SIDECAR)) == from_bids(_SIDECAR) + + +def test_diffusion_is_not_a_sidecar_key() -> None: + meta = Metadata(bvalues=(0, 1000), description="dwi") + with pytest.warns(MetadataLossWarning): + sidecar = to_bids(meta) + assert sidecar == {"Description": "dwi"} + with pytest.raises(MetadataLossError): + to_bids(meta, on_loss="raise") + + +def test_times_are_iso_strings() -> None: + meta = from_bids({"AcquisitionTime": "2020-01-02T03:04:05"}) + assert meta.acquisition_time.year == 2020 + assert to_bids(meta) == {"AcquisitionTime": "2020-01-02T03:04:05"} + + +# ---------------------------------------------------------------------- +# THE DATA MODEL FIELD +# ---------------------------------------------------------------------- + + +def test_a_copy_shares_the_record_and_copies_the_snapshot() -> None: + meta = DictMetadata.from_raw({"desc": "read", "Lab": "x"}) + other = meta.copy() + assert other.raw is meta.raw + assert other == meta + other.description = "changed" + other._snapshot["description"] = "forged" + assert meta._changed_fields() == {} + assert meta._snapshot["description"] == "read" + + +def test_the_hub_holds_any_record_and_opaque_none() -> None: + assert Metadata().raw is None + assert Metadata(raw={"a": 1}).raw == {"a": 1} + assert OpaqueMetadata().raw is None + with pytest.raises(TypeError): + OpaqueMetadata(raw={"a": 1}) + + +def test_a_decoder_may_not_return_an_unsupported_field() -> None: + class Wrong( + FileBasedMetadata, + on={"format": "test-wrong"}, + supports=("description",), + ): + @classmethod + def _decode_raw(cls, raw, *, image=None) -> dict: # noqa: ANN001 + return {"description": "d", "echo_time": 0.03} + + with pytest.raises(TypeError, match="echo_time"): + Wrong.from_raw({}) + + +# ---------------------------------------------------------------------- +# A NEW RECORD, FORCED FIELDS, DERIVED FIELDS +# ---------------------------------------------------------------------- + + +def test_update_from_raw_keeps_the_changes_over_a_new_record() -> None: + meta = DictMetadata.from_raw({"desc": "old", "cal": (0, 1), "A": 1}) + meta.description = "mine" + meta.display_range = None + meta.extra = {"A": 1, "B": 2} + new = meta.update_from_raw({"desc": "new", "cal": (2, 3), "C": 3}) + assert new.description == "mine" + assert new.display_range is None + assert new.extra == {"C": 3, "B": 2} + assert new._changed_fields() == { + "description": "mine", + "display_range": None, + "extra": {"B": 2}, + } + + +def test_force_writes_a_field_equal_to_the_snapshot() -> None: + meta = DictMetadata.from_raw({"desc": "read"}) + meta.raw["desc"] = "record edit" + assert meta.update_raw(dict(meta.raw))["desc"] == "record edit" + forced = meta.update_raw(dict(meta.raw), force=("description",)) + assert forced["desc"] == "read" + meta.description = None + assert "desc" not in meta.update_raw( + dict(meta.raw), force=("description",) + ) + + +class GeoMetadata( + FileBasedMetadata[dict], + on={"format": "test-geo"}, + supports=("repetition_time",), +): + """A format whose `repetition_time` is the image's time step (the + image is a number here), or the record's when it has none.""" + + @classmethod + def _decode_raw(cls, raw, *, image=None) -> dict: # noqa: ANN001 + if image is not None: + return {"repetition_time": image} + return {"repetition_time": (raw or {}).get("tr")} + + def _encode_raw(self, raw, changed, *, image=None, report) -> dict: # noqa: ANN001 + # The time step of an image is the writer's to store. + if "repetition_time" in changed and image is None: + raw["tr"] = changed["repetition_time"] + return raw + + +def test_a_field_the_record_does_not_hold_is_reported() -> None: + meta = GeoMetadata(repetition_time=2.0) + report = ConversionReport() + raw = meta.update_raw(image=2.0, on_loss=report) + assert raw == {} + meta.check_raw(raw, image=2.0, on_loss=report) + assert not report.lossy + report = ConversionReport() + raw = meta.update_raw(image=1.5, on_loss=report) + assert raw == {} + meta.check_raw(raw, image=1.5, on_loss=report) + assert set(report.approximated) == {"repetition_time"} + assert "1.5" in report.approximated["repetition_time"] + # The data model says nothing: the value is the format's to write. + raw = meta.update_raw(image=None) + assert raw == {"tr": 2.0} + assert not meta.check_raw(raw, on_loss="ignore").lossy + assert meta.check_writable(image=1.5).approximated + assert not meta.check_writable(image=2.0).lossy + # Numbers, and sequences of numbers, agree within rounding. + assert not meta.check_raw({"tr": 2.0 + 1e-9}, on_loss="ignore").lossy + + +# ---------------------------------------------------------------------- +# ONE WARNING +# ---------------------------------------------------------------------- + + +def test_collected_reports_merge_into_one() -> None: + with warnings.catch_warnings(): + warnings.simplefilter("error") + with collect_loss_reports() as reports: + apply_loss_policy(ConversionReport(source="a", lost={"x": 1})) + apply_loss_policy( + ConversionReport(target="b", approximated={"y": "z"}) + ) + apply_loss_policy(ConversionReport(lost={"q": 1}), "ignore") + assert len(reports) == 2 + report = ConversionReport.merged(reports) + assert (report.source, report.target) == ("a", "b") + assert report.lost == {"x": 1} and report.approximated == {"y": "z"} + with pytest.raises(MetadataLossError): + with collect_loss_reports(): + apply_loss_policy(ConversionReport(lost={"x": 1}), "raise") + + +# ---------------------------------------------------------------------- +# HIERARCHY AND VOCABULARY GROUPS +# ---------------------------------------------------------------------- + + +def test_the_hierarchy_mirrors_the_images() -> None: + assert issubclass(FileBasedMetadata, Metadata) + assert issubclass(OpaqueMetadata, FileBasedMetadata) + assert issubclass(LiteMetadata, FileBasedMetadata) + for group in GROUPS: + assert issubclass(Metadata, group) + assert issubclass(group, Vocabulary) + # The groups are public: `supports=` names them. + assert group.__name__ in brainhops.datamodel.metadata.__all__ + # Every class has a raw record and a snapshot; reading and writing + # them is what a file-based class adds. + names = {f.name for f in fields(Metadata)} + assert {"raw", "_snapshot"} <= names + assert not hasattr(Metadata, "from_raw") + # No input or output on the data model: `FileBasedMetadata.load` + # reads a file, and `brainhops.io.metadata.bids` the sidecars. + for name in ("load", "from_bids", "to_bids"): + assert not hasattr(Metadata, name) + + +def test_the_record_a_class_keeps() -> None: + from brainhops.datamodel.metadata._base import _accepts_raw + + # Generic metadata declares no type, and keeps any record. + assert Metadata._raw_class is None + assert _accepts_raw(Metadata, object()) + # The base of the formats declares none until a format does, and a + # format keeps only a record of the type it declares. + assert FileBasedMetadata._raw_class is type(None) + assert not _accepts_raw(FileBasedMetadata, object()) + assert not _accepts_raw(OpaqueMetadata, {}) + assert OpaqueMetadata()._raw_or_default() is None + + +def test_the_vocabulary_is_the_groups_in_order() -> None: + assert list(GROUPS) == [ + ProvenanceVocabulary, + MRIVocabulary, + DiffusionVocabulary, + DisplayVocabulary, + StorageVocabulary, + MicroscopyVocabulary, + TransformVocabulary, + ] + assert VOCABULARY == sum(GROUPS.values(), ()) + assert VOCABULARY[:2] == ("name", "description") + assert GROUPS[DiffusionVocabulary] == ("bvalues", "bvectors") + assert "data_type" in GROUPS[StorageVocabulary] + assert "extra" not in VOCABULARY + assert len(set(VOCABULARY)) == len(VOCABULARY) + # A group's fields convert, as the class that inherits them does. + assert Metadata(echo_time="0.03", bvalues=[0, 1000]).bvalues == ( + 0.0, + 1000.0, + ) + + +def test_supports_takes_groups() -> None: + class ByGroup( + FileBasedMetadata, + on={"format": "test-group"}, + supports=(ProvenanceVocabulary, "echo_time"), + ): + pass + + assert ByGroup.supported_fields == set( + GROUPS[ProvenanceVocabulary] + ("echo_time",) + ) + assert ByGroup.supports("history") and not ByGroup.supports("bvalues") + with pytest.raises(TypeError, match="not a vocabulary group"): + + class NotAGroup(FileBasedMetadata, supports=(Channel,)): + pass + + +def test_a_new_field_is_unsupported_until_a_format_opts_in() -> None: + # `supports=` lists what a format stores: everything else, including + # what the vocabulary gains later, defaults to UNSUPPORTED. + assert LiteMetadata.supported_fields <= set(VOCABULARY) | {"extra"} + assert "data_type" in LiteMetadata.unsupported_fields + + +# ---------------------------------------------------------------------- +# TERMS +# ---------------------------------------------------------------------- + + +def test_known_terms_become_enum_members() -> None: + meta = Metadata( + space="MNI152NLin6Asym", + intent="label", + manufacturer="Siemens", + illumination_type="Epifluorescence", + contrast_method="DIC", + input_space="scanner", + ) + assert meta.space is SpaceEnum.MNI152NLin6Asym + assert meta.intent is IntentEnum.label + assert meta.manufacturer is Manufacturer.Siemens + assert meta.illumination_type is IlluminationType.Epifluorescence + assert meta.contrast_method is ContrastMethod.DIC + assert meta.input_space is SpaceEnum.scanner + # They are strings, and compare equal to their value. + assert meta.space == "MNI152NLin6Asym" and meta.intent == "label" + # An unknown term stays a string: the vocabulary is not closed. + meta.space = "my-template" + assert type(meta.space) is str and meta.space == "my-template" + assert Metadata(intent=UNSUPPORTED).intent is UNSUPPORTED + with pytest.raises((TypeError, ValueError)): + Metadata(space=3) + assert json.dumps(to_bids(Metadata(space="mni"))) == ( + '{"SpatialReference": "mni"}' + ) + + +def test_data_unit_is_a_unit_when_known() -> None: + from brainhops.datamodel.units import Unit + + meta = Metadata(data_unit="ms") + assert isinstance(meta.data_unit, Unit) + assert meta.data_unit == Unit("millisecond") + # Any name the units module parses is a `Unit`, compared as a unit. + assert Metadata(data_unit="a.u.").data_unit == Unit("au") + assert Metadata(data_unit="mm/s").data_unit == Unit("millimeter/second") + # A name it cannot parse stays the file's own string. + assert Metadata(data_unit="mm2/s").data_unit == "mm2/s" + with pytest.raises(TypeError): + Metadata(data_unit=3) + # Written as its symbol, which parses back to the same unit. + assert to_bids(meta) == {"DataUnit": "ms"} + for name in ("a.u.", "mm/s", "degC", "uV", "HU"): + bids = to_bids(Metadata(data_unit=name)) + assert from_bids(bids).data_unit == Unit(name) + assert to_bids(Metadata(data_unit="mm2/s")) == {"DataUnit": "mm2/s"} + + +def test_data_type_is_a_native_dtype() -> None: + meta = Metadata(data_type=">i2") + assert meta.data_type == np.dtype("int16") + assert meta.data_type.isnative + assert to_bids(meta) == {"DataType": "int16"} + assert from_bids({"DataType": "uint8"}).data_type == np.uint8 + # A resampling changes the kind of the values: it is grid-bound. + # How the file stores the values: kept by `derive` (`file` scope). + assert meta.derive(_Mapped()).data_type == np.int16 + assert meta.derive(_indexed(Ellipsis, 0)).data_type == np.int16 + + +def test_stored_values_round_only_into_integers() -> None: + values = np.array([1.0, 2.2, 3.6]) + # Into an integer type, the unscaled values are rounded... + stored = stored_values(values, np.int16, 2.0, 1.0) + assert stored.dtype == np.int16 + assert stored.tolist() == [0, 1, 1] + # ... into a floating-point type, they are kept as they are. + stored = stored_values(values, np.float32, 2.0, 1.0) + assert stored.dtype == np.float32 + assert stored.tolist() == pytest.approx([0.0, 0.6, 1.3]) + assert stored_values(values, np.float64).tolist() == values.tolist() + + +def test_preferred_dtype() -> None: + labels = Metadata(data_type="uint8") + # The data type wins when the values are of its kind... + assert preferred_dtype(labels, np.int64) == np.uint8 + assert preferred_dtype(labels, np.bool_) == np.uint8 + assert preferred_dtype(Metadata(data_type="f4"), np.float64) == np.float32 + # ... an explicit dtype wins over it ... + assert preferred_dtype(labels, np.int64, "int16") == np.int16 + # ... and floats are never quantised into it, nor integers made floats. + report = ConversionReport() + assert preferred_dtype(labels, np.float64, on_loss=report) == np.float64 + assert "data_type" in report.approximated + with pytest.warns(MetadataLossWarning): + # With no report, the policy in effect. + assert preferred_dtype(Metadata(data_type="f4"), np.int16) == np.int16 + assert preferred_dtype(Metadata(), np.int16) == np.int16 + # A data type that was only read is dropped silently. + + class Typed( + FileBasedMetadata, on={"format": "test-typed"}, supports=("data_type",) + ): + @classmethod + def _decode_raw(cls, raw, *, image=None) -> dict: # noqa: ANN001 + return {"data_type": raw} + + read = Typed.from_raw("uint8") + assert read.data_type == np.uint8 and not read._changed_fields() + report = ConversionReport() + preferred_dtype(read, np.float64, on_loss=report) + assert not report.lossy + + +# ---------------------------------------------------------------------- +# ENCODING DIRECTIONS +# ---------------------------------------------------------------------- + + +def test_an_encoding_direction_is_a_vector_in_voxel_axes() -> None: + direction = EncodingDirection("j-") + assert direction.vector == (0.0, -1.0, 0.0) + assert direction.space is None + assert direction.to_bids() == "j-" + assert direction == EncodingDirection((0, -1, 0)) + assert direction != EncodingDirection("j") + # Equality is the fields': a direction is not its BIDS string. + assert direction != "j-" + assert repr(direction) == "EncodingDirection('j-')" + assert EncodingDirection("k") == EncodingDirection((0, 0, 2)) + with pytest.raises(ValueError): + EncodingDirection("x") + with pytest.raises(ValueError): + EncodingDirection((0, 0, 0)) + oblique = EncodingDirection((1, 1, 0)) + assert oblique.vector == pytest.approx((2**-0.5, 2**-0.5, 0.0)) + assert oblique.to_bids() is None + world = EncodingDirection((0, 1, 0), space="scanner") + assert world.space is SpaceEnum.scanner and world.to_bids() is None + + +def test_a_direction_may_be_in_a_coordinate_system() -> None: + from brainhops.datamodel.systems import CoordinateSystem + from brainhops.io.metadata._json import encode_changes, to_json + + named = CoordinateSystem(name="RAS") + direction = EncodingDirection((1, 0, 0), space=named) + assert direction.space is named + assert direction.to_bids() is None + # A system equals an equal system, never a label, not even its name. + assert direction == EncodingDirection( + "i", space=CoordinateSystem(name="RAS") + ) + assert direction != EncodingDirection((1, 0, 0), space="RAS") + assert "space=CoordinateSystem(" in repr(direction) + # JSON writes the name of the system, which reads back as a label. + assert to_json("phase_encoding_direction", direction) == { + "Vector": [1.0, 0.0, 0.0], + "Space": "RAS", + } + # Without a name, the direction cannot be written, and is lost. + unnamed = EncodingDirection((1, 0, 0), space=CoordinateSystem()) + obj = {"PhaseEncodingDirection": "j"} + report = ConversionReport() + encode_changes(obj, {"phase_encoding_direction": unnamed}, report=report) + assert obj == {} + assert report.lost == {"phase_encoding_direction": unnamed} + + +def test_a_direction_is_mapped_by_a_transformation() -> None: + from brainhops.datamodel.transformations import Affine, DisplacementField + + swap = np.array( + [[0, 1, 0, 5], [1, 0, 0, -2], [0, 0, 1, 3], [0, 0, 0, 1]], dtype=float + ) + direction = EncodingDirection("j-") + # The linear part of an affine maps it; its translation does not. + assert direction.transform(Affine(swap)) == EncodingDirection("i-") + assert direction.transform(swap[:3, :3]) == EncodingDirection("i-") + with pytest.raises(TypeError, match="affine"): + direction.transform(DisplacementField(np.zeros((2, 2, 2, 3)))) + + +def test_the_direction_fields_take_bids_strings() -> None: + meta = Metadata(phase_encoding_direction="j-") + assert isinstance(meta.phase_encoding_direction, EncodingDirection) + assert meta.phase_encoding_direction.to_bids() == "j-" + meta.slice_encoding_direction = {"Vector": [0, 0, 1]} + assert meta.slice_encoding_direction == EncodingDirection("k") + assert to_bids(meta) == { + "PhaseEncodingDirection": "j-", + "SliceEncodingDirection": "k", + } + assert from_bids(to_bids(meta)) == meta + + +def test_an_oblique_direction_is_lost_in_a_sidecar() -> None: + meta = Metadata(phase_encoding_direction=(1, 1, 0)) + with pytest.raises(MetadataLossError) as info: + to_bids(meta, on_loss="raise") + assert set(info.value.report.lost) == {"phase_encoding_direction"} + + +# ---------------------------------------------------------------------- +# to() +# ---------------------------------------------------------------------- + + +def test_to_converts_and_reports() -> None: + lite = _rich().to(LiteMetadata, on_loss="ignore") + assert type(lite) is LiteMetadata + assert lite.description == "a scan" + report = ConversionReport() + _rich().to("test-lite", on_loss=report) + assert report.lost == {"echo_time": 0.03} + assert (report.source, report.target) == ("generic", "test-lite") + with pytest.raises(MetadataLossError): + _rich().to(LiteMetadata, on_loss="raise") + with pytest.warns(MetadataLossWarning): + _rich().to(LiteMetadata) + + +def test_a_report_as_on_loss_is_filled_silently() -> None: + report = ConversionReport() + with warnings.catch_warnings(), metadata_loss_policy("raise"): + warnings.simplefilter("error") + _rich().to(LiteMetadata, on_loss=report) + # A second conversion adds to the same report. + Metadata(extra={"Key": 1}).to(OpaqueMetadata, on_loss=report) + assert report.lost == {"echo_time": 0.03, "extra": {"Key": 1}} + assert (report.source, report.target) == ("generic", "test-lite") + # `report=` is gone: it would be a field, which there is not. + with pytest.raises(TypeError): + _rich().to(LiteMetadata, report=ConversionReport()) + + +def test_update_raw_applies_the_policy_unless_given_a_report() -> None: + meta = _read() + meta.echo_time = 0.03 # not supported: lost on write + with pytest.warns(MetadataLossWarning): + meta.update_raw() + with pytest.raises(MetadataLossError): + meta.update_raw(on_loss="raise") + report = ConversionReport() + with warnings.catch_warnings(): + warnings.simplefilter("error") + meta.update_raw(on_loss=report) + assert report.lost == {"echo_time": 0.03} + with pytest.raises(ValueError): + _rich().to("no-such-format") + + +def test_to_none_keeps_the_class() -> None: + meta = _read() + same = meta.to(description="other") + assert type(same) is DictMetadata + assert same.raw is meta.raw + assert same._changed_fields() == {"description": "other"} + generic = Metadata(description="d").to() + assert type(generic) is Metadata and generic.description == "d" + + +def test_metadata_field_is_an_annotation() -> None: + class Holder(Magic): + meta: MetadataField[LiteMetadata, Factory(), tx.Doc("Some metadata.")] + + field = next(f for f in fields(Holder) if f.name == "meta") + assert not field.repr and not field.eq and field.kw + assert field.doc == "Some metadata." + assert type(Holder().meta) is LiteMetadata + given = LiteMetadata(description="d") + held = Holder(meta=given) + assert held.meta is not given and held.meta == given + # Converted even though `Holder` does not convert its fields. + with metadata_loss_policy("ignore"): + held.meta = _rich() + assert type(held.meta) is LiteMetadata + + +def test_lazy_is_not_a_class_keyword() -> None: + with pytest.raises(TypeError, match="unknown class keyword"): + + class Lazy( + FileBasedMetadata, supports=("history",), lazy=("history",) + ): + pass + + +def test_the_pinned_format_narrows_the_field() -> None: + # `on={"format": ...}` alone gives the field its literal type and its + # default, and refuses any other format. + field = next(f for f in fields(LiteMetadata) if f.name == "format") + assert field.default == "test-lite" + assert tx.get_args(field.type) == ("test-lite",) + assert LiteMetadata().format == "test-lite" + with pytest.raises(Exception, match="test-lite"): + LiteMetadata(format="test-dict") + assert type(Metadata(format="test-lite")) is LiteMetadata + + +def test_a_channel_color_is_held_as_rgba() -> None: + # One spelling per color, so that a record read back agrees with it. + assert Channel(color="#0000ff").color == "0000FFFF" + assert Channel(color="00FF0080").color == "00FF0080" + assert Channel().color is None diff --git a/tests/test_io_metadata_load.py b/tests/test_io_metadata_load.py new file mode 100644 index 00000000..f19f9f11 --- /dev/null +++ b/tests/test_io_metadata_load.py @@ -0,0 +1,191 @@ +""" +`FileBasedMetadata.load`: reading the metadata of a file without its +data, through the `FileBasedMetadata` registry (NIfTI, MGH, plain Zarr +and OME-Zarr, x5, ITK `.h5`, BIDS sidecars), and the `to_raw` / +`to_file` side. +""" + +import gzip +import json +from io import BytesIO +from pathlib import Path + +import numpy as np +import pytest + +nb = pytest.importorskip("nibabel") + +import brainhops.io as io # noqa: E402 +from brainhops.datamodel.metadata import Metadata # noqa: E402 +from brainhops.io.base._metadata_parser import MetadataParser # noqa: E402 +from brainhops.io.base.parsers import ( # noqa: E402 + ParserContentError, + ParserNotImplementedError, +) +from brainhops.io.images.freesurfer.mgh import ( # noqa: E402 + MghMetadata, +) +from brainhops.io.images.nifti import NiftiMetadata # noqa: E402 +from brainhops.io.metadata import FileBasedMetadata # noqa: E402 +from brainhops.io.transformations.fsl.flirt import FlirtMetadata # noqa: E402 + + +def _nifti(path: Path) -> Path: + values = np.arange(60, dtype=np.int16).reshape(3, 4, 5) + nii = nb.Nifti1Image(values, np.eye(4)) + nii.header["descrip"] = b"a header" + nii.header.set_slope_inter(0.5, 10.0) + nb.save(nii, str(path)) + return path + + +# ---------------------------------------------------------------------- +# DISPATCH +# ---------------------------------------------------------------------- + + +@pytest.mark.parametrize("name", ["scan.nii", "scan.nii.gz"]) +def test_nifti_metadata_is_read_from_the_header(tmp_path, name) -> None: # noqa: ANN001 + meta = FileBasedMetadata.load(_nifti(tmp_path / name)) + assert type(meta) is NiftiMetadata + assert meta.description == "a header" + assert meta.data_type == np.int16 + # The scaling is read from the header, as stored. + assert (meta.scale_slope, meta.scale_intercept) == (0.5, 10.0) + assert isinstance(meta.raw, nb.Nifti1Header) + assert not meta._changed_fields() + + +def test_every_entry_point_reads_the_same(tmp_path) -> None: # noqa: ANN001 + path = _nifti(tmp_path / "scan.nii.gz") + by_path = FileBasedMetadata.load(str(path)) + with open(path, "rb") as f: + by_stream = FileBasedMetadata.load(f) + assert f.tell() == 0 + by_bytes = FileBasedMetadata.load(path.read_bytes()) + by_class = NiftiMetadata.load(path) + assert by_path == by_stream == by_bytes == by_class + # A name says nothing: the content is sniffed. + renamed = tmp_path / "scan.bin" + renamed.write_bytes(gzip.decompress(path.read_bytes())) + assert FileBasedMetadata.load(renamed) == by_path + + +def test_a_hint_selects_the_format(tmp_path) -> None: # noqa: ANN001 + path = _nifti(tmp_path / "scan.nii") + assert type(FileBasedMetadata.load(path, hint="nifti")) is NiftiMetadata + with pytest.raises(ParserContentError): + FileBasedMetadata.load(path, hint="mgh") + + +def test_metadata_files_stay_out_of_the_generic_load(tmp_path) -> None: # noqa: ANN001 + path = _nifti(tmp_path / "scan.nii") + # `io.load` still reads an image; the metadata formats have their + # own registry. + assert not isinstance(io.load(path), Metadata) + assert NiftiMetadata in FileBasedMetadata._REGISTRY + assert NiftiMetadata not in io.FileBasedObject._REGISTRY + assert not issubclass(FileBasedMetadata, io.FileBasedObject) + # The parsers own no registry: `FileBasedMetadata` dispatches. + assert "_REGISTRY" not in vars(MetadataParser) + assert type(FileBasedMetadata.load(path)) is NiftiMetadata + + +def test_an_unknown_file_is_refused(tmp_path) -> None: # noqa: ANN001 + path = tmp_path / "notes.txt" + path.write_text("not metadata") + with pytest.raises(ParserContentError): + FileBasedMetadata.load(path) + + +def test_a_format_without_metadata_in_its_files_refuses(tmp_path) -> None: # noqa: ANN001 + path = tmp_path / "affine.mat" + np.savetxt(path, np.eye(4)) + with pytest.raises(ParserNotImplementedError): + FlirtMetadata.load(path) + + +@pytest.mark.parametrize("name", ["ItkMetadata", "OpaqueMetadata"]) +def test_an_opaque_format_refuses(tmp_path, name) -> None: # noqa: ANN001 + from brainhops.io.metadata import OpaqueMetadata + from brainhops.io.transformations.itk._metadata import ItkMetadata + + cls = {"ItkMetadata": ItkMetadata, "OpaqueMetadata": OpaqueMetadata} + path = _nifti(tmp_path / "a.nii") + # Even a file that another format reads: the class decides, not the + # file. + with pytest.raises(ParserNotImplementedError, match=name): + cls[name].load(path) + + +def test_load_is_resolved_by_the_bases() -> None: + from brainhops.io.base._base import FormatDispatcher + from brainhops.io.metadata import OpaqueMetadata + + def owner(cls: type) -> type: + return next(c for c in cls.__mro__ if "load" in c.__dict__) + + # `FileBasedMetadata` dispatches; on the class of a format whose + # files hold metadata, its `load` reads the file with the parser. + assert owner(FileBasedMetadata) is FileBasedMetadata + assert FileBasedMetadata._is_dispatcher() + assert issubclass(FileBasedMetadata, FormatDispatcher) + assert owner(NiftiMetadata) is FileBasedMetadata + assert owner(MghMetadata) is FileBasedMetadata + assert not NiftiMetadata._is_dispatcher() + # A format whose files hold none refuses, explicitly. + assert owner(FlirtMetadata) is FlirtMetadata + assert owner(OpaqueMetadata) is OpaqueMetadata + # The data model does no input or output. + assert not hasattr(Metadata, "load") + + +# ---------------------------------------------------------------------- +# FORMATS +# ---------------------------------------------------------------------- + + +def test_itk_h5_metadata_reads_the_root_header(tmp_path) -> None: # noqa: ANN001 + h5py = pytest.importorskip("h5py") + from brainhops.io.transformations.itk import ItkH5Metadata + + path = tmp_path / "affine.h5" + with h5py.File(path, "w") as f: + f.create_dataset("ITKVersion", data="5.4.0") + meta = FileBasedMetadata.load(path) + assert type(meta) is ItkH5Metadata + assert [(g.name, g.version) for g in meta.generated_by] == [ + ("ITK", "5.4.0") + ] + + +def test_a_bids_sidecar_reads_as_generic_metadata(tmp_path) -> None: # noqa: ANN001 + path = tmp_path / "sub-01_bold.json" + path.write_text(json.dumps({"RepetitionTime": 2.0, "TaskName": "rest"})) + meta = FileBasedMetadata.load(path) + assert type(meta) is Metadata + assert meta.repetition_time == 2.0 + assert meta.extra == {"TaskName": "rest"} + assert FileBasedMetadata.load(BytesIO(path.read_bytes())) == meta + + +# ---------------------------------------------------------------------- +# WRITING +# ---------------------------------------------------------------------- + + +def test_to_raw_encodes_into_a_copy(tmp_path) -> None: # noqa: ANN001 + meta = FileBasedMetadata.load(_nifti(tmp_path / "scan.nii")) + meta.description = "edited" + record = meta.to_raw() + assert record is not meta.raw + assert record["descrip"].item() == b"edited" + assert meta.raw["descrip"].item() == b"a header" + + +@pytest.mark.parametrize("cls", [NiftiMetadata, MghMetadata]) +def test_formats_written_with_their_data_have_no_to_file(cls) -> None: # noqa: ANN001 + # Their record is written with the data; only a format whose record + # is an object of its own on disk (plain Zarr) defines `to_file`. + assert issubclass(cls, FileBasedMetadata) + assert not hasattr(cls, "to_file") diff --git a/tests/test_io_metadata_matrix.py b/tests/test_io_metadata_matrix.py new file mode 100644 index 00000000..ed5264a8 --- /dev/null +++ b/tests/test_io_metadata_matrix.py @@ -0,0 +1,246 @@ +""" +The cross-format matrix test (`docs/design/format-metadata.md`, section +12 (c)): a fully populated, *encodable* `Metadata` is converted into each +format and back, and the conversion loses exactly the fields the format +declares unsupported, no more, no less, so a wrong `supports=` list +fails here. Where a fresh record can hold the values, they are written +into it and read back too. +""" + +import datetime + +import pytest + + +def _to(source, target, **kwargs): # noqa: ANN001, ANN003, ANN202 + """`source.to(target, ...)`, and the report it filled.""" + report = ConversionReport() + return source.to(target, on_loss=report, **kwargs), report + + +nb = pytest.importorskip("nibabel") + +import brainhops.io as io # noqa: E402, F401 +from brainhops.datamodel.metadata import ( # noqa: E402 + UNSUPPORTED, + Channel, + ConversionReport, + GeneratedBy, + Metadata, +) +from brainhops.datamodel.metadata._vocabulary import VOCABULARY # noqa: E402 +from brainhops.io.images.freesurfer.mgh import MghMetadata # noqa: E402 +from brainhops.io.images.nifti import NiftiMetadata # noqa: E402 +from brainhops.io.metadata import ( # noqa: E402 # noqa: E402 + FileBasedMetadata, + OpaqueMetadata, +) +from brainhops.io.transformations.fsl.flirt import FlirtMetadata # noqa: E402 +from brainhops.io.transformations.itk import ( # noqa: E402 + ItkH5Metadata, + ItkMetadata, +) +from brainhops.io.transformations.x5 import X5Metadata # noqa: E402 + +zarr = pytest.importorskip("brainhops.io.images.zarr") +OmeZarrImage = zarr.OmeZarrImage +OmeZarrMetadata = zarr.OmeZarrMetadata +ZarrMetadata = zarr.ZarrMetadata + +# Three volumes (or channels), four slices along `k`. +FULL = dict( + name="sub-01", + description="a short description", + history=("mri_convert a b", "recon-all -s bert"), + generated_by=(GeneratedBy(name="ITK", version="5.4.0"),), + creation_time=datetime.datetime(2024, 1, 2, 3, 4, 5), + sources=("sub-01_T1w.nii",), + space="MNI152NLin2009cAsym", + intent="label", + repetition_time=2.0, + echo_time=0.03, + inversion_time=0.9, + flip_angle=9.0, + magnetic_field_strength=3.0, + manufacturer="Siemens", + manufacturers_model_name="Prisma", + institution_name="Somewhere", + acquisition_time=datetime.datetime(2024, 1, 2, 3, 0, 0), + phase_encoding_direction="j", + total_readout_time=0.05, + effective_echo_spacing=0.0005, + slice_encoding_direction="k", + slice_timing=(0.0, 0.5, 1.0, 1.5), + multiband_acceleration_factor=2, + bvalues=(0.0, 1000.0, 1000.0), + bvectors=((0.0, 0.0, 0.0), (1.0, 0.0, 0.0), (0.0, 1.0, 0.0)), + display_range=(0.0, 100.0), + channels=tuple( + Channel(name=name, color="FFFFFFFF", display_range=(0.0, 100.0)) + for name in ("a", "b", "c") + ), + data_unit="a.u.", + data_type="float32", + scale_slope=0.5, + scale_intercept=10.0, + objective_magnification=10.0, + objective_numerical_aperture=0.3, + illumination_type="epifluorescence", + contrast_method="fluorescence", + moving="sub-01_T1w.nii.gz", + fixed="tpl-MNI_T1w.nii.gz", + input_space="T1w", + output_space="MNI152NLin2009cAsym", + extra={"TaskName": "rest"}, +) +# The values as the hub holds them (`"a.u."` is a `Unit`), to compare +# with what a conversion gives back. +HUB = Metadata(**FULL) + + +def _nifti_record() -> nb.Nifti1Header: + header = nb.Nifti1Header() + header.set_data_shape((5, 5, 4, 3)) + header.set_xyzt_units("mm", "sec") + return header + + +# Format -> a fresh record its writer would hold, or `None` when the +# format has no record to hold the values (or is checked otherwise). +FORMATS = { + NiftiMetadata: _nifti_record, + MghMetadata: MghMetadata._raw_class, + X5Metadata: X5Metadata._raw_class, + ZarrMetadata: ZarrMetadata._raw_class, + ItkH5Metadata: ItkH5Metadata._raw_class, + OmeZarrMetadata: None, # written by a real save below + ItkMetadata: None, + FlirtMetadata: None, +} + + +def _subclasses(cls: type) -> set: + out = set() + for sub in cls.__subclasses__(): + out |= {sub} | _subclasses(sub) + return out + + +def test_every_format_is_in_the_matrix() -> None: + formats = { + cls + for cls in _subclasses(Metadata) + if cls not in (FileBasedMetadata, OpaqueMetadata) + and cls.__module__.startswith("brainhops.io") + # A shared base, not a format (`_ZarrMetadataParser`). + and not cls.__name__.startswith("_") + # A parametrized base (`FileBasedMetadata[nb.Nifti1Header]`). + and "[" not in cls.__name__ + } + assert formats == set(FORMATS) + + +def test_the_fixture_is_fully_populated() -> None: + full = Metadata(**FULL) + assert all(getattr(full, name) is not None for name in VOCABULARY) + + +@pytest.mark.parametrize("cls", list(FORMATS), ids=lambda c: c.__name__) +def test_a_conversion_loses_exactly_the_unsupported_fields(cls) -> None: # noqa: ANN001 + converted, report = _to(Metadata(**FULL), cls) + assert set(report.lost) == set(cls.unsupported_fields) + assert not report.approximated + for name in cls.unsupported_fields: + assert getattr(converted, name) is UNSUPPORTED + # And back: nothing more is lost on the way to the hub. + back, report = _to(converted, Metadata) + assert not report.lossy + for name in VOCABULARY + ("extra",): + expected = getattr(HUB, name) + if name in cls.unsupported_fields: + expected = {} if name == "extra" else None + assert getattr(back, name) == expected, name + + +# The fields each format derives from the geometry of the data model. +_GEOMETRY = { + # The image writer stores the scaling with the data, not the record. + NiftiMetadata: { + "repetition_time", + "intent", + "space", + "scale_slope", + "scale_intercept", + }, + ZarrMetadata: {"data_type"}, +} + + +@pytest.mark.parametrize( + "cls", + [cls for cls, record in FORMATS.items() if record is not None], + ids=lambda c: c.__name__, +) +def test_a_fresh_record_holds_what_the_format_supports(cls) -> None: # noqa: ANN001 + converted, _ = _to(Metadata(**FULL), cls) + report = ConversionReport() + record = converted.update_raw(FORMATS[cls](), on_loss=report) + back = cls.from_raw(record) + if cls is ItkH5Metadata: + # Only the ITK version is recorded: the fixture names ITK alone. + assert back.generated_by == FULL["generated_by"] + return + # Fields that are views of geometry (the writer sets them), which a bare + # record does not hold. + expected = set(VOCABULARY) & cls.supported_fields + expected -= _GEOMETRY.get(cls, set()) + assert not report.lost + for name in sorted(expected): + assert getattr(back, name) == getattr(HUB, name), name + if cls.supports("extra"): + assert back.extra == FULL["extra"] + + +def _format_classes() -> list: + """Every metadata class of a format in the package.""" + found, stack = [], [Metadata] + while stack: + cls = stack.pop() + for sub in cls.__subclasses__(): + stack.append(sub) + if ( + sub.__module__.startswith("brainhops.") + and issubclass(sub, FileBasedMetadata) + and not sub.__name__.startswith("_") + # Not the class `bagof` builds for a subscript such as + # `FileBasedMetadata[MghRaw]`: its subclass is the format. + and "__magic_generic_origin__" not in vars(sub) + ): + found.append(sub) + return found + + +def test_every_format_declares_a_record_type_of_its_own() -> None: + # A conversion gives a record back to a format only when the record is + # of the type the format declares (`_accepts_raw`): no two formats + # may declare the same type, or one that is a subclass of another's. + declared = {} + for cls in _format_classes(): + if cls is FileBasedMetadata: + continue + raw_type = cls._raw_class + assert raw_type is not None, f"{cls.__name__} declares no raw type" + if raw_type is not type(None): + declared[cls] = raw_type + assert {cls.__name__ for cls in declared} >= { + "NiftiMetadata", + "MghMetadata", + "ZarrMetadata", + "OmeZarrMetadata", + "X5Metadata", + "ItkH5Metadata", + } + for cls, raw_type in declared.items(): + for other, other_type in declared.items(): + if other is not cls: + assert not issubclass(raw_type, other_type), (cls, other) diff --git a/tests/test_io_metadata_mgh.py b/tests/test_io_metadata_mgh.py new file mode 100644 index 00000000..9bca83c4 --- /dev/null +++ b/tests/test_io_metadata_mgh.py @@ -0,0 +1,81 @@ +""" +Tests for the metadata of MGH/MGZ files (`MghMetadata`). + +The footer of MRI parameters is decoded into the vocabulary in BIDS +units (ms -> s, rad -> deg), the command-line tags into `history`; a +read-then-save keeps the footer and the tags, a field set by the user is +written over them, and a conversion to or from another format reports +what the target cannot hold. +""" + +import copy +import struct + +import pytest + +nb = pytest.importorskip("nibabel") + + +from brainhops.io.base._mgh_metadata import MghRaw # noqa: E402 +from brainhops.io.base._mgh_tags import ( # noqa: E402 + decode_history, + encode_history, + parse_tags, +) + + +def _cmdline(command: bytes) -> bytes: + payload = command + b"\0" + return struct.pack(">iq", 3, len(payload)) + payload + + +# A tag FreeSurfer writes with a 64-bit length, then two command lines, +# then a legacy tag with no length (which must stay last). +OTHER_TAG = struct.pack(">iq", 43, 4) + struct.pack(">f", 3.0) +TAGS = ( + OTHER_TAG + + _cmdline(b"mri_convert in.nii orig.mgz") + + _cmdline(b"mri_normalize orig.mgz T1.mgz") + + struct.pack(">i", 2) +) + + +# ---------------------------------------------------------------------- +# TAGS +# ---------------------------------------------------------------------- + + +def test_tags_parse_into_chunks() -> None: + parsed = parse_tags(TAGS) + assert [tag for tag, _ in parsed] == [43, 3, 3, 2] + assert b"".join(chunk for _, chunk in parsed) == TAGS + assert parse_tags(b"") == [] + assert parse_tags(b"\0\0") is None + + +def test_history_replaces_only_the_command_lines() -> None: + tags = encode_history(TAGS, ("a", "b", "c")) + assert decode_history(tags) == ("a", "b", "c") + parsed = parse_tags(tags) + assert [tag for tag, _ in parsed] == [43, 3, 3, 3, 2] + assert parsed[0][1] == OTHER_TAG + # No command line yet: they go before the legacy tag. + tags = encode_history(OTHER_TAG + struct.pack(">i", 2), ("a",)) + assert [tag for tag, _ in parse_tags(tags)] == [43, 3, 2] + assert encode_history(TAGS, None) == OTHER_TAG + struct.pack(">i", 2) + + +# ---------------------------------------------------------------------- +# CROSS-FORMAT +# ---------------------------------------------------------------------- + + +def test_record_copies_are_independent() -> None: + record = MghRaw(tags=b"x") + record.header["tr"] = 5.0 + other = copy.deepcopy(record) + other.header["tr"] = 6.0 + assert float(record.header["tr"]) == 5.0 + assert other != record + other.header["tr"] = 5.0 + assert other == record diff --git a/tests/test_io_metadata_nifti.py b/tests/test_io_metadata_nifti.py new file mode 100644 index 00000000..a1052594 --- /dev/null +++ b/tests/test_io_metadata_nifti.py @@ -0,0 +1,97 @@ +""" +Tests for the metadata of NIfTI files (`NiftiMetadata`), on images and +on every NIfTI-based transformation. + +A NIfTI file read and saved again keeps its description, auxiliary file, +display range, slice timing and extensions; the common fields decode +them, and a common field set by the user is written over the header, +with what NIfTI cannot hold reported. +""" + +import numpy as np +import pytest + + +def _to(source, target, **kwargs): # noqa: ANN001, ANN003, ANN202 + """`source.to(target, ...)`, and the report it filled.""" + report = ConversionReport() + return source.to(target, on_loss=report, **kwargs), report + + +nb = pytest.importorskip("nibabel") + + +from brainhops.datamodel.metadata import ( # noqa: E402 + ConversionReport, + Metadata, +) +from brainhops.io.images.nifti import NiftiMetadata # noqa: E402 + +# ---------------------------------------------------------------------- +# READING +# ---------------------------------------------------------------------- + + +def test_capabilities() -> None: + assert not NiftiMetadata.supports("extra") + assert not NiftiMetadata.supports("echo_time") + assert NiftiMetadata.supports("slice_timing") + + +# ---------------------------------------------------------------------- +# CONVERSION +# ---------------------------------------------------------------------- + + +def test_what_nifti_cannot_hold_is_reported() -> None: + generic = Metadata( + description="d", echo_time=0.03, history=("a",), extra={"K": 1} + ) + nifti, report = _to(generic, NiftiMetadata) + assert nifti.description == "d" + assert report.lost == { + "echo_time": 0.03, + "history": ("a",), + "extra": {"K": 1}, + } + + +# ---------------------------------------------------------------------- +# DERIVATION +# ---------------------------------------------------------------------- + + +def _header() -> "nb.Nifti1Header": + """A 4-D header with encoding axes and a slice timing.""" + h = nb.Nifti1Header() + h.set_data_shape((4, 5, 6, 3)) + h.set_dim_info(freq=0, phase=1, slice=2) + h["slice_code"] = 1 + h["slice_start"], h["slice_end"] = 0, 5 + h["slice_duration"] = 0.5 + return h + + +def test_a_header_follows_the_spatial_axes() -> None: + from brainhops.datamodel.axes import Axis + from brainhops.datamodel.geometry import Geometry + from brainhops.datamodel.metadata import Indexed, Resampled + from brainhops.datamodel.systems import CoordinateSystem + from brainhops.datamodel.transformations import Affine + + meta = NiftiMetadata.from_raw(_header()) + # A resampling clears the slice slots and `dim_info` of the copy. + resampled = meta.derive(Resampled(Affine(np.eye(4)[:3]), Geometry())) + assert resampled.raw is not meta.raw + assert resampled.raw.get_dim_info() == (None, None, None) + assert resampled.raw["slice_code"] == 0 + assert meta.raw.get_dim_info() == (0, 1, 2) + # An index along time keeps them. + system = CoordinateSystem( + axes=[Axis(n, "space") for n in "ijk"] + [Axis("t", "time")] + ) + indexed = meta.derive(Indexed((Ellipsis, [0, 2]), (4, 5, 6, 3), system)) + assert indexed.raw is not meta.raw + assert indexed.raw.get_dim_info() == (0, 1, 2) + assert indexed.raw["slice_code"] == 1 + assert indexed.raw["slice_duration"] == 0.5 diff --git a/tests/test_io_metadata_zarr.py b/tests/test_io_metadata_zarr.py new file mode 100644 index 00000000..6e945571 --- /dev/null +++ b/tests/test_io_metadata_zarr.py @@ -0,0 +1,44 @@ +""" +Tests for the metadata of Zarr images: `ZarrMetadata` (a plain array, +the vocabulary as a sidecar under the attribute `"brainhops"`) and +`OmeZarrMetadata` (an OME-Zarr pyramid: the multiscale name, `omero` and +the other group attributes). +""" + +import pytest + +pytest.importorskip("abczarr") + + +from brainhops.datamodel.metadata import ( # noqa: E402 + ConversionReport, + Metadata, +) +from brainhops.io.images.zarr import ( # noqa: E402 + ZarrMetadata, +) + + +def _to(source, target, **kwargs): # noqa: ANN001, ANN003, ANN202 + """`source.to(target, ...)`, and the report it filled.""" + report = ConversionReport() + return source.to(target, on_loss=report, **kwargs), report + + +# ---------------------------------------------------------------------- +# PLAIN ZARR +# ---------------------------------------------------------------------- + + +def test_plain_zarr_cannot_hold_diffusion() -> None: + assert ZarrMetadata.unsupported_fields == { + "bvalues", + "bvectors", + "scale_slope", + "scale_intercept", + } + _, report = _to( + Metadata(bvalues=(0.0, 1000.0)), + ZarrMetadata, + ) + assert report.lost == {"bvalues": (0.0, 1000.0)} diff --git a/zensical.toml b/zensical.toml index 4aa1def8..fffd7a07 100644 --- a/zensical.toml +++ b/zensical.toml @@ -26,6 +26,8 @@ nav = [ "api/datamodel/transformations.md", "api/datamodel/images.md", "api/datamodel/kinds.md", + "api/datamodel/metadata.md", + {"metadata (format authors)" = "api/datamodel/metadata-formats.md"}, ]}, {"io" = [ "api/io/index.md", @@ -77,6 +79,11 @@ nav = [ {"niftyreg" = "api/io/transformations/niftyreg.md"}, "api/io/transformations/zarr.md", ]}, + {"metadata" = [ + "api/io/metadata/index.md", + "api/io/metadata/base.md", + "api/io/metadata/bids.md", + ]}, {"vectors" = "api/io/vectors/index.md"}, ]}, {"For developers" = [ @@ -88,7 +95,10 @@ nav = [ {"enum" = "api/_core/enum.md"}, {"invfield" = "api/_ext/invfield/index.md"}, ]} - ]} + ]}, + {"Developer guide" = [ + {"Metadata of a format" = "dev/metadata-formats.md"}, + ]}, ] # The site_name is shown in the page header and the browser window title