From 39749dee64beb36f9df503d2a752cf9f93c2153e Mon Sep 17 00:00:00 2001 From: Sandor Kertesz Date: Tue, 23 Jun 2026 14:33:07 +0100 Subject: [PATCH] Improve field component docs --- docs/source/concepts/field.rst | 33 -- docs/source/concepts/field/ensemble.rst | 65 +++ docs/source/concepts/field/field.rst | 121 +++++ docs/source/concepts/field/geography.rst | 114 ++++ docs/source/concepts/field/index.rst | 15 + docs/source/concepts/field/parameter.rst | 120 +++++ docs/source/concepts/field/proc.rst | 95 ++++ docs/source/concepts/field/time.rst | 97 ++++ docs/source/concepts/field/vertical.rst | 498 ++++++++++++++++++ docs/source/concepts/fieldlist.rst | 197 +++++++ docs/source/concepts/index.rst | 4 +- docs/source/conf.py | 1 + .../data/field/component/level_type.py | 8 +- 13 files changed, 1333 insertions(+), 35 deletions(-) delete mode 100644 docs/source/concepts/field.rst create mode 100644 docs/source/concepts/field/ensemble.rst create mode 100644 docs/source/concepts/field/field.rst create mode 100644 docs/source/concepts/field/geography.rst create mode 100644 docs/source/concepts/field/index.rst create mode 100644 docs/source/concepts/field/parameter.rst create mode 100644 docs/source/concepts/field/proc.rst create mode 100644 docs/source/concepts/field/time.rst create mode 100644 docs/source/concepts/field/vertical.rst diff --git a/docs/source/concepts/field.rst b/docs/source/concepts/field.rst deleted file mode 100644 index b45f3dc2d..000000000 --- a/docs/source/concepts/field.rst +++ /dev/null @@ -1,33 +0,0 @@ -Field and FieldList -======================= - -Field --------- -The :py:class:`~earthkit.data.core.field.Field` class represents a horizontal slice of the atmosphere/hydrosphere at a given time. - -A field is a fundamental data structure in EarthKit Data, and it contains both metadata and data values. - -The Field class is not polymorphic, but is a composition of polymorphic components. Each component is designed to handle a specific aspect of the field's metadata, such as time, level, or spatial information. This design allows for greater flexibility and modularity in handling different types of data and metadata. - -Fields are typically created as part of a :py:class:`~earthkit.data.core.fieldlist.FieldList` when reading data from a source, and they can be accessed and manipulated through the FieldList's methods. - - -FieldList ------------- -A :py:class:`~earthkit.data.core.fieldlist.FieldList` is a collection of :py:class:`~earthkit.data.core.field.Field` objects. It provides methods to access and manipulate the fields it contains, such as selecting fields based on metadata values, extracting data values and converting the fieldlists to other formats (e.g., Xarray datasets). - - -How-tos: - - - :ref:`/how-tos/field/field_overview.ipynb` - - :ref:`/how-tos/grib/grib_overview.ipynb` - -Immutability of field values ------------------------------- - -The values of a field are immutable, meaning that they cannot be modified in place. When you access the values of a field, you will receive a copy of the data, and any modifications to that copy will not affect the original field's data. This design choice ensures that the integrity of the field's data is maintained and prevents unintended side effects when working with field values. - -Arithmetic operations on fields ------------------------------- -Arithmetic operations on fields (e.g., addition, subtraction, multiplication, division) are directly supported, and they return new field objects with the result of the operation. When performing arithmetic operations, the metadata of the resulting field is derived from first operand without any -alterations. diff --git a/docs/source/concepts/field/ensemble.rst b/docs/source/concepts/field/ensemble.rst new file mode 100644 index 000000000..fb10f35ef --- /dev/null +++ b/docs/source/concepts/field/ensemble.rst @@ -0,0 +1,65 @@ +.. _ensemble_component: + +Ensemble component +================== + +Every :py:class:`~earthkit.data.core.field.Field` may carry an *ensemble component* that +identifies which ensemble member the field belongs to. The ensemble component is accessible +via the :attr:`ensemble` attribute of a field and is represented by a subclass of +:py:class:`~earthkit.data.field.component.ensemble.EnsembleBase`. + +Fields from deterministic (non-ensemble) data have an +:py:class:`~earthkit.data.field.component.ensemble.EmptyEnsemble` component where +:meth:`~earthkit.data.field.component.ensemble.EnsembleBase.member` returns ``None``. + +.. code-block:: python + + >>> import earthkit.data as ekd + >>> field = ekd.from_source("sample", "ens_cf_pf.grib").to_fieldlist()[2] + >>> field.ensemble.member() + '1' + >>> field.get("ensemble.member") + '1' + +The ensemble component is **immutable**. Use the +:meth:`~earthkit.data.field.component.ensemble.EnsembleBase.set` method (or +:meth:`~earthkit.data.core.field.Field.set` on the field) to derive a modified copy: + +.. code-block:: python + + >>> new_field = field.set({"ensemble.member": "5"}) + >>> new_field.ensemble.member() + '5' + + +Member representation +--------------------- + +The member value is stored internally as a string. Integer values passed during construction +or via :meth:`set` are converted automatically. ``None`` represents the absence of an +ensemble member (deterministic data). + + +Accessing ensemble information +------------------------------- + +.. list-table:: + :header-rows: 1 + :widths: 32 68 + + * - Key + - Description + * - ``ensemble.member`` + - Ensemble member identifier as a string, or ``None`` for deterministic data. + * - ``ensemble.realization`` + - Alias of ``ensemble.member``. + * - ``ensemble.realisation`` + - Alias of ``ensemble.member``. + + +How-tos +------- + +- :ref:`/how-tos/field/field_overview.ipynb` +- :ref:`/how-tos/grib/grib_overview.ipynb` +- :ref:`/how-tos/xr_engine/xarray_engine_ensemble.ipynb` diff --git a/docs/source/concepts/field/field.rst b/docs/source/concepts/field/field.rst new file mode 100644 index 000000000..04590116b --- /dev/null +++ b/docs/source/concepts/field/field.rst @@ -0,0 +1,121 @@ +.. _field_concept: + +Field +===== + +A :py:class:`~earthkit.data.core.field.Field` represents a single horizontal slice of a +geophysical quantity at a particular time and vertical level. It is the fundamental data +structure in EarthKit Data and bundles together: + +- the **data values** — a 2-D (or 1-D unstructured) array of floating-point numbers; +- a set of **metadata components** that fully describe the field (see below). + +Fields are created automatically when reading data through +:py:func:`~earthkit.data.from_source` and are normally accessed as members of a +:py:class:`~earthkit.data.core.fieldlist.FieldList`. See :doc:`fieldlist` for how to work +with collections of fields. + + +Component-based metadata +------------------------ + +The Field class is not polymorphic. Instead it is composed of a set of replaceable, +polymorphic *components*, each responsible for a distinct aspect of the metadata: + +.. list-table:: + :header-rows: 1 + :widths: 20 25 55 + + * - Attribute + - Component class + - What it describes + * - ``field.parameter`` + - :py:class:`~earthkit.data.field.component.parameter.ParameterBase` + - Physical quantity: variable name, units, CF names, chemical or optical properties. + * - ``field.time`` + - :py:class:`~earthkit.data.field.component.time.TimeBase` + - Temporal coordinate: base datetime, forecast step, valid datetime. + * - ``field.vertical`` + - :py:class:`~earthkit.data.field.component.vertical.VerticalBase` + - Vertical coordinate: level value, level type, layer bounds. + * - ``field.geography`` + - :py:class:`~earthkit.data.field.component.geography.GeographyBase` + - Horizontal grid: lat/lon arrays, bounding box, projection, grid type. + * - ``field.ensemble`` + - :py:class:`~earthkit.data.field.component.ensemble.EnsembleBase` + - Ensemble member identifier. + * - ``field.proc`` + - :py:class:`~earthkit.data.field.component.proc.ProcBase` + - Post-processing operations applied to produce the field (e.g. accumulation). + +Each component exposes its metadata through named methods (e.g. +``field.vertical.level()``) and through the generic +:meth:`~earthkit.data.core.field.Field.get` method using a ``"component.key"`` prefix: + +.. code-block:: python + + >>> import earthkit.data as ekd + >>> field = ekd.from_source("sample", "test.grib").to_fieldlist()[0] + >>> field.parameter.variable() + '2t' + >>> field.get("parameter.variable") + '2t' + >>> field.vertical.level() + 0 + >>> field.get("time.base_datetime") + datetime.datetime(2020, 1, 1, 0, 0) + +For a full description of each component's available keys see the dedicated pages: +:doc:`parameter`, :doc:`time`, :doc:`vertical`, :doc:`geography`, :doc:`ensemble`, +and :doc:`proc`. + + +Immutability of field values +----------------------------- + +Field values are immutable: :meth:`~earthkit.data.core.field.Field.values` always returns +a **copy** of the underlying array. Modifications to that copy do not affect the stored +data. This guarantees that the original data remains consistent no matter how many +downstream operations consume it. + + +Modifying a field +----------------- + +Because both values and metadata are immutable, changes are expressed by creating a +new field via :meth:`~earthkit.data.core.field.Field.set`. The method accepts a dictionary +of ``"component.key": value`` pairs (and/or a ``"values"`` entry) and returns a new field +with the requested changes applied while leaving all other attributes unchanged: + +.. code-block:: python + + >>> new_field = field.set({"vertical.level": 500, "time.step": 6}) + >>> new_field.vertical.level() + 500 + >>> new_field.time.step() + datetime.timedelta(seconds=21600) + + +Arithmetic operations +---------------------- + +Fields support element-wise arithmetic directly (``+``, ``-``, ``*``, ``/``). Each +operation returns a new field whose data is the result of the operation. The metadata +(parameter, time, vertical, geography, ensemble, proc) of the **left-hand operand** is +retained in the result without modification: + +.. code-block:: python + + >>> fl = ekd.from_source("sample", "tuv_pl.grib").to_fieldlist() + >>> result = fl[0] + fl[1] + >>> result.parameter.variable() == fl[0].parameter.variable() + True + + +How-tos +------- + +- :ref:`/how-tos/field/field_overview.ipynb` +- :ref:`/how-tos/grib/grib_overview.ipynb` +- :ref:`/how-tos/grib/grib_modify_values.ipynb` +- :ref:`/how-tos/grib/grib_modify_metadata.ipynb` diff --git a/docs/source/concepts/field/geography.rst b/docs/source/concepts/field/geography.rst new file mode 100644 index 000000000..8c2169032 --- /dev/null +++ b/docs/source/concepts/field/geography.rst @@ -0,0 +1,114 @@ +.. _geography_component: + +Geography component +=================== + +Every :py:class:`~earthkit.data.core.field.Field` carries a *geography component* that describes +the horizontal spatial coordinate system of the field. The geography component is accessible via +the :attr:`geography` attribute of a field and is represented by a subclass of +:py:class:`~earthkit.data.field.component.geography.GeographyBase`. + +.. code-block:: python + + >>> import earthkit.data as ekd + >>> field = ekd.from_source("sample", "test.grib").to_fieldlist()[0] + >>> field.geography.shape + (19, 36) + >>> field.geography.area() + (70, -20, 35, 40) + >>> field.geography.grid_type() + 'regular_ll' + +The same information is available through the generic :meth:`~earthkit.data.core.field.Field.get` +interface using the ``"geography."`` prefix: + +.. code-block:: python + + >>> field.get("geography.area") + (70, -20, 35, 40) + >>> field.get("geography.shape") + (19, 36) + +The geography component is **immutable**. Use the +:meth:`~earthkit.data.field.component.geography.GeographyBase.set` method (or +:meth:`~earthkit.data.core.field.Field.set` on the field) to derive a modified copy. +When changing the grid it is typically necessary to supply new field values to match the +new grid shape: + +.. code-block:: python + + >>> import numpy as np + >>> values = np.random.rand(19, 36) + >>> new_field = field.set({"geography.grid_spec": [10, 10], "values": values}) + >>> new_field.geography.area() + (90.0, 0, -90.0, 360.0) + + +Geography types +--------------- + +The appropriate geography subclass is determined automatically from the data: + +- **LatLonGeography** — irregular or curvilinear lat/lon grid where every point has an + explicit latitude and longitude. +- **MeshedLatLonGeography** — regular lat/lon grid defined by distinct (1-D) latitude and + longitude arrays that form a full Cartesian mesh. +- **GridsSpecBasedGeography** — grid defined by an eckit-geo grid specification string or + dictionary (requires optional eckit-geo grid support). +- **EmptyGeography** — placeholder used when no coordinate information is available. + + +Accessing geography information +-------------------------------- + +All geography keys are accessible through :meth:`~earthkit.data.core.field.Field.get` with the +``"geography."`` prefix, and can therefore be used in +:meth:`~earthkit.data.core.fieldlist.FieldList.sel`, +:meth:`~earthkit.data.core.fieldlist.FieldList.order_by`, and +:meth:`~earthkit.data.core.fieldlist.FieldList.metadata`. + +.. list-table:: + :header-rows: 1 + :widths: 32 68 + + * - Key + - Description + * - ``geography.latitudes`` + - Array of latitude values for every grid point. ``dtype`` argument supported for type control. + * - ``geography.longitudes`` + - Array of longitude values for every grid point. ``dtype`` argument supported for type control. + * - ``geography.distinct_latitudes`` + - 1-D array of unique latitude values for regular grids, or ``None`` for irregular grids. + * - ``geography.distinct_longitudes`` + - 1-D array of unique longitude values for regular grids, or ``None`` for irregular grids. + * - ``geography.x`` + - Array of x-coordinates in the native CRS. ``dtype`` argument supported. + * - ``geography.y`` + - Array of y-coordinates in the native CRS. ``dtype`` argument supported. + * - ``geography.shape`` + - Grid shape as a tuple of integers (e.g. ``(latitude_size, longitude_size)``). + * - ``geography.projection`` + - :py:class:`~earthkit.data.utils.projections.Projection` object describing the CRS, or ``None``. + * - ``geography.bounding_box`` + - :py:class:`~earthkit.data.utils.bbox.BoundingBox` for the grid extent. + * - ``geography.area`` + - Bounding box as a ``(north, west, south, east)`` tuple of floats. + * - ``geography.grid_type`` + - String identifying the grid type (e.g. ``"regular_ll"``, ``"reduced_gg"``). + * - ``geography.grid_spec`` + - Grid specification. Can be used to construct a new geography of the same type. + * - ``geography.grid`` + - Grid definition understood by eckit-geo (when grid support is available). + * - ``geography.unique_grid_id`` + - A hashable identifier that is the same for two fields sharing an identical grid. + + +How-tos +------- + +- :ref:`/how-tos/field/field_overview.ipynb` +- :ref:`/how-tos/grib/grib_overview.ipynb` +- :ref:`/how-tos/grib/grib_lat_lon_value_ll.ipynb` +- :ref:`/how-tos/grib/grib_lat_lon_value_rgg.ipynb` +- :ref:`/how-tos/grib/grib_nearest_gridpoint.ipynb` +- :ref:`/how-tos/misc/projection.ipynb` diff --git a/docs/source/concepts/field/index.rst b/docs/source/concepts/field/index.rst new file mode 100644 index 000000000..554875571 --- /dev/null +++ b/docs/source/concepts/field/index.rst @@ -0,0 +1,15 @@ +.. _concepts-field: + +Field +=========================== + +.. toctree:: + :maxdepth: 1 + + field.rst + vertical.rst + parameter.rst + time.rst + ensemble.rst + geography.rst + proc.rst diff --git a/docs/source/concepts/field/parameter.rst b/docs/source/concepts/field/parameter.rst new file mode 100644 index 000000000..19401cc0a --- /dev/null +++ b/docs/source/concepts/field/parameter.rst @@ -0,0 +1,120 @@ +.. _parameter_component: + +Parameter component +=================== + +Every :py:class:`~earthkit.data.core.field.Field` carries a *parameter component* that describes +the physical quantity the field represents. The parameter component is accessible via the +:attr:`parameter` attribute of a field and is represented by a subclass of +:py:class:`~earthkit.data.field.component.parameter.ParameterBase`. + +.. code-block:: python + + >>> import earthkit.data as ekd + >>> field = ekd.from_source("sample", "test.grib").to_fieldlist()[0] + >>> field.parameter.variable() + '2t' + >>> field.parameter.units() + K + >>> field.parameter.standard_name() + '2_metre_temperature' + +The same information is available through the generic :meth:`~earthkit.data.core.field.Field.get` +interface using the ``"parameter."`` prefix: + +.. code-block:: python + + >>> field.get("parameter.variable") + '2t' + >>> field.get("parameter.units") + K + +The parameter component is **immutable**. Use the +:meth:`~earthkit.data.field.component.parameter.ParameterBase.set` method (or +:meth:`~earthkit.data.core.field.Field.set` on the field) to derive a modified copy: + +.. code-block:: python + + >>> new_field = field.set({"parameter.variable": "msl", "parameter.units": "Pa"}) + >>> new_field.parameter.variable() + 'msl' + + +Parameter types +--------------- + +The appropriate parameter subclass is determined automatically from the data: + +- :py:class:`~earthkit.data.field.component.parameter.Parameter` — standard meteorological + parameter with a variable name and units. +- :py:class:`~earthkit.data.field.component.parameter.ChemicalParameter` — parameter that + also carries a chemical constituent or aerosol type (``chem`` / ``chem_long_name`` keys). +- :py:class:`~earthkit.data.field.component.parameter.OpticalParameter` — parameter defined + at a specific wavelength (``wavelength``, ``wavelength_bounds``, ``wavelength_units`` keys). +- :py:class:`~earthkit.data.field.component.parameter.ChemicalOpticalParameter` — combines + both chemical and optical information. +- :py:class:`~earthkit.data.field.component.parameter.WaveSpectraParameter` — 2-D wave spectra + parameter carrying direction and frequency bins (``wave_direction*`` and + ``wave_frequency*`` keys). + + +Accessing parameter information +-------------------------------- + +All parameter keys are accessible through :meth:`~earthkit.data.core.field.Field.get` with the +``"parameter."`` prefix, and can therefore be used in +:meth:`~earthkit.data.core.fieldlist.FieldList.sel`, +:meth:`~earthkit.data.core.fieldlist.FieldList.order_by`, and +:meth:`~earthkit.data.core.fieldlist.FieldList.metadata`. + +.. list-table:: + :header-rows: 1 + :widths: 32 68 + + * - Key + - Description + * - ``parameter.variable`` + - Parameter variable name as a string (e.g. ``"2t"``). Not normalised; taken directly from the source data. For example, for GRIB data, it will be the value of the "shortName" key in the GRIB message. + * - ``parameter.param`` + - Alias of ``parameter.variable``. + * - ``parameter.standard_name`` + - CF standard name of the parameter variable. + * - ``parameter.long_name`` + - CF long name of the parameter variable. + * - ``parameter.units`` + - Units of the parameter as a :py:class:`~earthkit.utils.units.Units` object. + * - ``parameter.chem`` + - Chemical constituent or aerosol type, or ``None`` for non-chemical parameters. + * - ``parameter.chem_long_name`` + - Long name of the chemical constituent or aerosol type, or ``None``. + * - ``parameter.wavelength`` + - Central wavelength for optical parameters, or ``None``. Accepts an optional ``units`` argument for conversion. + * - ``parameter.wavelength_bounds`` + - Wavelength bounds as a 2-tuple for optical parameters, or ``None``. + * - ``parameter.wavelength_units`` + - Units of the wavelength (e.g. nanometres), or ``None``. + * - ``parameter.wave_direction`` + - Wave propagation direction for 2-D wave spectra parameters, or ``None``. + * - ``parameter.wave_direction_index`` + - 0-based index of the wave direction bin, or ``None``. + * - ``parameter.wave_direction_bounds`` + - Direction bounds as a 2-tuple, or ``None``. + * - ``parameter.wave_direction_units`` + - Units of the wave direction (e.g. degrees), or ``None``. + * - ``parameter.wave_frequency`` + - Wave frequency for 2-D wave spectra parameters, or ``None``. + * - ``parameter.wave_frequency_index`` + - 0-based index of the wave frequency bin, or ``None``. + * - ``parameter.wave_frequency_bounds`` + - Frequency bounds as a 2-tuple, or ``None``. + * - ``parameter.wave_frequency_units`` + - Units of the wave frequency (e.g. 1/s), or ``None``. + + +How-tos +------- + +- :ref:`/how-tos/field/field_overview.ipynb` +- :ref:`/how-tos/grib/grib_overview.ipynb` +- :ref:`/how-tos/xr_engine/xarray_engine_chem.ipynb` +- :ref:`/how-tos/xr_engine/xarray_engine_wave_spectra.ipynb` diff --git a/docs/source/concepts/field/proc.rst b/docs/source/concepts/field/proc.rst new file mode 100644 index 000000000..179b34b04 --- /dev/null +++ b/docs/source/concepts/field/proc.rst @@ -0,0 +1,95 @@ +.. _proc_component: + +Processing (proc) component +============================ + +Every :py:class:`~earthkit.data.core.field.Field` may carry a *processing component* (``proc``) +that describes post-processing operations applied to produce the field. The processing component +is accessible via the :attr:`proc` attribute of a field and is represented by a subclass of +:py:class:`~earthkit.data.field.component.proc.ProcBase`. + +Fields that carry no processing information have an +:py:class:`~earthkit.data.field.component.proc.EmptyProc` component where all keys +return ``None``. + +.. note:: + + The ``proc`` component interface is still under active development. Its final form + is not yet fully defined, and breaking changes may occur in future releases. + +.. code-block:: python + + >>> import earthkit.data as ekd + >>> field = ekd.from_source("file", "lsp_step_range.grib2").to_fieldlist()[0] + >>> field.proc.time_method() + accum + >>> field.proc.time_value() + datetime.timedelta(seconds=259200) + >>> field.get("proc.time_method") + accum + + +Processing items +---------------- + +The processing component stores a list of +:py:class:`~earthkit.data.field.component.proc.ProcItem` instances. Currently the only +supported item type is +:py:class:`~earthkit.data.field.component.proc.TimeProcItem`, which pairs a time-span value +with a processing method. + +Time processing methods +~~~~~~~~~~~~~~~~~~~~~~~ + +.. list-table:: + :header-rows: 1 + :widths: 20 20 60 + + * - Enum member + - Name string + - Description + * - ``TimeMethods.ACCUMULATED`` + - ``"accum"`` + - Field values represent an accumulation over the time span (e.g. precipitation totals). + * - ``TimeMethods.AVERAGE`` + - ``"avg"`` + - Field values represent a time average over the time span. + * - ``TimeMethods.INSTANT`` + - ``"instant"`` + - Field values are instantaneous (no time averaging or accumulation). + * - ``TimeMethods.MAX`` + - ``"max"`` + - Field values represent the maximum over the time span. + + +Accessing processing information +--------------------------------- + +All proc keys are accessible through :meth:`~earthkit.data.core.field.Field.get` with the +``"proc."`` prefix, and can therefore be used in +:meth:`~earthkit.data.core.fieldlist.FieldList.sel`, +:meth:`~earthkit.data.core.fieldlist.FieldList.order_by`, and +:meth:`~earthkit.data.core.fieldlist.FieldList.metadata`. + +.. list-table:: + :header-rows: 1 + :widths: 32 68 + + * - Key + - Description + * - ``proc.time`` + - The first :py:class:`~earthkit.data.field.component.proc.TimeProcItem` in the processing + list, or ``None`` if no time processing is present. + * - ``proc.time_value`` + - The time-span value from the first time processing item as a :py:class:`datetime.timedelta`, + or ``None``. + * - ``proc.time_method`` + - The processing method from the first time processing item as a + :py:class:`~earthkit.data.field.component.time_span.TimeMethod`, or ``None``. + + +How-tos +------- + +- :ref:`/how-tos/field/field_overview.ipynb` +- :ref:`/how-tos/grib/grib_overview.ipynb` diff --git a/docs/source/concepts/field/time.rst b/docs/source/concepts/field/time.rst new file mode 100644 index 000000000..316c6dd68 --- /dev/null +++ b/docs/source/concepts/field/time.rst @@ -0,0 +1,97 @@ +.. _time_component: + +Time component +============== + +Every :py:class:`~earthkit.data.core.field.Field` carries a *time component* that describes the +temporal coordinate of the field. The time component is accessible via the :attr:`time` attribute +of a field and is represented by a subclass of +:py:class:`~earthkit.data.field.component.time.TimeBase`. + +.. code-block:: python + + >>> import earthkit.data as ekd + >>> field = ekd.from_source("sample", "test.grib").to_fieldlist()[0] + >>> field.time.base_datetime() + datetime.datetime(2020, 1, 1, 0, 0) + >>> field.time.step() + datetime.timedelta(0) + >>> field.time.valid_datetime() + datetime.datetime(2020, 1, 1, 0, 0) + +The same information is available through the generic :meth:`~earthkit.data.core.field.Field.get` +interface using the ``"time."`` prefix: + +.. code-block:: python + + >>> field.get("time.base_datetime") + datetime.datetime(2020, 1, 1, 0, 0) + >>> field.get("time.step") + datetime.timedelta(0) + +The time component is **immutable**. Use the +:meth:`~earthkit.data.field.component.time.TimeBase.set` method (or +:meth:`~earthkit.data.core.field.Field.set` on the field) to derive a modified copy: + +.. code-block:: python + + >>> new_field = field.set({"time.step": 6}) + >>> new_field.time.step() + datetime.timedelta(seconds=21600) + + +Time types +---------- + +The appropriate time subclass is determined automatically from the data: + +- :py:class:`~earthkit.data.field.component.time.ForecastTime` — standard forecast time + defined by a base datetime and a forecast step. The valid datetime is computed as + ``base_datetime + step``. +- :py:class:`~earthkit.data.field.component.time.MonthlyForecastTime` — monthly forecast time + that also carries a ``forecast_month`` value. + + +Accessing time information +-------------------------- + +All time keys are accessible through :meth:`~earthkit.data.core.field.Field.get` with the +``"time."`` prefix, and can therefore be used in +:meth:`~earthkit.data.core.fieldlist.FieldList.sel`, +:meth:`~earthkit.data.core.fieldlist.FieldList.order_by`, and +:meth:`~earthkit.data.core.fieldlist.FieldList.metadata`. + +.. list-table:: + :header-rows: 1 + :widths: 32 68 + + * - Key + - Description + * - ``time.base_datetime`` + - The base (reference) datetime of the forecast as a :py:class:`datetime.datetime`. + * - ``time.forecast_reference_time`` + - Alias of ``time.base_datetime``. + * - ``time.base_date`` + - The date part of the base datetime as a :py:class:`datetime.date`. + * - ``time.base_time`` + - The time-of-day part of the base datetime as a :py:class:`datetime.time`. + * - ``time.valid_datetime`` + - The valid datetime (``base_datetime + step``) as a :py:class:`datetime.datetime`. + * - ``time.step`` + - The forecast step as a :py:class:`datetime.timedelta`. Integer values are interpreted as hours. + * - ``time.forecast_period`` + - Alias of ``time.step``. + * - ``time.forecast_month`` + - The forecast month as an integer. Only available for monthly forecast time; returns ``None`` otherwise. + * - ``time.indexing_datetime`` + - The indexing datetime used for ordering within a dataset. Returns ``None`` for types that do not define it. + + +How-tos +------- + +- :ref:`/how-tos/field/field_overview.ipynb` +- :ref:`/how-tos/grib/grib_overview.ipynb` +- :ref:`/how-tos/grib/grib_time_series.ipynb` +- :ref:`/how-tos/xr_engine/xarray_engine_temporal.ipynb` +- :ref:`/how-tos/xr_engine/xarray_engine_step_ranges.ipynb` diff --git a/docs/source/concepts/field/vertical.rst b/docs/source/concepts/field/vertical.rst new file mode 100644 index 000000000..2ca623c63 --- /dev/null +++ b/docs/source/concepts/field/vertical.rst @@ -0,0 +1,498 @@ +.. _vertical_component: + +Vertical component +================== + +Every :py:class:`~earthkit.data.core.field.Field` carries a *vertical component* that describes +where in the atmosphere, ocean, or land the field is defined. The vertical component is accessible +via the :attr:`vertical` attribute of a field and is represented by a +:py:class:`~earthkit.data.field.component.vertical.Vertical` object (or its parametric subclass +for hybrid levels). + +.. code-block:: python + + >>> import earthkit.data as ekd + >>> field = ekd.from_source("sample", "tuv_pl.grib").to_fieldlist()[0] + >>> field.vertical.level() + 1000 + >>> field.vertical.level_type() + 'pressure' + >>> field.vertical.units() + hPa + >>> field.vertical.positive() + 'down' + +The same information is available through the generic :meth:`~earthkit.data.core.field.Field.get` +interface using the ``"vertical."`` prefix: + +.. code-block:: python + + >>> field.get("vertical.level") + 1000 + >>> field.get("vertical.level_type") + 'pressure' + +The vertical component is **immutable**. Use the :meth:`~earthkit.data.field.component.vertical.Vertical.set` +method (or :meth:`~earthkit.data.core.field.Field.set` on the field) to derive a modified copy: + +.. code-block:: python + + >>> new_field = field.set({"vertical.level": 500}) + >>> new_field.vertical.level() + 500 + + +Level types +----------- + +The *level type* classifies the nature of the vertical coordinate. It is represented by a +:py:class:`~earthkit.data.field.component.level_type.LevelType` object, which carries the +following metadata: + +- **name** – canonical identifier used throughout earthkit-data (e.g. ``"pressure"``) +- **abbreviation** – short label (e.g. ``"pl"``) +- **standard_name** – CF standard name (e.g. ``"air_pressure"``) +- **long_name** – human-readable description +- **units** – native units of the level value +- **layer** – ``True`` when the type represents a layer (two bounding levels) rather than a single level +- **level** – indicates which bound is the *representative* level for layer types: ``"top"``, ``"bottom"``, or empty string when undefined +- **positive** – vertical direction in which values increase: ``"up"``, ``"down"``, or empty string when undefined + +When earthkit-data encounters a level type that is not in the predefined list below, it is +automatically registered at runtime so that arbitrary model-specific level types are handled +transparently. + + +Predefined level types +~~~~~~~~~~~~~~~~~~~~~~ + +.. list-table:: + :header-rows: 1 + :widths: 22 16 14 14 8 6 8 12 + + * - Name + - CF standard name + - Long name + - Abbreviation + - Units + - Layer + - Level + - Positive + * - ``abstract_single_level`` + - abstract_single_level + - abstract single level + - ``abstractSingleLevel`` + - + - No + - + - + * - ``cloud_base`` + - cloud_base + - cloud base + - ``cloudBase`` + - + - No + - + - + * - ``depth_below_land_layer`` + - depth + - soil depth + - ``d_bll_layer`` + - m + - Yes + - top + - down + * - ``depth_below_land_level`` + - depth + - soil depth + - ``d_bll`` + - m + - No + - + - down + * - ``depth_below_sea_layer`` + - depth + - depth + - ``d_bsl_layer`` + - m + - Yes + - top + - down + * - ``entire_atmosphere`` + - entire_atmosphere + - entire atmosphere + - ``entire_atmosphere`` + - + - No + - + - + * - ``entire_lake`` + - entire_lake + - entire lake + - ``entireLake`` + - + - No + - + - + * - ``entire_melt_pond`` + - entire_melt_pond + - entire melt pond + - ``entireMeltPond`` + - + - No + - + - + * - ``general`` + - general + - general + - ``gen`` + - 1 + - No + - + - down + * - ``height_above_ground_level`` + - height + - height above the surface + - ``h_agl`` + - m + - No + - + - up + * - ``height_above_mean_sea_level`` + - height_above_mean_sea_level + - height above mean sea level + - ``h_asl`` + - m + - No + - + - up + * - ``high_cloud_layer`` + - high_cloud_layer + - high cloud layer + - ``highCloudLayer`` + - hPa + - Yes + - top + - + * - ``hybrid`` + - atmosphere_hybrid_sigma_pressure_coordinate + - hybrid level + - ``ml`` + - 1 + - No (parametric) + - + - down + * - ``ice_layer_on_water`` + - ice_layer_on_water + - ice layer on water + - ``iceLayerOnWater`` + - 1 + - Yes + - top + - up + * - ``ice_top_on_water`` + - ice_top_on_water + - ice top on water + - ``iceTopOnWater`` + - 1 + - No + - + - + * - ``lake_bottom`` + - lake_bottom + - lake bottom + - ``lakeBottom`` + - 1 + - No + - + - + * - ``low_cloud_layer`` + - low_cloud_layer + - low cloud layer + - ``lowCloudLayer`` + - hPa + - Yes + - top + - + * - ``mean_sea`` + - mean_sea + - mean sea level + - ``mean_sea`` + - + - No + - + - + * - ``medium_cloud_layer`` + - medium_cloud_layer + - medium cloud layer + - ``mediumCloudLayer`` + - hPa + - Yes + - top + - + * - ``mixed_layer_depth_by_density`` + - mixed_layer_depth_by_density + - mixed layer depth by density + - ``mixedLayerDepthByDensity`` + - m + - No + - + - down + * - ``mixed_layer_parcel`` + - mixed_layer_parcel + - mixed layer parcel + - ``mixedLayerParcel`` + - Pa + - No + - + - + * - ``mixing_layer`` + - mixing_layer + - mixing layer + - ``mixingLayer`` + - + - No + - + - + * - ``most_unstable_parcel`` + - most_unstable_parcel + - most unstable parcel + - ``mostUnstableParcel`` + - + - No + - + - + * - ``nominal_top_of_atmosphere`` + - nominal_top_of_atmosphere + - nominal top of atmosphere + - ``nominalTopOfAtmosphere`` + - + - No + - + - + * - ``ocean_model`` + - ocean_model + - ocean model + - ``oceanModel`` + - 1 + - No + - + - down + * - ``ocean_model_layer`` + - ocean_model_layer + - ocean model layer + - ``oceanModelLayer`` + - 1 + - Yes + - bottom + - down + * - ``ocean_surface`` + - ocean_surface + - ocean surface + - ``ocean_surface`` + - + - No + - + - + * - ``ocean_surface_to_bottom`` + - ocean_surface_to_bottom + - ocean surface to bottom + - ``oceanSurfaceToBottom`` + - 1 + - No + - + - + * - ``potential_temperature`` + - air_potential_temperature + - air potential temperature + - ``pt`` + - K + - No + - + - up + * - ``potential_vorticity`` + - ertel_potential_vorticity + - potential vorticity + - ``pv`` + - 10E-9 K m2 kg-1 s-1 + - No + - + - down + * - ``pressure`` + - air_pressure + - pressure + - ``pl`` + - hPa + - No + - + - down + * - ``pressure_layer`` + - air_pressure + - pressure + - ``p_layer`` + - hPa + - Yes + - top + - down + * - ``sea_ice_layer`` + - sea_ice_layer + - sea ice layer + - ``seaIceLayer`` + - 1 + - Yes + - bottom + - down + * - ``snow`` + - unknown + - snow layer + - ``snow`` + - 1 + - Yes + - bottom + - down + * - ``snow_layer_over_ice_on_water`` + - snow_layer_over_ice_on_water + - snow layer over ice on water + - ``snowLayerOverIceOnWater`` + - 1 + - No + - + - + * - ``soil_layer`` + - soil_layer + - soil layer + - ``soilLayer`` + - 1 + - Yes + - bottom + - down + * - ``stratosphere`` + - stratosphere + - stratosphere + - ``stratosphere`` + - 1 + - No + - + - + * - ``surface`` + - surface + - surface + - ``sfc`` + - + - No + - + - + * - ``temperature`` + - temperature + - temperature + - ``isothermal`` + - K + - No + - + - + * - ``tropopause`` + - tropopause + - tropopause + - ``tropopause`` + - + - No + - + - + * - ``troposphere`` + - troposphere + - troposphere + - ``troposphere`` + - 1 + - No + - + - + * - ``unknown`` + - unknown + - unknown + - ``unknown`` + - + - No + - + - + * - ``water_surface_to_isothermal_ocean_layer`` + - water_surface_to_isothermal_ocean_layer + - water surface to isothermal ocean layer + - ``waterSurfaceToIsothermalOceanLayer`` + - 1 + - No + - + - + + +Layers +------ + +When ``layer`` is ``True``, the vertical component stores two bounding level values rather than a +single value. The ``level`` attribute (and the ``"vertical.level"`` key) returns the *representative* +scalar level, derived from either the top or the bottom bound depending on the *Level* column in the +table above. The full pair of bounds is available via +:meth:`~earthkit.data.field.component.vertical.Vertical.layer`. + + +Parametric levels +----------------- + +The ``hybrid`` level type is *parametric*: the actual pressure at any grid point must be computed +from a set of coefficients (``A`` and ``B``) together with the surface pressure field. For hybrid +levels the vertical component exposes the following additional methods: + +- :meth:`~earthkit.data.field.component.vertical.VerticalBase.coefficients` – the ``A`` and ``B`` + coefficient arrays +- :meth:`~earthkit.data.field.component.vertical.VerticalBase.coefficient_names` – the names of + the coefficients (``("A", "B")``) +- :meth:`~earthkit.data.field.component.vertical.VerticalBase.number_of_levels` – total number of + model levels + + +Accessing vertical information +------------------------------- + +All vertical keys are accessible through :meth:`~earthkit.data.core.field.Field.get` with the +``"vertical."`` prefix, and can therefore be used in :meth:`~earthkit.data.core.fieldlist.FieldList.sel`, +:meth:`~earthkit.data.core.fieldlist.FieldList.order_by`, and +:meth:`~earthkit.data.core.fieldlist.FieldList.metadata`. + +Supported keys: + +.. list-table:: + :header-rows: 1 + :widths: 28 72 + + * - Key + - Description + * - ``vertical.level`` + - Scalar level value in the native units of the level type + * - ``vertical.level_type`` + - Level type name as a string (e.g. ``"pressure"``) + * - ``vertical.layer`` + - Layer bounds as a ``(bottom, top)`` tuple, or ``None`` for non-layer types + * - ``vertical.abbreviation`` + - Abbreviation of the level type (e.g. ``"pl"``) + * - ``vertical.units`` + - Units of the level value + * - ``vertical.positive`` + - Positive direction of the coordinate (``"up"``, ``"down"``, or ``None``) + * - ``vertical.cf`` + - Dictionary of CF metadata (``standard_name``, ``long_name``, ``units``, ``positive``) + * - ``vertical.parametric`` + - ``True`` if the level type is parametric (e.g. hybrid levels) + * - ``vertical.coefficients`` + - Coefficient arrays for parametric level types, ``None`` otherwise + * - ``vertical.coefficient_names`` + - Names of the coefficients for parametric level types, ``None`` otherwise + * - ``vertical.number_of_levels`` + - Number of model levels for parametric level types, ``None`` otherwise + + +How-tos +------- + +- :ref:`/how-tos/field/field_overview.ipynb` +- :ref:`/how-tos/grib/grib_overview.ipynb` +- :ref:`/how-tos/xr_engine/xarray_engine_level.ipynb` +- :ref:`/how-tos/xr_engine/xarray_engine_field_dims.ipynb` diff --git a/docs/source/concepts/fieldlist.rst b/docs/source/concepts/fieldlist.rst index e69de29bb..f212ffdbd 100644 --- a/docs/source/concepts/fieldlist.rst +++ b/docs/source/concepts/fieldlist.rst @@ -0,0 +1,197 @@ +.. _fieldlist_concept: + +FieldList +========= + +A :py:class:`~earthkit.data.core.fieldlist.FieldList` is an ordered, indexable collection of +:py:class:`~earthkit.data.core.field.Field` objects. It is the primary interface returned by +:py:func:`~earthkit.data.from_source` and acts as the main entry point for working with +multi-field datasets in EarthKit Data. + +.. code-block:: python + + >>> import earthkit.data as ekd + >>> ds = ekd.from_source("sample", "tuv_pl.grib").to_fieldlist() + >>> len(ds) + 18 + >>> ds[0] + GribField(u, 1000, 2020-01-01 00:00, 0, None, None) + + +Indexing and slicing +-------------------- + +FieldLists support integer indexing and Python slice notation. Slicing returns a new +FieldList containing the selected fields: + +.. code-block:: python + + >>> ds[0] # single field + >>> ds[0:3] # slice — returns a FieldList + >>> ds[-1] # last field + + +Iteration +--------- + +Iterating over a FieldList yields individual +:py:class:`~earthkit.data.core.field.Field` objects one at a time: + +.. code-block:: python + + >>> for field in ds: + ... print(field.parameter.variable(), field.vertical.level()) + + +Selection +--------- + +:meth:`~earthkit.data.core.fieldlist.FieldList.sel` filters a FieldList by metadata values and +returns a new FieldList containing only the matching fields. Keys follow the +``"component.key"`` convention used throughout EarthKit Data: + +.. code-block:: python + + >>> pl500 = ds.sel({"vertical.level": 500}) + >>> wind = ds.sel({"parameter.variable": ["u", "v"]}) + +Source-native keys (e.g. GRIB ``shortName``, ``level``) can also be used by prefixing them +with ``"metadata."``: + +.. code-block:: python + + >>> plt500 = ds.sel({"metadata.shortName": "t", "metadata.level": 500}) + + +Ordering +-------- + +:meth:`~earthkit.data.core.fieldlist.FieldList.order_by` returns a new FieldList sorted by +one or more metadata keys. Multiple keys can be passed as a list and are applied in order: + +.. code-block:: python + + >>> ds_sorted = ds.order_by(["parameter.variable", "vertical.level"]) + + +Metadata access +--------------- + +:meth:`~earthkit.data.core.fieldlist.FieldList.get` returns a list of metadata values, +one per field, using the ``"component.key"`` keys described in the component pages: + +.. code-block:: python + + >>> ds.get("parameter.variable") + ['u', 'v', 't', ...] + >>> ds.get(["parameter.variable", "vertical.level"]) + [('u', 1000), ('v', 1000), ('t', 1000), ...] + +For source-native keys (e.g. GRIB ``shortName``, ``level``), +:meth:`~earthkit.data.core.fieldlist.FieldList.metadata` can be used directly without any +prefix: + +.. code-block:: python + + >>> ds.metadata("shortName") + ['u', 'v', 't', ...] + +The :meth:`~earthkit.data.core.fieldlist.FieldList.ls` method provides a quick tabular +summary of the most commonly used metadata keys, In Jupyter notebooks, the output is rendered as a table; in other environments e.g. terminal, it has to be printed with ``print()`` to see the table: + + +.. code-block:: python + + >>> ds.ls() # in Jupyter notebook, this is rendered as a table + +.. code-block:: python + + >>> print(ds.ls()) # in terminal, an extra print() is needed to render the table + +Extracting data +--------------- + +:meth:`~earthkit.data.core.fieldlist.FieldList.to_numpy` returns the field values stacked +into a NumPy array and accepts ``dtype``, ``copy``, and other arguments. The shape of the +result depends on the shape of the individual fields: + +- for fields with a 1-D grid (e.g. unstructured grids), the result has shape + ``(number_of_fields, number_of_grid_points)``; +- for fields with a 2-D grid (e.g. regular lat/lon grids), the result has shape + ``(number_of_fields, Ny, Nx)``. + +By default a **copy** of the data is returned. Pass ``copy=False`` to avoid the copy when +the data layout allows it; note that a copy may still be made if the underlying source +cannot provide a zero-copy view: + +.. code-block:: python + + >>> ds.to_numpy(dtype="float32").shape + (18, 19, 36) + >>> ds.to_numpy(copy=False).shape + (18, 19, 36) + +:attr:`~earthkit.data.core.fieldlist.FieldList.values` is a convenience property that +always returns a 2-D array of shape ``(number_of_fields, number_of_grid_points)``, +where each row is the flat 1-D array of values for one field. The array type matches the +native array format of the underlying data (e.g. NumPy for GRIB and NetCDF fields, or a +GPU array for array-backed FieldLists). Each access returns a **copy**: + +.. code-block:: python + + >>> ds.values.shape + (18, 684) + + +Converting to Xarray +-------------------- + +:meth:`~earthkit.data.core.fieldlist.FieldList.to_xarray` converts the FieldList to an +:py:class:`xarray.Dataset`. EarthKit Data uses a dedicated Xarray engine that maps +field metadata to dataset dimensions and coordinates: + +.. code-block:: python + + >>> xr_ds = ds.to_xarray() + >>> xr_ds + + Dimensions: ... + + +Concatenation +------------- + +Two FieldLists can be concatenated with the ``+`` operator, producing a new FieldList that +contains all fields from both operands in order: + +.. code-block:: python + + >>> combined = ds1 + ds2 + >>> len(combined) == len(ds1) + len(ds2) + True + + +FieldList types +--------------- + +There are several concrete FieldList implementations, each suited to a different access +pattern: + +- **SimpleFieldList** — in-memory list, produced by most operations such as + :meth:`sel`, :meth:`order_by`, and ``+``. +- **StreamFieldList** — backed by a streaming source (e.g. FDB or URL stream). Supports + forward iteration only; use :meth:`to_fieldlist` with ``read_all=True`` to materialise + into memory. +- **ArrayFieldList** — backed by NumPy arrays, used when constructing fields + programmatically. +- **FileFieldList** — backed by an on-disk file (e.g. a cached GRIB file). + + +How-tos +------- + +- :ref:`/how-tos/field/field_overview.ipynb` +- :ref:`/how-tos/grib/grib_overview.ipynb` +- :ref:`/how-tos/grib/grib_selection.ipynb` +- :ref:`/how-tos/grib/grib_order_by.ipynb` +- :ref:`/how-tos/xr_engine/xarray_engine_overview.ipynb` diff --git a/docs/source/concepts/index.rst b/docs/source/concepts/index.rst index d7e9218c5..0cc0f1ac0 100644 --- a/docs/source/concepts/index.rst +++ b/docs/source/concepts/index.rst @@ -15,4 +15,6 @@ Concepts xarray/index.rst plugins/index.rst misc/index.rst - field + field/index.rst + fieldlist + vertical diff --git a/docs/source/conf.py b/docs/source/conf.py index 6825ea0e7..2cba9363d 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -231,6 +231,7 @@ "ncdata": ("https://ncdata.readthedocs.io/en/latest/", None), "iris": ("https://scitools-iris.readthedocs.io/en/latest/", None), "pyodc": ("https://pyodc.readthedocs.io/en/latest/", None), + "earthkit-utils": ("https://earthkit-utils.readthedocs.io/en/latest/", None), } diff --git a/src/earthkit/data/field/component/level_type.py b/src/earthkit/data/field/component/level_type.py index 98a761769..4d3f6687b 100644 --- a/src/earthkit/data/field/component/level_type.py +++ b/src/earthkit/data/field/component/level_type.py @@ -67,7 +67,7 @@ def __init__( units: Union[str, Units], layer: bool, positive: str, - level: str = TOP_LEVEL, + level: str = str(), parametric: bool = False, coefficient_names: tuple[str, ...] | None = None, ) -> None: @@ -119,6 +119,12 @@ def __init__( "positive": self.positive, } + if layer and level not in (TOP_LEVEL, BOTTOM_LEVEL, ""): + raise ValueError( + f"Invalid level value for layer type {name}: {level}. Must be " + f"one of: {TOP_LEVEL}, {BOTTOM_LEVEL}, or empty string." + ) + def __eq__(self, other) -> bool: """Check if this LevelType is equal to another LevelType or a string.