From d77ba320765ca5da65e64ef8052a09d7702084e3 Mon Sep 17 00:00:00 2001 From: Sandor Kertesz Date: Tue, 30 Jun 2026 11:25:30 +0100 Subject: [PATCH 1/3] Fix to_xarray kwargs for netcdf data object --- src/earthkit/data/data/netcdf.py | 20 ++++- src/earthkit/data/readers/netcdf/reader.py | 20 +++-- tests/netcdf/test_netcdf_convert.py | 89 +++++++++++++++++++++- 3 files changed, 114 insertions(+), 15 deletions(-) diff --git a/src/earthkit/data/data/netcdf.py b/src/earthkit/data/data/netcdf.py index 38c130022..ce9a6db45 100644 --- a/src/earthkit/data/data/netcdf.py +++ b/src/earthkit/data/data/netcdf.py @@ -59,18 +59,32 @@ def to_xarray(self, *args, xarray_open_mfdataset_kwargs=None, **kwargs): Parameters ---------- + *args + Positional arguments to pass to the reader's to_xarray method. + Not used currently. It is there to allow for future extensions or + additional parameters that may be needed for specific use cases. xarray_open_mfdataset_kwargs: dict, None, optional Keyword arguments passed to :py:func:`xarray.open_mfdataset`. + Can only be used when converting multiple NetCDF files into a single + Xarray dataset. When specified, this argument takes precedence over + any other keyword arguments passed to the method. **kwargs - Keyword arguments passed to :py:func:`xarray.open_mfdataset` - ``xarray_open_mfdataset_kwargs`` is not set. + Keyword arguments passed to :py:func:`xarray.open_dataset` for + single file conversion or to :py:func:`xarray.open_mfdataset` for + multiple file conversion. Ignored if `xarray_open_mfdataset_kwargs` + is specified. Returns ------- :py:class:`xarray.Dataset` An Xarray dataset containing the NetCDF data. """ - return self._reader.to_xarray(*args, **kwargs) + if xarray_open_mfdataset_kwargs: + options = dict(xarray_open_mfdataset_kwargs=xarray_open_mfdataset_kwargs) + else: + options = dict() + + return self._reader.to_xarray(*args, **options, **kwargs) def to_pandas(self, *args, **kwargs): """Convert into a Pandas DataFrame. diff --git a/src/earthkit/data/readers/netcdf/reader.py b/src/earthkit/data/readers/netcdf/reader.py index 96fd5d5ee..39085d3d1 100644 --- a/src/earthkit/data/readers/netcdf/reader.py +++ b/src/earthkit/data/readers/netcdf/reader.py @@ -40,12 +40,6 @@ def to_pandas(self): def to_xarray(self, **kwargs): return type(self).to_xarray_single_from_path(self.path, **kwargs) - # if isinstance(self.path, str): - # return type(self).to_xarray_single_from_path(self.path, **kwargs) - - # return NetCDFReader.to_xarray_multi_from_paths([self.path], **kwargs) - - # return type(self).to_xarray_multi_from_paths([self.path], **kwargs) @classmethod def to_xarray_multi_from_paths(cls, paths, **kwargs): @@ -54,8 +48,13 @@ def to_xarray_multi_from_paths(cls, paths, **kwargs): if not isinstance(paths, list): paths = [paths] + kwargs = kwargs.copy() # Make a copy to avoid modifying the original dictionary + xarray_open_mfdataset_kwargs = kwargs.pop("xarray_open_mfdataset_kwargs", None) + options = dict() - options.update(kwargs.get("xarray_open_mfdataset_kwargs", {})) + if xarray_open_mfdataset_kwargs is not None: + options = dict(xarray_open_mfdataset_kwargs) + if not options: options = dict(**kwargs) @@ -74,8 +73,13 @@ def to_xarray_single_from_path(cls, path, **kwargs): if "xarray_open_mfdataset_kwargs" in kwargs: raise ValueError("xarray_open_mfdataset_kwargs is not supported for single file") + kwargs = kwargs.copy() # Make a copy to avoid modifying the original dictionary + xarray_open_dataset_kwargs = kwargs.pop("xarray_open_dataset_kwargs", None) + options = dict() - options.update(kwargs.get("xarray_open_dataset_kwargs", {})) + if xarray_open_dataset_kwargs is not None: + options = dict(xarray_open_dataset_kwargs) + if not options: options = dict(**kwargs) diff --git a/tests/netcdf/test_netcdf_convert.py b/tests/netcdf/test_netcdf_convert.py index d7f2db17c..87c329215 100644 --- a/tests/netcdf/test_netcdf_convert.py +++ b/tests/netcdf/test_netcdf_convert.py @@ -9,16 +9,17 @@ # nor does it submit to any jurisdiction. # + import numpy as np import pytest from earthkit.data import from_source -from earthkit.data.utils.testing import earthkit_remote_test_data_file +from earthkit.data.utils.testing import earthkit_remote_test_data_file, earthkit_test_data_file -@pytest.mark.long_test -@pytest.mark.download -def test_netcdf_to_xarray_args(): +# @pytest.mark.long_test +# @pytest.mark.download +def test_netcdf_fieldlist_to_xarray_args(): # The JD variable in the NetCDF is defined as follows: # # short JD(time, lat, lon) ; @@ -55,6 +56,86 @@ def test_netcdf_to_xarray_args(): r["JD"].shape == (1, 20880, 28440) +# @pytest.mark.long_test +# @pytest.mark.download +def test_netcdf_single_to_xarray_args(): + # The JD variable in the NetCDF is defined as follows: + # + # short JD(time, lat, lon) ; + # string JD:long_name = "Date of the first detection" ; + # string JD:units = "days since 2022-01-01" ; + # string JD:comment = "Possible values: 0 when the pixel is not burned; 1 to 366 day of + # the first detection when the pixel is burned; -1 when the pixel is not observed + # in the month; -2 when pixel is not burnable: water bodies, bare areas, urban areas, + # and permanent snow and ice. + # + # when loaded with xarray.open_dataset/xarray.open_mdataset without any kwargs the + # type of the JD variable is datetime64[ns], which is wrong. The correct type should + # be int16. + + ds = from_source( + "url", + earthkit_remote_test_data_file("20220401-C3S-L3S_FIRE-BA-OLCI-AREA_3-fv1.1.nc"), + ) + + r = ds.to_xarray(decode_cf=False, decode_times=False) + assert r["JD"].dtype == "int16" + r["JD"].shape == (1, 20880, 28440) + assert np.isclose(r["JD"].values.min(), -2) + assert np.isclose(r["JD"].values.max(), 120) + + r = ds.to_xarray(decode_cf=False, decode_times=False) + assert r["JD"].dtype == "int16" + r["JD"].shape == (1, 20880, 28440) + assert np.isclose(r["JD"].values.min(), -2) + assert np.isclose(r["JD"].values.max(), 120) + + r = ds.to_xarray() + assert r["JD"].dtype == " Date: Tue, 30 Jun 2026 11:27:21 +0100 Subject: [PATCH 2/3] Fix to_xarray kwargs for netcdf data object --- tests/netcdf/test_netcdf_convert.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/netcdf/test_netcdf_convert.py b/tests/netcdf/test_netcdf_convert.py index 87c329215..c5001d753 100644 --- a/tests/netcdf/test_netcdf_convert.py +++ b/tests/netcdf/test_netcdf_convert.py @@ -17,8 +17,8 @@ from earthkit.data.utils.testing import earthkit_remote_test_data_file, earthkit_test_data_file -# @pytest.mark.long_test -# @pytest.mark.download +@pytest.mark.long_test +@pytest.mark.download def test_netcdf_fieldlist_to_xarray_args(): # The JD variable in the NetCDF is defined as follows: # @@ -56,8 +56,8 @@ def test_netcdf_fieldlist_to_xarray_args(): r["JD"].shape == (1, 20880, 28440) -# @pytest.mark.long_test -# @pytest.mark.download +@pytest.mark.long_test +@pytest.mark.download def test_netcdf_single_to_xarray_args(): # The JD variable in the NetCDF is defined as follows: # From 510c6e9d222e89fa8e41434ec485626371d0d11a Mon Sep 17 00:00:00 2001 From: Sandor Kertesz Date: Tue, 30 Jun 2026 11:54:55 +0100 Subject: [PATCH 3/3] Fix to_xarray kwargs for netcdf data object --- src/earthkit/data/readers/netcdf/reader.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/earthkit/data/readers/netcdf/reader.py b/src/earthkit/data/readers/netcdf/reader.py index 39085d3d1..302b80e83 100644 --- a/src/earthkit/data/readers/netcdf/reader.py +++ b/src/earthkit/data/readers/netcdf/reader.py @@ -70,11 +70,11 @@ def to_xarray_single_from_path(cls, path, **kwargs): if not isinstance(path, str): raise ValueError("path must be a string") - if "xarray_open_mfdataset_kwargs" in kwargs: - raise ValueError("xarray_open_mfdataset_kwargs is not supported for single file") - kwargs = kwargs.copy() # Make a copy to avoid modifying the original dictionary xarray_open_dataset_kwargs = kwargs.pop("xarray_open_dataset_kwargs", None) + xarray_open_mfdataset_kwargs = kwargs.pop("xarray_open_mfdataset_kwargs", None) + if xarray_open_mfdataset_kwargs: + raise ValueError("xarray_open_mfdataset_kwargs is not supported for single file") options = dict() if xarray_open_dataset_kwargs is not None: