From 47e77bc8638e317fbc12bc92cc106a0e4d3498cd Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Tue, 18 Aug 2026 16:46:06 -0500 Subject: [PATCH 01/14] Rewrite `OneHotEncoder`, `OrdinalEncoder` Rewrites `OneHotEncoder` and `OrdinalEncoder` - Fixes several bugs in implementation of each, improving sklearn compatibility. - Improves testing across estimators - Fixes type reflection handling of each to match documented behavior and be consistent with other cuml estimators. --- python/cuml/cuml/preprocessing/encoders.py | 1051 +++++++++++--------- python/cuml/tests/test_one_hot_encoder.py | 590 +++++------ python/cuml/tests/test_ordinal_encoder.py | 322 ++++-- 3 files changed, 1064 insertions(+), 899 deletions(-) diff --git a/python/cuml/cuml/preprocessing/encoders.py b/python/cuml/cuml/preprocessing/encoders.py index ba874365f4..2835be41f1 100644 --- a/python/cuml/cuml/preprocessing/encoders.py +++ b/python/cuml/cuml/preprocessing/encoders.py @@ -1,182 +1,188 @@ # SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# -import warnings -from typing import Optional +from collections.abc import Sequence import cudf import cupy as cp -import cupyx +import cupyx.scipy.sparse as cp_sp import numpy as np -from cudf import Index -import cuml from cuml.common.doc_utils import generate_docstring from cuml.internals.base import Base from cuml.internals.mixins import DeprecatedGetFeatureNamesMixin from cuml.internals.outputs import mlfunc from cuml.internals.validation import ( + check_array, + check_cudf, check_features, check_input_features, check_is_fitted, ) -from cuml.preprocessing._label import LabelEncoder -class BaseEncoder(Base): - """Base implementation for encoding categorical values, uses - :py:class:`~cuml.preprocessing.LabelEncoder` for obtaining unique values. - - Parameters - ---------- - verbose : int or boolean, default=False - Sets logging level. It must be one of `cuml.common.logger.level_*`. - See :ref:`verbosity-levels` for more info. - output_type : {None, 'input', 'cupy', 'numpy', 'cudf', 'pandas'}, default=None - Return results and set estimator attributes to the indicated output - type. If None, the output type set at the module level - (`cuml.global_settings.output_type`) will be used. See - :ref:`output-data-type-configuration` for more info. - """ - - def _set_input_type(self, value): - if self.input_type is None: - self.input_type = value - - def _check_input(self, X, is_categories=False): - """If input is cupy, convert it to a DataFrame with 0 copies.""" - if isinstance(X, cp.ndarray): - self._set_input_type("array") - if is_categories: - X = X.transpose() - return cudf.DataFrame(X) - else: - self._set_input_type("df") - return X - - def _check_input_fit(self, X, is_categories=False): - """Helper function used in fit, can be overridden in subclasses.""" - return self._check_input(X, is_categories=is_categories) +def _safe_is_nan(x): + """Check if `x` is NaN, without erroring if non-numeric""" + try: + return np.isnan(x) + except (TypeError, ValueError): + pass + return False + + +def _get_diff(unique_vals, cats): + """Equal to ``unique_vals.difference(cats)``, but for numpy arrays not sets.""" + # Since NaN's sort last and we enforce NaN is last value in cats if + # present, we only need to check the last values. + if _safe_is_nan(cats[-1]) and _safe_is_nan(unique_vals[-1]): + unique_vals = unique_vals[:-1] + return np.setdiff1d(unique_vals, cats, assume_unique=True).tolist() + + +def _cats_to_series(cats): + """Coerce `cats` to a Series, but supporting `NaN` in object arrays""" + # XXX: `cudf.Series(['a', 'b', NaN])` errors. Here we coerce NaN->None for + # this edge case. Since we enforce NaN is last value in cats if present, we + # only need to check the last value. + if cats.dtype.kind == "O" and _safe_is_nan(cats[-1]): + cats = cats.copy() + cats[-1] = None + return cudf.Series(cats) + + +def _compute_categories(X, categories="auto", handle_unknown="error"): + """Compute `categories_` for an encoder.""" + X = check_cudf(X, input_name="X") + n_features = X.shape[1] + + if handle_unknown not in ("ignore", "error"): + raise ValueError( + "Expected `handle_unknown` to be one of ['error', 'ignore'], " + f"got {handle_unknown!r}" + ) + + if auto := (isinstance(categories, str) and categories == "auto"): + pass + elif isinstance(categories, Sequence) and not isinstance(categories, str): + if len(categories) != n_features: + raise ValueError( + "Shape mismatch: if categories is an array," + " it has to be of shape (n_features,)." + ) + else: + raise ValueError( + "Expected `categories` to be 'auto' or a sequence of " + f"array-likes, got {categories!r}" + ) - def _unique(self, inp): - """Helper function used in fit. Can be overridden in subclasses.""" + out = [] - # Default implementation passes input through directly since this is - # performed in `LabelEncoder.fit()` - return inp + for i in range(n_features): + Xi = X.iloc[:, i] - def _fit(self, X, need_drop: bool): - check_features(self, X, reset=True) - X = self._check_input_fit(X) - if type(self.categories) is str and self.categories == "auto": - self._features = X.columns - self._encoders = { - feature: LabelEncoder( - verbose=self.verbose, - output_type="cudf", - handle_unknown=self.handle_unknown, - ).fit(self._unique(X[feature])) - for feature in self._features - } + if auto: + cats = Xi.drop_duplicates().sort_values().to_numpy() else: - self.categories = self._check_input(self.categories, True) - self._features = self.categories.columns - if len(self._features) != X.shape[1]: + dtype = Xi.dtype if isinstance(Xi.dtype, np.dtype) else "O" + cats = categories[i] + cats = ( + cats.to_numpy(dtype=dtype) + if hasattr(cats, "to_numpy") + else cats.get().astype(dtype, copy=False) + if isinstance(cats, cp.ndarray) + else np.asarray(cats, dtype=dtype) + ) + + # `nan` must be the last stated category + if cats.dtype.kind == "f" and np.isnan(cats[:-1]).any(): raise ValueError( - "Shape mismatch: if categories is not 'auto'," - " it has to be of shape (n_features, _)." + "Nan should be the last element in user" + f" provided categories, see categories {cats}" + f" in column #{i}" ) - self._encoders = dict() - for feature in self._features: - le = LabelEncoder( - verbose=self.verbose, - output_type="cudf", - handle_unknown=self.handle_unknown, - ) - - self._encoders[feature] = le.fit(self.categories[feature]) - if self.handle_unknown == "error": - if self._has_unknown( - X[feature], - cudf.Series(self._encoders[feature].classes_), - ): - msg = ( - "Found unknown categories in column {0}" - " during fit".format(feature) - ) - raise KeyError(msg) + # Try using numpy.unique to check for uniqueness, falling back + # to pure python if that fails + try: + n_cats = len(np.unique(cats)) + except (TypeError, ValueError): + n_cats = len(set(cats)) + if cats.size != n_cats: + raise ValueError( + f"In column {i}, the predefined categories" + " contain duplicate elements." + ) - if need_drop: - self.drop_idx_ = self._compute_drop_idx() - self._fitted = True + if handle_unknown == "error": + present = Xi.drop_duplicates().sort_values().to_numpy() + diff = _get_diff(present, cats) + if diff: + raise ValueError( + f"Found unknown categories {diff} in column {i} during fit" + ) + out.append(cats) - @property - def categories_(self): - """Returns categories used for the one hot encoding in the correct order.""" - return [self._encoders[f].classes_ for f in self._features] + return out -class OneHotEncoder(DeprecatedGetFeatureNamesMixin, BaseEncoder): +class OneHotEncoder(DeprecatedGetFeatureNamesMixin, Base): """ Encode categorical features as a one-hot numeric array. - The input to this estimator should be a :py:class:`cuDF.DataFrame` or a - :py:class:`cupy.ndarray`, denoting the unique values taken on by categorical - (discrete) features. The features are encoded using a one-hot (aka 'one-of-K' or - 'dummy') encoding scheme. This creates a binary column for each category and returns - a sparse matrix or dense array (depending on the ``sparse_output`` parameter). + + The input to this transformer should be an array-like of integers or + strings, denoting the values taken on by categorical (discrete) features. + The features are encoded using a one-hot (aka 'one-of-K' or 'dummy') + encoding scheme. This creates a binary column for each category and + returns a sparse matrix or dense array (depending on the ``sparse_output`` + parameter). By default, the encoder derives the categories based on the unique values in each feature. Alternatively, you can also specify the `categories` manually. - .. note:: a one-hot encoding of y labels should use a LabelBinarizer - instead. - Parameters ---------- - categories : 'auto' an cupy.ndarray or a cudf.DataFrame, default='auto' - Categories (unique values) per feature: + categories : 'auto' or a list of array-like, default='auto' + Categories (unique values) per feature: - 'auto' : Determine categories automatically from the training data. + - list : ``categories[i]`` holds the categories expected in the ith + column. - - DataFrame/ndarray : ``categories[col]`` holds the categories expected - in the feature col. - - drop : 'first', None, a dict or a list, default=None + drop : 'first', None, or array-like of shape (n_features,), default=None Specifies a methodology to use to drop one of the categories per feature. This is useful in situations where perfectly collinear features cause problems, such as when feeding the resulting data - into a neural network or an unregularized regression. + into an unregularized linear regression model. - - None : retain all features (the default). + However, dropping one category breaks the symmetry of the original + representation and can therefore induce a bias in downstream models, + for instance for penalized linear classification or regression models. + - None : retain all features (the default). - 'first' : drop the first category in each feature. If only one category is present, the feature will be dropped entirely. - - - dict/list : ``drop[col]`` is the category in feature col that + - array : ``drop[i]`` is the category in feature ``X[:, i]`` that should be dropped. sparse_output : bool, default=True - This feature is not fully supported by cupy - yet, causing incorrect values when computing one hot encodings. - See https://github.com/cupy/cupy/issues/3223 + When ``True``, transform returns a sparse matrix/array in CSR format. - .. versionadded:: 24.06 - `sparse` was renamed to `sparse_output` + dtype : dtype, default=np.float32 + Desired dtype of transformed output. - dtype : number type, default=np.float - Desired datatype of transform's output. handle_unknown : {'error', 'ignore'}, default='error' - Whether to raise an error or ignore if an unknown categorical feature - is present during transform (default is to raise). When this parameter - is set to 'ignore' and an unknown category is encountered during - transform, the resulting one-hot encoded columns for this feature - will be all zeros. In the inverse transform, an unknown category - will be denoted as None. + Specifies the way unknown categories are handled during :meth:`transform`. + + - 'error' : Raise an error if an unknown category is present during transform. + - 'ignore' : When an unknown category is encountered during + transform, the resulting one-hot encoded columns for this feature + will be all zeros. In the inverse transform, an unknown category + will be denoted as None. + verbose : int or boolean, default=False Sets logging level. It must be one of `cuml.common.logger.level_*`. See :ref:`verbosity-levels` for more info. + output_type : {None, 'input', 'cupy', 'numpy', 'cudf', 'pandas'}, default=None Return results and set estimator attributes to the indicated output type. If None, the output type set at the module level @@ -185,10 +191,49 @@ class OneHotEncoder(DeprecatedGetFeatureNamesMixin, BaseEncoder): Attributes ---------- + categories_ : list of arrays + The categories of each feature determined during fitting + (in order of the features in X and corresponding with the output + of ``transform``). This includes the category specified in ``drop`` + (if any). + drop_idx_ : array of shape (n_features,) - ``drop_idx_[i]`` is the index in ``categories_[i]`` of the category to - be dropped for each feature. None if all the transformed features will - be retained. + - ``drop_idx_[i]`` is the index in ``categories_[i]`` of the category + to be dropped for feature ``i``, or ``None`` if no category is to be + dropped. + - ``drop_idx_ = None`` if all the transformed features will be + retained. + + n_features_in_ : int + Number of features seen during ``fit``. + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during ``fit``. Defined only when `X` + has feature names that are all strings. + + Examples + -------- + Given a dataset with two features, we let the encoder find the unique + values per feature and transform the data to a binary one-hot encoding. + + >>> from sklearn.preprocessing import OneHotEncoder + + One can discard categories not seen during `fit`: + + >>> enc = OneHotEncoder(handle_unknown='ignore') + >>> X = [['Male', 1], ['Female', 3], ['Female', 2]] + >>> enc.fit(X) + OneHotEncoder(handle_unknown='ignore') + >>> enc.categories_ + [array(['Female', 'Male'], dtype=object), array([1, 2, 3], dtype=object)] + >>> enc.transform([['Female', 1], ['Male', 4]]).toarray() + array([[1., 0., 1., 0., 0.], + [0., 1., 0., 0., 0.]]) + >>> enc.inverse_transform([[0, 1, 1, 0, 0], [0, 0, 0, 1, 0]]) + array([['Male', 1], + [None, 2]], dtype=object) + >>> enc.get_feature_names_out(['gender', 'group']) + array(['gender_Female', 'gender_Male', 'group_1', 'group_2', 'group_3'], ...) """ def __init__( @@ -199,290 +244,281 @@ def __init__( sparse_output=True, dtype=np.float32, handle_unknown="error", - verbose=False, output_type=None, + verbose=False, ): - super().__init__(verbose=verbose, output_type=output_type) + super().__init__(output_type=output_type, verbose=verbose) self.categories = categories + self.drop = drop self.sparse_output = sparse_output self.dtype = dtype self.handle_unknown = handle_unknown - self.drop = drop - self.drop_idx_ = None - self._features = None - self._encoders = None - self.input_type = None - # This parameter validation should be performed in `fit` instead - # of in the constructor. Hence the awkwark `if` clause - if sparse_output and np.dtype(dtype) not in ["f", "d", "F", "D"]: - raise ValueError( - "Only float32, float64, complex64 and complex128 " - "are supported when using sparse_output" - ) - def _validate_keywords(self): - if self.handle_unknown not in ("error", "ignore"): - msg = ( - "handle_unknown should be either 'error' or 'ignore', " - "got {0}.".format(self.handle_unknown) - ) - raise ValueError(msg) - # If we have both dropped columns and ignored unknown - # values, there will be ambiguous cells. This creates difficulties - # in interpreting the model. - if self.drop is not None and self.handle_unknown != "error": - raise ValueError( - "`handle_unknown` must be 'error' when the drop parameter is " - "specified, as both would create categories that are all " - "zero." - ) + @classmethod + def _get_param_names(cls): + return [ + "categories", + "drop", + "sparse_output", + "dtype", + "handle_unknown", + *super()._get_param_names(), + ] - def __sklearn_is_fitted__(self): - # TODO: fix state management of this class so `check_is_fitted` works - # without special casing - return getattr(self, "_fitted", False) + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.input_tags.categorical = True + tags.input_tags.allow_nan = True + return tags - def _compute_drop_idx(self): - """Helper to compute indices to drop from category to drop.""" + @mlfunc(set_input_type=True) + @generate_docstring(y=None) + def fit(self, X, y=None) -> "OneHotEncoder": + """Fit OneHotEncoder to X.""" + check_features(self, X, reset=True) + + categories = _compute_categories( + X, categories=self.categories, handle_unknown=self.handle_unknown + ) + + # Compute drop_idx_ if self.drop is None: - return None - elif isinstance(self.drop, str) and self.drop == "first": - return {feature: 0 for feature in self._encoders.keys()} - elif isinstance(self.drop, (dict, list)): - if isinstance(self.drop, list): - self.drop = dict(zip(range(len(self.drop)), self.drop)) - if len(self.drop.keys()) != len(self._encoders): - msg = ( - "`drop` should have as many columns as the number " - "of features ({}), got {}" - ) + drop_idx = None + elif isinstance(self.drop, str): + if self.drop == "first": + drop_idx = np.zeros(len(categories), dtype=object) + else: raise ValueError( - msg.format(len(self._encoders), len(self.drop.keys())) + "Expected `drop` to be 'first' or an array-like, " + f"got {self.drop!r}" ) - drop_idx = dict() - for feature in self.drop.keys(): - self.drop[feature] = cudf.Series(self.drop[feature]) - if len(self.drop[feature]) != 1: - msg = ( - "Trying to drop multiple values for feature {}, " - "this is not supported." - ).format(feature) - raise ValueError(msg) - cats = self._encoders[feature].classes_ - drop_vals = self.drop[feature] - cats = cudf.Series(cats) - # Match the dtype of the drop values to the dtype of the categories - # seen during `fit`. In particular if arrow strings and object dtypes - # are used, then having a mix means `isin` won't work correctly. - if drop_vals.dtype != cats.dtype: - drop_vals = drop_vals.astype(cats.dtype) - if not drop_vals.isin(cats).all(): - msg = ( - "Some categories for feature {} were supposed " - "to be dropped, but were not found in the encoder " - "categories.".format(feature) - ) - raise ValueError(msg) - idx = cats.isin(drop_vals) - drop_idx[feature] = cp.asarray(cats[idx].index) - return drop_idx else: - msg = ( - "Wrong input for parameter `drop`. Expected " - "'first', None or a dict, got {}" - ) - raise ValueError(msg.format(type(self.drop))) + drop = np.asarray(self.drop, dtype=object) - def _has_unknown(self, X_cat, encoder_cat): - """Check if X_cat has categories that are not present in encoder_cat.""" - if X_cat.dtype != encoder_cat.dtype: - encoder_cat = encoder_cat.astype(X_cat.dtype) - return not X_cat.isin(encoder_cat).all() + if len(drop) != len(categories): + raise ValueError( + "`drop` should have length equal to the number of features " + f"({len(categories)}), got {len(drop)}" + ) + missing_drops = [] + drop_indices = [] + for feature, (drop_val, cat) in enumerate(zip(drop, categories)): + if _safe_is_nan(drop_val): + if _safe_is_nan(cat[-1]): + drop_indices.append(cat.size - 1) + else: + missing_drops.append((feature, drop_val)) + else: + idx = np.where(cat == drop_val)[0] + if idx.size: + drop_indices.append(idx.item()) + else: + missing_drops.append((feature, drop_val)) + + if any(missing_drops): + raise ValueError( + "The following categories were supposed to be " + "dropped, but were not found in the training " + "data.\n{}".format( + "\n".join( + [ + "Category: {}, Feature: {}".format(c, v) + for c, v in missing_drops + ] + ) + ) + ) + drop_idx = np.array(drop_indices, dtype=object) - @generate_docstring(y=None) - def fit(self, X, y=None): - """Fit OneHotEncoder to X.""" - self._validate_keywords() - self._fit(X, True) - return self + # Compute n_features_out per input feature + n_features_outs = [len(cats) for cats in categories] + if drop_idx is not None: + for i, idx in enumerate(drop_idx): + if idx is not None: + n_features_outs[i] -= 1 - @generate_docstring( - y=None, - return_values={ - "name": "X_out", - "description": "Transformed input.", - "type": "sparse matrix if sparse_output=True else a 2-d array", - }, - ) - def fit_transform(self, X, y=None): - """ - Fit OneHotEncoder to X, then transform X. Equivalent to fit(X).transform(X). + # Store fitted attributes + self.categories_ = categories + self.drop_idx_ = drop_idx + self._n_features_outs = n_features_outs - """ - X = self._check_input(X) - return self.fit(X).transform(X) + return self + @mlfunc(preserve_index=True) @generate_docstring( return_values={ "name": "X_out", - "description": "Transformed input.", - "type": "sparse matrix if sparse_output=True else a 2-d array", + "description": ( + "Transformed input. A sparse matrix if ``sparse_output=True``, " + "dense otherwise." + ), + "type": "dense_sparse", + "shape": "(n_samples, n_encoded_features)", } ) - @mlfunc(convert_output=False) def transform(self, X): """Transform X using one-hot encoding.""" check_is_fitted(self) check_features(self, X) + X = check_cudf(X, input_name="X") + + raw_inds = cp.zeros(X.shape, dtype="int32") + has_unknown = False + drop_idx = self.drop_idx_ + + for i in range(X.shape[1]): + Xi = X.iloc[:, i] + cats = self.categories_[i] + + if _safe_is_nan(cats[-1]): + # cudf's CategoricalDtype doesn't allow encoding null values, + # we have to handle these manually. + codes = Xi.astype(cudf.CategoricalDtype(cats[:-1])).cat.codes + if Xi.has_nulls: + codes[Xi.isnull()] = len(cats) - 1 + else: + codes = Xi.astype(cudf.CategoricalDtype(cats)).cat.codes + + if codes.has_nulls and self.handle_unknown == "error": + present = Xi.drop_duplicates().sort_values().to_numpy() + diff = _get_diff(present, self.categories_[i]) + raise ValueError( + f"Found unknown categories {diff} in column {i}" + " during transform" + ) - X = self._check_input(X) - - cols, rows = list(), list() - col_idx = None - j = 0 - - try: - for feature in X.columns: - encoder = self._encoders[feature] - with cuml.using_output_type("cudf"): - col_idx = encoder.transform(X[feature]) - idx_to_keep = col_idx.notnull().to_cupy() - col_idx = col_idx.dropna().to_cupy() - - # Simple test to auto upscale col_idx type as needed - # First, determine the maximum value we will add assuming - # monotonically increasing up to len(encoder.classes_) - # Ensure we dont go negative by clamping to 0 - max_value = int(max(len(encoder.classes_) - 1, 0) + j) - min_dtype = np.min_scalar_type(max_value) - col_idx = col_idx.astype(min_dtype, copy=False) - - # increase indices to take previous features into account - col_idx += j - - # Filter out rows with null values - row_idx = cp.arange(len(X))[idx_to_keep] - - if self.drop_idx_ is not None: - drop_idx = self.drop_idx_[feature] + j - mask = cp.ones(col_idx.shape, dtype=bool) - mask[col_idx == drop_idx] = False - col_idx = col_idx[mask] - row_idx = row_idx[mask] - # account for dropped category in indices - col_idx[col_idx > drop_idx] -= 1 - # account for dropped category in current cats number - j -= 1 - - j += len(encoder.classes_) - cols.append(col_idx) - rows.append(row_idx) - - cols = cp.concatenate(cols) - rows = cp.concatenate(rows) - val = cp.ones(rows.shape[0], dtype=self.dtype) - ohe = cupyx.scipy.sparse.coo_matrix( - (val, (rows, cols)), shape=(len(X), j), dtype=self.dtype + if drop_idx is not None and drop_idx[i] is not None: + has_unknown = True + if drop_idx[i] == 0: + codes -= 1 + else: + codes[codes == drop_idx[i]] = -1 + codes[codes > drop_idx[i]] -= 1 + else: + has_unknown |= codes.has_nulls + raw_inds[:, i] = codes.fillna(-1) + + n_samples, n_features = raw_inds.shape + + feature_indices = np.cumsum([0] + self._n_features_outs) + indices = (raw_inds + cp.asarray(feature_indices[:-1])).ravel() + + if has_unknown: + mask = raw_inds != -1 + indices = indices[mask.ravel()] + + indptr = cp.zeros(n_samples + 1, dtype=int) + cp.sum(mask, axis=1, out=indptr[1:], dtype=indptr.dtype) + cp.cumsum(indptr[1:], out=indptr[1:]) + else: + indptr = cp.arange( + 0, n_features * n_samples + 1, n_features, dtype=int ) - if not self.sparse_output: - ohe = ohe.toarray() - - return ohe + data = cp.ones(indptr[-1].item(), dtype=self.dtype) - except TypeError as e: - # Append to cols to include the column that threw the error - cols.append(col_idx) + out = cp_sp.csr_matrix( + (data, indices, indptr), + shape=(n_samples, feature_indices[-1]), + dtype=self.dtype, + ) + if self.sparse_output: + return out + return out.toarray() - # Build a string showing what the types are - input_types_str = ", ".join([str(x.dtype) for x in cols]) - - raise TypeError( - "A TypeError occurred while calculating column " - "category indices, most likely due to integer overflow. This " - "can occur when columns have a large difference in the number " - "of categories, resulting in different category code dtypes " - "for different columns." - "Calculated column code dtypes: {}.\n" - "Internal Error: {}".format(input_types_str, repr(e)) - ) + @mlfunc(preserve_index=True) + @generate_docstring( + y=None, + return_values={ + "name": "X_out", + "description": ( + "Transformed input. A sparse matrix if ``sparse_output=True``, " + "dense otherwise." + ), + "type": "dense_sparse", + "shape": "(n_samples, n_encoded_features)", + }, + ) + def fit_transform(self, X, y=None): + """Fit OneHotEncoder to X, then transform X.""" + X = check_cudf(X, input_name="X") + return self.fit(X).transform(X) - @mlfunc(convert_output=False) + @mlfunc(preserve_index=True) def inverse_transform(self, X): - """Convert the data back to the original representation. In case unknown - categories are encountered (all zeros in the one-hot encoding), ``None`` is used - to represent this category. - - The return type is the same as the type of the input used by the first - call to fit on this estimator instance. + """Convert the data back to the original representation. Parameters ---------- - X : array-like or sparse matrix, shape [n_samples, n_encoded_features] + X : {array-like, sparse matrix} of shape (n_samples, n_encoded_features) The transformed data. Returns ------- - X_tr : cudf.DataFrame or cupy.ndarray + X_original : array of shape (n_samples, n_features) Inverse transformed array. """ check_is_fitted(self) + X = check_array(X, accept_sparse="csr") + + n_features_out = np.sum(self._n_features_outs) + if X.shape[1] != n_features_out: + raise ValueError( + f"Shape of the passed X data is not correct. Expected " + f"{n_features_out} columns, got {X.shape[1]}." + ) - if cupyx.scipy.sparse.issparse(X): - # cupyx.scipy.sparse 7.x does not support argmax, - # when we upgrade cupy to 8.x, we should add a condition in the - # if close: `and not cupyx.scipy.sparse.issparsecsc(X)` - # and change the following line by `X = X.tocsc()` - X = X.toarray() - result = cudf.DataFrame(columns=self._encoders.keys()) j = 0 - for feature in self._encoders.keys(): - feature_enc = self._encoders[feature] - cats = cudf.Series(feature_enc.classes_) - - if self.drop is not None: - # Remove dropped categories - dropped_class_idx = cudf.Series(self.drop_idx_[feature]) - dropped_class_mask = cats.isin(cats[dropped_class_idx]) - if len(cats) == 1: - inv = cudf.Series(Index([cats[0]]).repeat(X.shape[0])) - result[feature] = inv - continue - cats = cats[~dropped_class_mask] - - enc_size = len(cats) - x_feature = X[:, j : j + enc_size] - idx = cp.argmax(x_feature, axis=1) - inv = cudf.Series(cats.iloc[idx]).reset_index(drop=True) - - if self.handle_unknown == "ignore": - not_null_idx = x_feature.any(axis=1) - inv.iloc[~not_null_idx] = None - elif self.drop is not None: - # drop will either be None or handle_unknown will be error. If - # self.drop is not None, then we can safely assume that all of - # the nulls in each column are the dropped value - dropped_mask = cp.asarray(x_feature.sum(axis=1) == 0).flatten() - if dropped_mask.any(): - with cuml.using_output_type("cudf"): - inv[dropped_mask] = feature_enc.inverse_transform( - cudf.Series(self.drop_idx_[feature]) - )[0] - - result[feature] = inv - j += enc_size - if self.input_type == "array": - try: - result = result.to_cupy() - except ValueError: - warnings.warn( - "The input one hot encoding contains rows with " - "unknown categories. Since device arrays do not " - "support null values, the output will be " - "returned as a DataFrame " - "instead." + found_unknown = {} + columns = {} + + for i, (cats, n_cols) in enumerate( + zip(self.categories_, self._n_features_outs) + ): + drop_idx = None if self.drop_idx_ is None else self.drop_idx_[i] + + if len(cats) == 1 and drop_idx is not None: + columns[i] = ( + cudf.Series(cats[drop_idx]) + .repeat(X.shape[0]) + .reset_index(drop=True) ) - return result + else: + if drop_idx is not None: + cats = np.delete(cats, drop_idx) + cats = _cats_to_series(cats) + sub = X[:, j : j + n_cols] + labels = cp.asarray(sub.argmax(axis=1)).ravel() + columns[i] = cats.take(labels).reset_index(drop=True) + + unknown = cp.asarray(sub.sum(axis=1) == 0).ravel() + if unknown.any(): + if drop_idx is not None: + # Treat all zeros as the dropped category + columns[i][unknown] = self.categories_[i][drop_idx] + else: + if self.handle_unknown == "ignore": + # Could be anything, fill with None later + found_unknown[i] = unknown + else: + all_zero_samples = cp.flatnonzero(unknown) + raise ValueError( + f"Samples {all_zero_samples} can not be inverted " + "when drop=None and handle_unknown='error' " + "because they contain all zeros" + ) + + j += n_cols + + out = cudf.DataFrame(columns) + + for idx, mask in found_unknown.items(): + out.loc[mask, idx] = None + + if getattr(self, "feature_names_in_", None) is not None: + out.columns = self.feature_names_in_ + + return out def get_feature_names_out(self, input_features=None): """Get output feature names for transformation. @@ -504,69 +540,87 @@ def get_feature_names_out(self, input_features=None): out = [] for i, (col, cats) in enumerate(zip(input_features, self.categories_)): - # TODO: when `drop_idx_` is actually implemented properly, this can - # be simplified to - # drop_idx = None if self.drop_idx_ is None else self.drop_idx_[i] - drop_idx = cp.asnumpy( - None - if self.drop_idx_ is None - else self.drop_idx_[self._features[i]] - ).item() + drop_idx = None if self.drop_idx_ is None else self.drop_idx_[i] if drop_idx is not None: cats = np.delete(cats, drop_idx) out.extend(f"{col}_{val!s}" for val in cats) return np.array(out, dtype=object) - @classmethod - def _get_param_names(cls): - return super()._get_param_names() + [ - "categories", - "drop", - "sparse_output", - "dtype", - "handle_unknown", - ] +class OrdinalEncoder(Base): + """Encode categorical features as an integer array. -def _slice_feat(X, i): - if hasattr(X, "iloc"): - return X[i] - return X[:, i] + The input to this transformer should be an array-like of integers or + strings, denoting the values taken on by categorical (discrete) features. + The features are converted to ordinal integers. This results in + a single column of integers (0 to n_categories - 1) per feature. + Parameters + ---------- + categories : 'auto' or a list of array-like, default='auto' + Categories (unique values) per feature: -def _get_output( - output_type: Optional[str], - input_type: Optional[str], - out: "cudf.DataFrame", - dtype, -): - if output_type in (None, "input"): - if input_type == "array": - output_type = "cupy" - elif input_type == "df": - output_type = "cudf" + - 'auto' : Determine categories automatically from the training data. + - list : ``categories[i]`` holds the categories expected in the ith + column. - if output_type is None: - output_type = "cupy" + The used categories can be found in the ``categories_`` attribute. - if output_type == "cudf": - return out - elif output_type == "cupy": - return out.astype(dtype).to_cupy(na_value=np.nan) - elif output_type == "numpy": - return cp.asnumpy(out.to_cupy(na_value=np.nan, dtype=dtype)) - elif output_type == "pandas": - import cudf.pandas - - if cudf.pandas.LOADED: - return cudf.pandas.as_proxy_object(out) - return out.to_pandas() - else: - raise ValueError("Unsupported output type.") + dtype : number type, default=np.float64 + Desired dtype of output. + handle_unknown : {'error', 'ignore'}, default='error' + When set to 'error' an error will be raised in case an unknown + categorical feature is present during transform. When set to 'ignore', + the encoded value of unknown categories will be set to NaN. In + :meth:`inverse_transform`, an unknown category will be denoted as None. + + verbose : int or boolean, default=False + Sets logging level. It must be one of `cuml.common.logger.level_*`. + See :ref:`verbosity-levels` for more info. + + output_type : {None, 'input', 'cupy', 'numpy', 'cudf', 'pandas'}, default=None + Return results and set estimator attributes to the indicated output + type. If None, the output type set at the module level + (`cuml.global_settings.output_type`) will be used. See + :ref:`output-data-type-configuration` for more info. + + Attributes + ---------- + categories_ : list of arrays + The categories of each feature determined during ``fit`` (in order of + the features in X and corresponding with the output of ``transform``). + This does not include categories that weren't seen during ``fit``. + + n_features_in_ : int + Number of features seen during ``fit``. + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during ``fit``. Defined only when `X` + has feature names that are all strings. + + Examples + -------- + Given a dataset with two features, we let the encoder find the unique + values per feature and transform the data to an ordinal encoding. + + >>> from sklearn.preprocessing import OrdinalEncoder + >>> enc = OrdinalEncoder() + >>> X = [['Male', 1], ['Female', 3], ['Female', 2]] + >>> enc.fit(X) + OrdinalEncoder() + >>> enc.categories_ + [array(['Female', 'Male'], dtype=object), array([1, 2, 3], dtype=object)] + >>> enc.transform([['Female', 3], ['Male', 1]]) + array([[0., 2.], + [1., 0.]]) + + >>> enc.inverse_transform([[1, 0], [0, 1]]) + array([['Male', 1], + ['Female', 2]], dtype=object) + """ -class OrdinalEncoder(BaseEncoder): def __init__( self, *, @@ -576,115 +630,162 @@ def __init__( verbose=False, output_type=None, ) -> None: - """Encode categorical features as an integer array. - - The input to this transformer should be an :py:class:`cudf.DataFrame` or a - :py:class:`cupy.ndarray`, denoting the unique values taken on by categorical - (discrete) features. The features are converted to ordinal integers. This - results in a single column of integers (0 to n_categories - 1) per feature. - - Parameters - ---------- - categories : 'auto' an cupy.ndarray or a cudf.DataFrame, default='auto' - Categories (unique values) per feature: - - 'auto' : Determine categories automatically from the training data. - - DataFrame/ndarray : ``categories[col]`` holds the categories expected - in the feature col. - handle_unknown : {'error', 'ignore'}, default='error' - Whether to raise an error or ignore if an unknown categorical feature is - present during transform (default is to raise). When this parameter is set - to 'ignore' and an unknown category is encountered during transform, the - resulting encoded value would be null when output type is cudf - dataframe. - verbose : int or boolean, default=False - Sets logging level. It must be one of `cuml.common.logger.level_*`. See - :ref:`verbosity-levels` for more info. - output_type : {None, 'input', 'cupy', 'numpy', 'cudf', 'pandas'}, default=None - Return results and set estimator attributes to the indicated output - type. If None, the output type set at the module level - (`cuml.global_settings.output_type`) will be used. See - :ref:`output-data-type-configuration` for more info. - """ super().__init__(verbose=verbose, output_type=output_type) - self.categories = categories self.dtype = dtype self.handle_unknown = handle_unknown - self.input_type = None + @classmethod + def _get_param_names(cls): + return [ + "categories", + "dtype", + "handle_unknown", + *super()._get_param_names(), + ] + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.input_tags.categorical = True + tags.input_tags.allow_nan = True + return tags + @mlfunc(set_input_type=True) @generate_docstring(y=None) def fit(self, X, y=None) -> "OrdinalEncoder": - """Fit Ordinal to X.""" - self._fit(X, need_drop=False) + """Fit OrdinalEncoder to X.""" + check_features(self, X, reset=True) + + self.categories_ = _compute_categories( + X, categories=self.categories, handle_unknown=self.handle_unknown + ) + + self._missing_indices = { + i: len(cats) - 1 + for i, cats in enumerate(self.categories_) + if _safe_is_nan(cats[-1]) + } + return self + @mlfunc(preserve_index=True) @generate_docstring( return_values={ "name": "X_out", "description": "Transformed input.", - "type": "Type is specified by the `output_type` parameter.", + "type": "dense", + "shape": "(n_samples, n_features)", } ) - @mlfunc(convert_output=False) def transform(self, X): """Transform X using ordinal encoding.""" check_is_fitted(self) check_features(self, X) + X = check_cudf(X, input_name="X") - result = {} - for feature in self._features: - Xi = _slice_feat(X, feature) - with cuml.using_output_type("cudf"): - col_idx = self._encoders[feature].transform(Xi) - result[feature] = col_idx + out = cp.zeros(X.shape, dtype=self.dtype) - r = cudf.DataFrame(result) - return _get_output(self.output_type, self.input_type, r, self.dtype) + for i in range(X.shape[1]): + Xi = X.iloc[:, i] + cats = self.categories_[i] + if _safe_is_nan(cats[-1]): + cats = cats[:-1] + codes = Xi.astype(cudf.CategoricalDtype(cats)).cat.codes + + if ( + self.handle_unknown == "error" + and codes.has_nulls + and (not Xi.has_nulls or codes[Xi.notnull()].has_nulls) + ): + present = ( + Xi.drop_duplicates().dropna().sort_values().to_numpy() + ) + diff = _get_diff(present, self.categories_[i]) + raise ValueError( + f"Found unknown categories {diff} in column {i}" + " during transform" + ) + + if codes.has_nulls: + codes = codes.to_cupy() + out[:, i] = codes + + return out + + @mlfunc(preserve_index=True) @generate_docstring( y=None, return_values={ "name": "X_out", "description": "Transformed input.", - "type": "Type is specified by the `output_type` parameter.", + "type": "dense", + "shape": "(n_samples, n_features)", }, ) def fit_transform(self, X, y=None): - """Fit OrdinalEncoder to X, then transform X. Equivalent to fit(X).transform(X).""" - X = self._check_input(X) + """Fit OrdinalEncoder to X, then transform X.""" + X = check_cudf(X, input_name="X") return self.fit(X).transform(X) - @mlfunc(convert_output=False) + @mlfunc(preserve_index=True) def inverse_transform(self, X): """Convert the data back to the original representation. Parameters ---------- - X : array-like or sparse matrix, shape [n_samples, n_encoded_features] + X : array-like of shape (n_samples, n_encoded_features) The transformed data. Returns ------- - X_tr : Type is specified by the `output_type` parameter. + X_original : ndarray of shape (n_samples, n_features) Inverse transformed array. """ check_is_fitted(self) + X = check_array(X, ensure_all_finite="allow-nan") + + if X.shape[1] != len(self.categories_): + raise ValueError( + f"Shape of the passed X data is not correct. Expected " + f"{len(self.categories_)} columns, got {X.shape[1]}." + ) - result = {} - for feature in self._features: - Xi = _slice_feat(X, feature) - with cuml.using_output_type("cudf"): - inv = self._encoders[feature].inverse_transform(Xi) - result[feature] = inv + columns = {} + found_unknown = {} + + for i, cats in enumerate(self.categories_): + labels = X[:, i] + cats = _cats_to_series(cats) + + if ( + labels.dtype.kind == "f" + and (nan_entries := cp.isnan(labels)).any() + ): + if i in self._missing_indices: + labels = labels.copy() + labels[nan_entries] = self._missing_indices[i] + elif self.handle_unknown == "ignore": + labels = labels.copy() + # Fill with an arbitrary valid label, will be replaced later + labels[nan_entries] = 0 + found_unknown[i] = nan_entries + else: + unknown_indices = cp.flatnonzero(nan_entries) + raise ValueError( + f"Samples {unknown_indices} can not be inverted " + "when handle_unknown='error' because they contain " + "NaN values" + ) - r = cudf.DataFrame(result) - return _get_output(self.output_type, self.input_type, r, self.dtype) + columns[i] = cats.take(labels.astype("int")).reset_index(drop=True) - @classmethod - def _get_param_names(cls): - return super()._get_param_names() + [ - "categories", - "dtype", - "handle_unknown", - ] + out = cudf.DataFrame(columns) + + for idx, mask in found_unknown.items(): + out.loc[mask, idx] = None + + if getattr(self, "feature_names_in_", None) is not None: + out.columns = self.feature_names_in_ + + return out diff --git a/python/cuml/tests/test_one_hot_encoder.py b/python/cuml/tests/test_one_hot_encoder.py index d826a93e4a..9429aaa7ed 100644 --- a/python/cuml/tests/test_one_hot_encoder.py +++ b/python/cuml/tests/test_one_hot_encoder.py @@ -1,383 +1,315 @@ # SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -import math - -import cupy as cp import numpy as np import pandas as pd import pytest -from cudf import DataFrame -from pandas.api.types import is_numeric_dtype -from sklearn.preprocessing import OneHotEncoder as SkOneHotEncoder +import scipy.sparse as sp +import sklearn.preprocessing from cuml.preprocessing import OneHotEncoder -from cuml.testing.utils import ( - assert_inverse_equal, - from_df_to_numpy, - generate_inputs_from_categories, - stress_param, -) - - -def _from_df_to_cupy(df): - """Transform char columns to integer columns, and then create an array""" - for col in df.columns: - if not is_numeric_dtype(df[col].dtype): - if isinstance(df, pd.DataFrame): - df[col] = [c if pd.isna(c) else ord(c) for c in df[col]] - else: - df[col] = [ - c if pd.isna(c) else ord(c) for c in df[col].to_numpy() - ] - return cp.array(from_df_to_numpy(df)) - - -def _convert_drop(drop): - if drop is None or drop == "first": - return drop - return [ord(x) if isinstance(x, str) else x for x in drop.values()] - - -@pytest.mark.parametrize("as_array", [True, False], ids=["cupy", "cudf"]) -def test_onehot_vs_skonehot(as_array): - X = DataFrame({"gender": ["M", "F", "F"], "int": [1, 3, 2]}) - skX = from_df_to_numpy(X) - if as_array: - X = _from_df_to_cupy(X) - skX = cp.asnumpy(X) - - enc = OneHotEncoder(sparse_output=True) - skohe = SkOneHotEncoder(sparse_output=True) - ohe = enc.fit_transform(X) - ref = skohe.fit_transform(skX) - cp.testing.assert_array_equal(ohe.toarray(), ref.toarray()) - - -@pytest.mark.parametrize("drop", [None, "first", {"g": "F", "i": 3}]) -@pytest.mark.parametrize("as_array", [True, False], ids=["cupy", "cudf"]) -def test_onehot_inverse_transform(drop, as_array): - X = DataFrame({"g": ["M", "F", "F"], "i": [1, 3, 2]}) - if as_array: - X = _from_df_to_cupy(X) - drop = _convert_drop(drop) - - enc = OneHotEncoder(drop=drop) - ohe = enc.fit_transform(X) - inv = enc.inverse_transform(ohe) - - assert_inverse_equal(inv, X) - - -@pytest.mark.parametrize("as_array", [True, False], ids=["cupy", "cudf"]) -def test_onehot_categories(as_array): - X = DataFrame({"chars": ["a", "b"], "int": [0, 2]}) - categories = DataFrame({"chars": ["a", "b", "c"], "int": [0, 1, 2]}) - if as_array: - X = _from_df_to_cupy(X) - categories = _from_df_to_cupy(categories).transpose() - - enc = OneHotEncoder(categories=categories, sparse_output=False) - ref = cp.array( - [[1.0, 0.0, 0.0, 1.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0, 0.0, 1.0]] - ) - res = enc.fit_transform(X) - cp.testing.assert_array_equal(res, ref) - - -@pytest.mark.parametrize("as_array", [True, False], ids=["cupy", "cudf"]) -@pytest.mark.filterwarnings( - "ignore:((.|\n)*)unknown((.|\n)*):UserWarning:cuml[.*]" -) -def test_onehot_fit_handle_unknown(as_array): - X = DataFrame({"chars": ["a", "b"], "int": [0, 2]}) - Y = DataFrame({"chars": ["c", "b"], "int": [0, 2]}) - if as_array: - X = _from_df_to_cupy(X) - Y = _from_df_to_cupy(Y) - - enc = OneHotEncoder(handle_unknown="error", categories=Y) - with pytest.raises(KeyError): - enc.fit(X) - - enc = OneHotEncoder(handle_unknown="ignore", categories=Y) - enc.fit(X) - - -@pytest.mark.parametrize("as_array", [True, False], ids=["cupy", "cudf"]) -def test_onehot_transform_handle_unknown(as_array): - X = DataFrame({"chars": ["a", "b"], "int": [0, 2]}) - Y = DataFrame({"chars": ["c", "b"], "int": [0, 2]}) - if as_array: - X = _from_df_to_cupy(X) - Y = _from_df_to_cupy(Y) - - enc = OneHotEncoder(handle_unknown="error", sparse_output=False) - enc = enc.fit(X) - with pytest.raises( - ValueError, match="y contains previously unseen labels" - ): - enc.transform(Y) - - enc = OneHotEncoder(handle_unknown="ignore", sparse_output=False) - enc = enc.fit(X) - ohe = enc.transform(Y) - ref = cp.array([[0.0, 0.0, 1.0, 0.0], [0.0, 1.0, 0.0, 1.0]]) - cp.testing.assert_array_equal(ohe, ref) - - -@pytest.mark.parametrize("as_array", [True, False], ids=["cupy", "cudf"]) -@pytest.mark.filterwarnings( - "ignore:((.|\n)*)unknown((.|\n)*):UserWarning:cuml[.*]" -) -def test_onehot_inverse_transform_handle_unknown(as_array): - X = DataFrame({"chars": ["a", "b"], "int": [0, 2]}) - Y_ohe = cp.array([[0.0, 0.0, 1.0, 0.0], [0.0, 1.0, 0.0, 1.0]]) - ref = DataFrame({"chars": [None, "b"], "int": [0, 2]}) - if as_array: - X = _from_df_to_cupy(X) - ref = _from_df_to_cupy(ref) - - enc = OneHotEncoder(handle_unknown="ignore") - enc = enc.fit(X) - df = enc.inverse_transform(Y_ohe) - assert_inverse_equal(df, ref) - - -@pytest.mark.parametrize("drop", [None, "first"]) -@pytest.mark.parametrize("sparse", [True, False], ids=["sparse", "dense"]) -@pytest.mark.parametrize("n_samples", [10, 1000, 20000, stress_param(250000)]) -@pytest.mark.parametrize("as_array", [True, False], ids=["cupy", "cudf"]) -def test_onehot_random_inputs(drop, sparse, n_samples, as_array): - X, ary = generate_inputs_from_categories( - n_samples=n_samples, as_array=as_array - ) - - enc = OneHotEncoder(sparse_output=sparse, drop=drop, categories="auto") - sk_enc = SkOneHotEncoder( - sparse_output=sparse, drop=drop, categories="auto" - ) - ohe = enc.fit_transform(X) - ref = sk_enc.fit_transform(ary) - if sparse: - cp.testing.assert_array_equal(ohe.toarray(), ref.toarray()) +@pytest.mark.parametrize("kind", ["array", "dataframe"]) +@pytest.mark.parametrize("drop", [None, "first", [2, 2, 1]]) +@pytest.mark.parametrize("dtype", ["float32", "float64"]) +@pytest.mark.parametrize("sparse_output", [True, False]) +def test_onehot_encoder(kind, drop, dtype, sparse_output): + X = np.array( + [ + [2, 2, 2, 2], + [1, 2, 1, 2], + [3, 2, 1, 1], + ] + ).T + if kind == "dataframe": + X = pd.DataFrame(X, columns=["a", "b", "c"]) + + kwargs = { + "drop": drop, + "dtype": dtype, + "sparse_output": sparse_output, + } + sk_enc = sklearn.preprocessing.OneHotEncoder(**kwargs).fit(X) + cu_enc = OneHotEncoder(output_type="numpy", **kwargs).fit(X) + + # Check fitted attributes + assert len(cu_enc.categories_) == len(sk_enc.categories_) + for res, sol in zip(cu_enc.categories_, sk_enc.categories_): + np.testing.assert_array_equal(res, sol) + + if drop is not None: + np.testing.assert_array_equal(cu_enc.drop_idx_, sk_enc.drop_idx_) + + # Check transform + res = cu_enc.transform(X) + Xt = sol = sk_enc.transform(X) + assert res.dtype == sol.dtype + if sparse_output: + np.testing.assert_array_equal(res.toarray(), sol.toarray()) else: - cp.testing.assert_array_equal(ohe, ref) - inv_ohe = enc.inverse_transform(ohe) - assert_inverse_equal(inv_ohe, X) - + np.testing.assert_array_equal(res, sol) -@pytest.mark.parametrize( - "as_array", - [True, False], - ids=["cupy", "cudf"], -) -def test_onehot_drop_idx_first(as_array): - X_ary = [["c", 2, "a"], ["b", 2, "b"]] - X = DataFrame({"chars": ["c", "b"], "int": [2, 2], "letters": ["a", "b"]}) - if as_array: - X = _from_df_to_cupy(X) - X_ary = cp.asnumpy(X) - - enc = OneHotEncoder(sparse_output=False, drop="first", categories="auto") - sk_enc = SkOneHotEncoder( - sparse_output=False, drop="first", categories="auto" - ) - ohe = enc.fit_transform(X) - ref = sk_enc.fit_transform(X_ary) - cp.testing.assert_array_equal(ohe, ref) - inv = enc.inverse_transform(ohe) - assert_inverse_equal(inv, X) + # Check inverse_transform + res = pd.DataFrame(cu_enc.inverse_transform(Xt)) + sol = pd.DataFrame(sk_enc.inverse_transform(Xt)) + pd.testing.assert_frame_equal(res, sol) @pytest.mark.parametrize( - "as_array", - [True, False], - ids=["cupy", "cudf"], + "drop", [None, "first", [True, 2, 2, float("nan"), 2, "banana", "b"]] ) -def test_onehot_drop_one_of_each(as_array): - X = DataFrame({"chars": ["c", "b"], "int": [2, 2], "letters": ["a", "b"]}) - drop = dict({"chars": "b", "int": 2, "letters": "b"}) - X_ary = from_df_to_numpy(X) - drop_ary = ["b", 2, "b"] - if as_array: - X = _from_df_to_cupy(X) - X_ary = cp.asnumpy(X) - drop = drop_ary = _convert_drop(drop) - - enc = OneHotEncoder(sparse_output=False, drop=drop, categories="auto") - ohe = enc.fit_transform(X) - print(ohe.dtype) - ref = SkOneHotEncoder( - sparse_output=False, drop=drop_ary, categories="auto" - ).fit_transform(X_ary) - cp.testing.assert_array_equal(ohe, ref) - inv = enc.inverse_transform(ohe) - assert_inverse_equal(inv, X) +@pytest.mark.parametrize("handle_unknown", ["error", "ignore"]) +def test_onehot_encoder_all_dtypes(drop, handle_unknown): + X = pd.DataFrame( + { + "bool": pd.Series([False, True, False, True, False], dtype="bool"), + "int32": pd.Series([1, 2, 1, 2, 1], dtype="int32"), + "int64": pd.Series([1, 2, 1, 2, 1], dtype="int64"), + "float32": pd.Series([1, 2, float("nan"), 2, 1], dtype="float32"), + "float64": pd.Series([1, 2, float("nan"), 2, 1], dtype="float64"), + "string": pd.Series(["apple", "banana", "carrot", "apple", None]), + "category": pd.Series( + ["a", "b", "a", "b", None], dtype="category" + ), + } + ) + kwargs = {"drop": drop, "handle_unknown": handle_unknown} + cu_enc = OneHotEncoder(**kwargs).fit(X) + sk_enc = sklearn.preprocessing.OneHotEncoder(**kwargs).fit(X) + + # Check fitted attributes + assert len(cu_enc.categories_) == len(sk_enc.categories_) + for res, sol in zip(cu_enc.categories_, sk_enc.categories_): + assert res.dtype == sol.dtype + # XXX: assert_array_equal doesn't compar NaN == NaN, we need to handle + # this case manually. Only need to check last element since NaN should + # always be last. + if res.dtype == "O" and isinstance(res[-1], float): + assert np.isnan(res[-1]) + assert np.isnan(sol[-1]) + res, sol = res[:-1], sol[:-1] + np.testing.assert_array_equal(res, sol) + + if drop is not None: + np.testing.assert_array_equal(cu_enc.drop_idx_, sk_enc.drop_idx_) + + # Check transform + res = cu_enc.transform(X) + Xt = sol = sk_enc.transform(X) + np.testing.assert_array_equal(res.toarray(), sol.toarray()) + + # Check inverse_transform on sparse + res = pd.DataFrame(cu_enc.inverse_transform(Xt)) + sol = pd.DataFrame(sk_enc.inverse_transform(Xt)) + pd.testing.assert_frame_equal(res, sol) + + # Check inverse_transform on dense + res = pd.DataFrame(cu_enc.inverse_transform(Xt.toarray())) + sol = pd.DataFrame(sk_enc.inverse_transform(Xt.toarray())) + pd.testing.assert_frame_equal(res, sol) @pytest.mark.parametrize( - "drop, pattern", + "cardinalities", [ - [dict({"chars": "b"}), "`drop` should have as many columns"], - [ - dict({"chars": "b", "int": [2, 0]}), - "Trying to drop multiple values", - ], - [ - dict({"chars": "b", "int": 3}), - "Some categories [0-9a-zA-Z, ]* were not found", - ], - [ - DataFrame({"chars": ["b"], "int": [3]}), - "Wrong input for parameter `drop`.", - ], + (1, 2), + (2, 1, 1, 2), + (2, 256), + (2, 65536), + (256, 1, 65536), ], ) -@pytest.mark.parametrize("as_array", [True, False], ids=["cupy", "cudf"]) -def test_onehot_drop_exceptions(drop, pattern, as_array): - X = DataFrame({"chars": ["c", "b", "d"], "int": [2, 1, 0]}) - if as_array: - X = _from_df_to_cupy(X) - drop = _convert_drop(drop) if not isinstance(drop, DataFrame) else drop - - with pytest.raises(ValueError, match=pattern): - OneHotEncoder(sparse_output=False, drop=drop).fit(X) - - -@pytest.mark.parametrize("as_array", [True, False], ids=["cupy", "cudf"]) -def test_onehot_get_categories(as_array): - X = DataFrame({"chars": ["c", "b", "d"], "ints": [2, 1, 0]}) - ref = [np.array(["b", "c", "d"]), np.array([0, 1, 2])] - if as_array: - X = _from_df_to_cupy(X) - ref[0] = np.array([ord(x) for x in ref[0]]) - - enc = OneHotEncoder().fit(X) - cats = enc.categories_ - - for i in range(len(ref)): - np.testing.assert_array_equal(ref[i], cats[i]) - - -@pytest.mark.parametrize("as_array", [True, False], ids=["cupy", "cudf"]) -def test_onehot_sparse_drop(as_array): - X = DataFrame({"g": ["M", "F", "F"], "i": [1, 3, 2], "l": [5, 5, 6]}) - drop = {"g": "F", "i": 3, "l": 6} - - ary = from_df_to_numpy(X) - drop_ary = ["F", 3, 6] - if as_array: - X = _from_df_to_cupy(X) - ary = cp.asnumpy(X) - drop = drop_ary = _convert_drop(drop) - - enc = OneHotEncoder(sparse_output=True, drop=drop, categories="auto") - sk_enc = SkOneHotEncoder( - sparse_output=True, drop=drop_ary, categories="auto" +@pytest.mark.parametrize("drop", [None, "first"]) +def test_onehot_encoder_cardinalities(cardinalities, drop): + """A stress test around mixed high and low cardinalities""" + n_samples = max(cardinalities) + X = np.empty(shape=(n_samples, len(cardinalities)), dtype="int32") + col = np.empty(n_samples, dtype="int32") + rng = np.random.default_rng(42) + for i, n_cats in enumerate(cardinalities): + # Pre-fill first n_cats to ensure 1 of each category present + col[:n_cats] = np.arange(n_cats) + col[n_cats:] = rng.choice(n_cats, n_samples - n_cats) + rng.shuffle(col) + X[:, i] = col + + cu_enc = OneHotEncoder(drop=drop).fit(X) + sk_enc = sklearn.preprocessing.OneHotEncoder(drop=drop).fit(X) + + # Check fitted attributes + assert len(cu_enc.categories_) == len(sk_enc.categories_) + for res, sol in zip(cu_enc.categories_, sk_enc.categories_): + np.testing.assert_array_equal(res, sol) + + if drop is not None: + np.testing.assert_array_equal(cu_enc.drop_idx_, sk_enc.drop_idx_) + + # Check transform + res = cu_enc.transform(X) + Xt = sol = sk_enc.transform(X) + # efficient equality check for sparse data + assert (res != sol).count_nonzero() == 0 + + # Check inverse_transform + res = pd.DataFrame(cu_enc.inverse_transform(Xt)) + pd.testing.assert_frame_equal(res, pd.DataFrame(X)) + + +def test_onehot_encoder_invalid_parameters(): + X = pd.DataFrame( + { + "x": [1.0, 2.0, 1.0, 2.0], + "y": [1.0, 2.0, 3.0, 1.0], + "z": [2.0, 2.0, float("nan"), 2.0], + } ) - ohe = enc.fit_transform(X) - ref = sk_enc.fit_transform(ary) - cp.testing.assert_array_equal(ohe.toarray(), ref.toarray()) + # Invalid `handle_unknown` errors + with pytest.raises( + ValueError, match="Expected `handle_unknown` .* got 'bad'" + ): + OneHotEncoder(handle_unknown="bad").fit(X) -@pytest.mark.parametrize("as_array", [True, False], ids=["cupy", "cudf"]) -def test_onehot_categories_shape_mismatch(as_array): - X = DataFrame({"chars": ["a"], "int": [0]}) - categories = DataFrame({"chars": ["a", "b", "c"]}) - if as_array: - X = _from_df_to_cupy(X) - categories = _from_df_to_cupy(categories).transpose() + # Invalid `drop` errors + with pytest.raises(ValueError, match="Expected `drop` .* got 'bad'"): + OneHotEncoder(drop="bad").fit(X) - with pytest.raises(ValueError): - OneHotEncoder(categories=categories, sparse_output=False).fit(X) + with pytest.raises( + ValueError, match="`drop` should have length .* \\(3\\), got 2" + ): + OneHotEncoder(drop=[2, 2]).fit(X) + with pytest.raises(ValueError, match="The following categories") as rec: + OneHotEncoder(drop=[10, 1, 9]).fit(X) + assert "Category: 0, Feature: 10" in str(rec.value) + assert "Category: 2, Feature: 9" in str(rec.value) -def test_onehot_category_specific_cases(): - # See this for reasoning: https://github.com/rapidsai/cuml/issues/2690 + # Invalid `categories` errors + with pytest.raises(ValueError, match="Expected `categories` .* got 'bad'"): + OneHotEncoder(categories="bad").fit(X) - # All of these cases use sparse_output=False, where - # test_onehot_category_class_count uses sparse_output=True + with pytest.raises(ValueError, match="Shape mismatch"): + OneHotEncoder(categories=[[2], [1, 2]]).fit(X) - # ==== 2 Rows (Low before High) ==== - example_df = DataFrame() - example_df["low_cardinality_column"] = ["A"] * 200 + ["B"] * 56 - example_df["high_cardinality_column"] = cp.linspace(0, 255, 256) + with pytest.raises(ValueError, match="Nan should be the last element"): + OneHotEncoder(categories=[[1, 2], [1, 2, 3], [float("nan"), 2]]).fit(X) - encoder = OneHotEncoder(handle_unknown="ignore", sparse_output=False) - encoder.fit_transform(example_df) + with pytest.raises(ValueError, match="In column 1, .* duplicate elements"): + OneHotEncoder( + categories=[[1, 2], [1, 2, 3, 3], [2, float("nan")]] + ).fit(X) - # ==== 2 Rows (High before Low, used to fail) ==== - example_df = DataFrame() - example_df["high_cardinality_column"] = cp.linspace(0, 255, 256) - example_df["low_cardinality_column"] = ["A"] * 200 + ["B"] * 56 - encoder = OneHotEncoder(handle_unknown="ignore", sparse_output=False) - encoder.fit_transform(example_df) +def test_onehot_encoder_unknown_categories_in_fit(): + X = np.array([[1, 2, float("nan"), 2]]).T + with pytest.raises(ValueError, match="Found unknown categories \\[nan\\]"): + OneHotEncoder(categories=[[1, 2]]).fit(X) -@pytest.mark.parametrize( - "total_classes", - [np.iinfo(np.uint8).max, np.iinfo(np.uint16).max], - ids=["uint8", "uint16"], -) -def test_onehot_category_class_count(total_classes: int): - # See this for reasoning: https://github.com/rapidsai/cuml/issues/2690 - # All tests use sparse_output=True to avoid memory errors + with pytest.raises( + ValueError, match="Found unknown categories \\[1.0, 2.0\\]" + ): + OneHotEncoder(categories=[[float("nan")]]).fit(X) - encoder = OneHotEncoder(handle_unknown="ignore", sparse_output=True) + enc = OneHotEncoder(categories=[[1, 2, float("nan")]]).fit(X) + np.testing.assert_array_equal(enc.categories_[0], [1, 2, float("nan")]) - # ==== 2 Rows ==== - example_df = DataFrame() - example_df["high_cardinality_column"] = cp.linspace( - 0, total_classes - 1, total_classes - ) - example_df["low_cardinality_column"] = ["A"] * 200 + ["B"] * ( - total_classes - 200 - ) - assert encoder.fit_transform(example_df).shape[1] == total_classes + 2 +@pytest.mark.parametrize("unknown_val", ["c", float("nan")]) +def test_onehot_encoder_transform_unknown(unknown_val): + X1 = pd.DataFrame({"x": ["a", "b", "a"]}) + X2 = pd.DataFrame({"x": ["b", unknown_val]}) - # ==== 3 Rows ==== - example_df = DataFrame() - example_df["high_cardinality_column"] = cp.linspace( - 0, total_classes - 1, total_classes - ) - example_df["low_cardinality_column"] = ["A"] * total_classes - example_df["med_cardinality_column"] = ["B"] * total_classes + enc = OneHotEncoder().fit(X1) - assert encoder.fit_transform(example_df).shape[1] == total_classes + 2 + # Unknown value errors by default + with pytest.raises( + ValueError, + match=f".* categories \\[{unknown_val!r}\\] in column 0 during transform", + ): + enc.transform(X2) + + # Passing `handle_unknown="ignore"` fixes things + kwargs = {"handle_unknown": "ignore"} + cu_enc = OneHotEncoder(**kwargs).fit(X1) + sk_enc = sklearn.preprocessing.OneHotEncoder(**kwargs).fit(X1) + res = cu_enc.transform(X2) + sol = sk_enc.transform(X2) + np.testing.assert_array_equal(res.toarray(), sol.toarray()) + + # Explicitly passing categories also fixes things + kwargs = {"categories": [["a", "b", unknown_val]]} + cu_enc = OneHotEncoder(**kwargs).fit(X1) + sk_enc = sklearn.preprocessing.OneHotEncoder(**kwargs).fit(X1) + res = cu_enc.transform(X2) + sol = sk_enc.transform(X2) + np.testing.assert_array_equal(res.toarray(), sol.toarray()) + + +@pytest.mark.parametrize("drop", [None, "first", ["b", 3, 1]]) +@pytest.mark.parametrize("handle_unknown", ["error", "ignore"]) +@pytest.mark.parametrize("sparse", [False, True]) +@pytest.mark.parametrize("unknown", [False, True]) +def test_onehot_encoder_inverse_transform( + drop, handle_unknown, sparse, unknown +): + X = pd.DataFrame({"x": ["a", "b", "b"], "y": [1, 3, 2], "z": [1, 1, 1]}) + + kwargs = {"handle_unknown": handle_unknown, "drop": drop} + cu_enc = OneHotEncoder(**kwargs).fit(X) + sk_enc = sklearn.preprocessing.OneHotEncoder(**kwargs).fit(X) + + if drop is None: + Xt = np.array( + [ + [0, 1, 1, 0, 0, 1], + [1, 0, 0, 0, 1, 1], + [1, 0, 0, 1, 0, 1], + ] + ) + else: + Xt = np.array( + [ + [0, 0, 0], + [1, 0, 1], + [1, 1, 0], + ] + ) - # ==== N Rows (Even Split) ==== - num_rows = [3, 10, 100] + if unknown: + Xt[1, 0] = 0 - for row_count in num_rows: - class_per_row = int(math.ceil(total_classes / float(row_count))) + 1 - example_df = DataFrame() + if sparse: + Xt = sp.csr_matrix(Xt) - for row_idx in range(row_count): - example_df[str(row_idx)] = cp.linspace( - row_idx * class_per_row, - ((row_idx + 1) * class_per_row) - 1, - class_per_row, - ) + if handle_unknown == "error" and unknown and drop is None: + with pytest.raises(ValueError, match="Samples .* can not be inverted"): + cu_enc.inverse_transform(Xt) + else: + res = pd.DataFrame(cu_enc.inverse_transform(Xt)) + sol = pd.DataFrame(sk_enc.inverse_transform(Xt)) + pd.testing.assert_frame_equal(res, sol) - assert ( - encoder.fit_transform(example_df).shape[1] - == class_per_row * row_count - ) + +@pytest.mark.parametrize("drop", [None, "first"]) +def test_onehot_encoder_inverse_transform_errors(drop): + X = np.array([[1, 2, 1], [3, 1, 2]]).T + + enc = OneHotEncoder(drop=drop) + Xt = enc.fit_transform(X) + with pytest.raises(ValueError, match="Shape of the passed X data"): + enc.inverse_transform(Xt[:, :-1]) @pytest.mark.parametrize("named", [True, False]) @pytest.mark.parametrize("drop", [None, "first"]) -def test_onehot_get_feature_names_out(named, drop): +def test_onehot_encoder_get_feature_names_out(named, drop): columns = [["apple", "banana", "strawberry"], [0, 1, 2]] names = ["fruits", "sizes"] if named else [0, 1] X = pd.DataFrame(dict(zip(names, columns))) cu_model = OneHotEncoder(drop=drop).fit(X) - sk_model = SkOneHotEncoder(drop=drop).fit(X) + sk_model = sklearn.preprocessing.OneHotEncoder(drop=drop).fit(X) res = cu_model.get_feature_names_out() sol = sk_model.get_feature_names_out() @@ -389,7 +321,7 @@ def test_onehot_get_feature_names_out(named, drop): assert np.array_equal(res, sol) -def test_onehot_get_feature_names_deprecated(): +def test_onehot_encoder_get_feature_names_deprecated(): X = pd.DataFrame( {"fruits": ["apple", "banana", "strawberry"], "sizes": [0, 1, 2]} ) diff --git a/python/cuml/tests/test_ordinal_encoder.py b/python/cuml/tests/test_ordinal_encoder.py index 0a144637fd..2fdd00bbf4 100644 --- a/python/cuml/tests/test_ordinal_encoder.py +++ b/python/cuml/tests/test_ordinal_encoder.py @@ -1,123 +1,255 @@ -# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -import cupy as cp import numpy as np import pandas as pd import pytest -from cudf import DataFrame -from cudf.testing import assert_frame_equal -from sklearn.preprocessing import OrdinalEncoder as skOrdinalEncoder +import sklearn.preprocessing from cuml.preprocessing import OrdinalEncoder -@pytest.fixture -def test_sample(): - X = DataFrame({"cat": ["M", "F", "F"], "num": [1, 3, 2]}) - return X - - -def test_ordinal_encoder_df(test_sample) -> None: - X = test_sample - enc = OrdinalEncoder() - enc.fit(X) - Xt = enc.transform(X) - - X_1 = DataFrame({"cat": ["F", "F"], "num": [1, 2]}) - Xt_1 = enc.transform(X_1) - - assert Xt_1.iloc[0, 0] == Xt.iloc[1, 0] - assert Xt_1.iloc[1, 0] == Xt.iloc[1, 0] - assert Xt_1.iloc[0, 1] == Xt.iloc[0, 1] - assert Xt_1.iloc[1, 1] == Xt.iloc[2, 1] +@pytest.mark.parametrize("kind", ["array", "dataframe"]) +@pytest.mark.parametrize("dtype", ["float32", "float64"]) +def test_ordinal_encoder(kind, dtype): + X = np.array( + [ + [2, 2, 2, 2], + [1, 2, 1, 2], + [3, 2, 1, 1], + ] + ).T + if kind == "dataframe": + X = pd.DataFrame(X, columns=["a", "b", "c"]) + + sk_enc = sklearn.preprocessing.OrdinalEncoder(dtype=dtype).fit(X) + cu_enc = OrdinalEncoder(output_type="numpy", dtype=dtype).fit(X) + + # Check fitted attributes + assert len(cu_enc.categories_) == len(sk_enc.categories_) + for res, sol in zip(cu_enc.categories_, sk_enc.categories_): + np.testing.assert_array_equal(res, sol) + + # Check transform + res = cu_enc.transform(X) + Xt = sol = sk_enc.transform(X) + assert res.dtype == sol.dtype + np.testing.assert_array_equal(res, sol) + + # Check inverse_transform + res = pd.DataFrame(cu_enc.inverse_transform(Xt)) + sol = pd.DataFrame(sk_enc.inverse_transform(Xt)) + pd.testing.assert_frame_equal(res, sol) + + +@pytest.mark.parametrize("handle_unknown", ["error", "ignore"]) +def test_ordinal_encoder_all_dtypes(handle_unknown): + X = pd.DataFrame( + { + "bool": pd.Series([False, True, False, True, False], dtype="bool"), + "int32": pd.Series([1, 2, 1, 2, 1], dtype="int32"), + "int64": pd.Series([1, 2, 1, 2, 1], dtype="int64"), + "float32": pd.Series([1, 2, float("nan"), 2, 1], dtype="float32"), + "float64": pd.Series([1, 2, float("nan"), 2, 1], dtype="float64"), + "string": pd.Series(["apple", "banana", "carrot", "apple", None]), + "category": pd.Series( + ["a", "b", "a", "b", None], dtype="category" + ), + } + ) + cu_enc = OrdinalEncoder(output_type="numpy", handle_unknown=handle_unknown) + if handle_unknown == "ignore": + sk_enc = sklearn.preprocessing.OrdinalEncoder() + else: + sk_enc = sklearn.preprocessing.OrdinalEncoder( + handle_unknown="use_encoded_value", + unknown_value=np.nan, + ) + cu_enc.fit(X) + sk_enc.fit(X) + + # Check fitted attributes + assert len(cu_enc.categories_) == len(sk_enc.categories_) + for res, sol in zip(cu_enc.categories_, sk_enc.categories_): + assert res.dtype == sol.dtype + pd.testing.assert_series_equal(pd.Series(res), pd.Series(sol)) + + # Check transform + res = cu_enc.transform(X) + Xt = sol = sk_enc.transform(X) + np.testing.assert_array_equal(res, sol) + + # Check inverse_transform + res = pd.DataFrame(cu_enc.inverse_transform(Xt)) + sol = pd.DataFrame(sk_enc.inverse_transform(Xt)) + pd.testing.assert_frame_equal(res, sol) + + +@pytest.mark.parametrize( + "cardinalities", + [ + (1, 2), + (2, 1, 1, 2), + (2, 256), + (2, 65536), + (256, 1, 65536), + ], +) +def test_ordinal_encoder_cardinalities(cardinalities): + """A stress test around mixed high and low cardinalities""" + n_samples = max(cardinalities) + X = np.empty(shape=(n_samples, len(cardinalities)), dtype="int32") + col = np.empty(n_samples, dtype="int32") + rng = np.random.default_rng(42) + for i, n_cats in enumerate(cardinalities): + # Pre-fill first n_cats to ensure 1 of each category present + col[:n_cats] = np.arange(n_cats) + col[n_cats:] = rng.choice(n_cats, n_samples - n_cats) + rng.shuffle(col) + X[:, i] = col + + cu_enc = OrdinalEncoder().fit(X) + sk_enc = sklearn.preprocessing.OrdinalEncoder().fit(X) + + # Check fitted attributes + assert len(cu_enc.categories_) == len(sk_enc.categories_) + for res, sol in zip(cu_enc.categories_, sk_enc.categories_): + np.testing.assert_array_equal(res, sol) + + # Check transform + res = cu_enc.transform(X) + Xt = sol = sk_enc.transform(X) + np.testing.assert_array_equal(res, sol) + + # Check inverse_transform + res = pd.DataFrame(cu_enc.inverse_transform(Xt)) + sol = pd.DataFrame(sk_enc.inverse_transform(Xt)) + pd.testing.assert_frame_equal(res, sol) + + +def test_ordinal_encoder_invalid_parameters(): + X = pd.DataFrame( + { + "x": [1.0, 2.0, 1.0, 2.0], + "y": [1.0, 2.0, 3.0, 1.0], + "z": [2.0, 2.0, float("nan"), 2.0], + } + ) + + # Invalid `handle_unknown` errors + with pytest.raises( + ValueError, match="Expected `handle_unknown` .* got 'bad'" + ): + OrdinalEncoder(handle_unknown="bad").fit(X) - inv_Xt = enc.inverse_transform(Xt) - inv_Xt_1 = enc.inverse_transform(Xt_1) + # Invalid `categories` errors + with pytest.raises(ValueError, match="Expected `categories` .* got 'bad'"): + OrdinalEncoder(categories="bad").fit(X) - assert_frame_equal(inv_Xt, X, check_dtype=False) - assert_frame_equal(inv_Xt_1, X_1, check_dtype=False) + with pytest.raises(ValueError, match="Shape mismatch"): + OrdinalEncoder(categories=[[2], [1, 2]]).fit(X) - assert enc.n_features_in_ == 2 + with pytest.raises(ValueError, match="Nan should be the last element"): + OrdinalEncoder(categories=[[1, 2], [1, 2, 3], [float("nan"), 2]]).fit( + X + ) + with pytest.raises(ValueError, match="In column 1, .* duplicate elements"): + OrdinalEncoder( + categories=[[1, 2], [1, 2, 3, 3], [2, float("nan")]] + ).fit(X) -def test_ordinal_encoder_array() -> None: - X = DataFrame({"A": [4, 1, 1], "B": [1, 3, 2]}).values - enc = OrdinalEncoder() - enc.fit(X) - Xt = enc.transform(X) - X_1 = DataFrame({"A": [1, 1], "B": [1, 2]}).values - Xt_1 = enc.transform(X_1) +def test_ordinal_encoder_unknown_categories_in_fit(): + X = np.array([[1, 2, float("nan"), 2]]).T - assert Xt_1[0, 0] == Xt[1, 0] - assert Xt_1[1, 0] == Xt[1, 0] - assert Xt_1[0, 1] == Xt[0, 1] - assert Xt_1[1, 1] == Xt[2, 1] + with pytest.raises(ValueError, match="Found unknown categories \\[nan\\]"): + OrdinalEncoder(categories=[[1, 2]]).fit(X) - inv_Xt = enc.inverse_transform(Xt) - inv_Xt_1 = enc.inverse_transform(Xt_1) + with pytest.raises( + ValueError, match="Found unknown categories \\[1.0, 2.0\\]" + ): + OrdinalEncoder(categories=[[float("nan")]]).fit(X) - cp.testing.assert_allclose(X, inv_Xt) - cp.testing.assert_allclose(X_1, inv_Xt_1) + enc = OrdinalEncoder(categories=[[1, 2, float("nan")]]).fit(X) + np.testing.assert_array_equal(enc.categories_[0], [1, 2, float("nan")]) - assert enc.n_features_in_ == 2 +def test_ordinal_encoder_transform_missing(): + X1 = pd.DataFrame({"x": [np.nan, "b", "a"], "y": [1, 2, np.nan]}) + X2 = pd.DataFrame({"x": ["b", np.nan], "y": [np.nan, 1]}) -def test_ordinal_array() -> None: - X = cp.arange(32).reshape(16, 2) + cu_enc = OrdinalEncoder().fit(X1) + sk_enc = sklearn.preprocessing.OrdinalEncoder().fit(X1) - enc = OrdinalEncoder() - enc.fit(X) - Xt = enc.transform(X) + res = cu_enc.transform(X2) + sol = sk_enc.transform(X2) + np.testing.assert_array_equal(res.to_numpy(), sol) - Xh = cp.asnumpy(X) - skenc = skOrdinalEncoder() - skenc.fit(Xh) - Xt_sk = skenc.transform(Xh) - cp.testing.assert_allclose(Xt, Xt_sk) +def test_ordinal_encoder_transform_unknown(): + X1 = pd.DataFrame({"x": ["a", "b", "a"]}) + X2 = pd.DataFrame({"x": ["b", "c"]}) + enc = OrdinalEncoder().fit(X1) -def test_output_type(test_sample) -> None: - X = test_sample - enc = OrdinalEncoder(output_type="cupy").fit(X) - assert isinstance(enc.transform(X), cp.ndarray) - enc = OrdinalEncoder(output_type="cudf").fit(X) - assert isinstance(enc.transform(X), DataFrame) - enc = OrdinalEncoder(output_type="pandas").fit(X) - assert isinstance(enc.transform(X), pd.DataFrame) - enc = OrdinalEncoder(output_type="numpy").fit(X) - assert isinstance(enc.transform(X), np.ndarray) - # output_type == "input" + # Unknown value errors by default + with pytest.raises( + ValueError, + match=".* categories \\['c'\\] in column 0 during transform", + ): + enc.transform(X2) + + # Passing `handle_unknown="ignore"` fixes things + cu_enc = OrdinalEncoder( + output_type="numpy", + handle_unknown="ignore", + ).fit(X1) + sk_enc = sklearn.preprocessing.OrdinalEncoder( + handle_unknown="use_encoded_value", + unknown_value=np.nan, + ).fit(X1) + res = cu_enc.transform(X2) + sol = sk_enc.transform(X2) + np.testing.assert_array_equal(res, sol) + + # Explicitly passing categories also fixes things + kwargs = {"categories": [["a", "b", "c"]]} + cu_enc = OrdinalEncoder(output_type="numpy", **kwargs).fit(X1) + sk_enc = sklearn.preprocessing.OrdinalEncoder(**kwargs).fit(X1) + res = cu_enc.transform(X2) + sol = sk_enc.transform(X2) + np.testing.assert_array_equal(res, sol) + + +def test_ordinal_encoder_inverse_transform(): + Xt = pd.DataFrame({"x": [np.nan, 0], "y": [0, 1], "z": [0, np.nan]}) + + # No unknown elements, fully invertible + X = pd.DataFrame( + {"x": ["a", "b", None], "y": [1, 3, 2], "z": [1, 1, np.nan]} + ) + enc = OrdinalEncoder().fit(X) + res = enc.inverse_transform(Xt) + sol = pd.DataFrame({"x": [np.nan, "a"], "y": [1, 2], "z": [1, np.nan]}) + pd.testing.assert_frame_equal(res, sol) + + # Incorrect input dimensions errors + with pytest.raises(ValueError, match="Shape of the passed X data"): + enc.inverse_transform(Xt[["x", "y"]]) + + # handle_unknown="ignore", fully invertible + X = pd.DataFrame( + {"x": ["a", "b", "b"], "y": [1, 3, 2], "z": [1, 1, np.nan]} + ) + enc = OrdinalEncoder(handle_unknown="ignore").fit(X) + res = enc.inverse_transform(Xt) + sol = pd.DataFrame({"x": [np.nan, "a"], "y": [1, 2], "z": [1, np.nan]}) + pd.testing.assert_frame_equal(res, sol) + + # handle_unknown="error", errors on unknown values + X = pd.DataFrame({"x": ["a", "b", "b"], "y": [1, 3, 2], "z": [1, 1, 1]}) enc = OrdinalEncoder().fit(X) - assert isinstance(enc.transform(X), DataFrame) - - -def test_feature_names(test_sample) -> None: - enc = OrdinalEncoder().fit(test_sample) - assert (enc.feature_names_in_ == ["cat", "num"]).all() - - -@pytest.mark.parametrize("as_array", [True, False], ids=["cupy", "cudf"]) -def test_handle_unknown(as_array: bool) -> None: - X = DataFrame({"data": [0, 1]}) - Y = DataFrame({"data": [3, 1]}) - - if as_array: - X = X.values - Y = Y.values - - enc = OrdinalEncoder(handle_unknown="error") - enc = enc.fit(X) with pytest.raises( - ValueError, match="y contains previously unseen labels" + ValueError, match="Samples \\[0\\] can not be inverted" ): - enc.transform(Y) - - enc = OrdinalEncoder(handle_unknown="ignore") - enc = enc.fit(X) - encoded = enc.transform(Y) - if as_array: - np.isnan(encoded[0, 0]) - else: - assert pd.isna(encoded.iloc[0, 0]) + enc.inverse_transform(Xt) From a091d4de938440e328a054b03f5cfdd7dd6894bb Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Tue, 18 Aug 2026 16:47:13 -0500 Subject: [PATCH 02/14] Add `OneHotEncoder` and `OrdinalEncoder` to sklearn compat tests --- python/cuml/tests/test_sklearn_compatibility.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/cuml/tests/test_sklearn_compatibility.py b/python/cuml/tests/test_sklearn_compatibility.py index 6d78dd0059..2ea6829315 100644 --- a/python/cuml/tests/test_sklearn_compatibility.py +++ b/python/cuml/tests/test_sklearn_compatibility.py @@ -120,6 +120,8 @@ SpectralClustering(), LogisticRegression(), StandardScaler(), + OneHotEncoder(), + OrdinalEncoder(), ] @@ -164,8 +166,6 @@ def _all_cuml_estimators(): LabelEncoder: "Not yet tested for sklearn compat", TargetEncoder: "Not yet tested for sklearn compat", LabelBinarizer: "Not yet tested for sklearn compat", - OneHotEncoder: "Not yet tested for sklearn compat", - OrdinalEncoder: "Not yet tested for sklearn compat", # Preprocessing (vendored sklearn) MinMaxScaler: "Vendored sklearn preprocessing, not yet tested", MaxAbsScaler: "Vendored sklearn preprocessing, not yet tested", From 33f4134568b128246447d984171fd7fa3535577d Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Tue, 18 Aug 2026 20:48:27 -0500 Subject: [PATCH 03/14] Split out unique logic, preparing for dask implementations --- python/cuml/cuml/preprocessing/encoders.py | 60 +++++++++++++++++----- 1 file changed, 48 insertions(+), 12 deletions(-) diff --git a/python/cuml/cuml/preprocessing/encoders.py b/python/cuml/cuml/preprocessing/encoders.py index 2835be41f1..1c989d97f9 100644 --- a/python/cuml/cuml/preprocessing/encoders.py +++ b/python/cuml/cuml/preprocessing/encoders.py @@ -49,10 +49,30 @@ def _cats_to_series(cats): return cudf.Series(cats) -def _compute_categories(X, categories="auto", handle_unknown="error"): - """Compute `categories_` for an encoder.""" - X = check_cudf(X, input_name="X") - n_features = X.shape[1] +def _compute_categories( + X_list, unique=False, categories="auto", handle_unknown="error" +): + """Compute `categories_` for an encoder. + + Parameters + ---------- + X_list : list[cudf.Series] + A list of columns in the input X. + unique : bool, default=False + Whether the columns already are reduced to only their unique entries. + categories : 'auto' or list[array-like], default='auto' + Explicitly provided categories per-column, or 'auto' to automatically + infer the categories from the input data. + handle_unknown : {'error', 'ignore'}, default='error' + If 'error', entries found in X that don't exist in explicitly provided + categories will lead to an error. If 'ignore' no error will be raised. + + Returns + ------- + categories_ : list[numpy.ndarray] + A list of the categories determined per-column. + """ + n_features = len(X_list) if handle_unknown not in ("ignore", "error"): raise ValueError( @@ -76,11 +96,11 @@ def _compute_categories(X, categories="auto", handle_unknown="error"): out = [] - for i in range(n_features): - Xi = X.iloc[:, i] - + for i, Xi in enumerate(X_list): if auto: - cats = Xi.drop_duplicates().sort_values().to_numpy() + if not unique: + Xi = Xi.drop_duplicates() + cats = Xi.sort_values().to_numpy() else: dtype = Xi.dtype if isinstance(Xi.dtype, np.dtype) else "O" cats = categories[i] @@ -113,7 +133,9 @@ def _compute_categories(X, categories="auto", handle_unknown="error"): ) if handle_unknown == "error": - present = Xi.drop_duplicates().sort_values().to_numpy() + if not unique: + Xi = Xi.drop_duplicates() + present = Xi.sort_values().to_numpy() diff = _get_diff(present, cats) if diff: raise ValueError( @@ -276,9 +298,16 @@ def __sklearn_tags__(self): def fit(self, X, y=None) -> "OneHotEncoder": """Fit OneHotEncoder to X.""" check_features(self, X, reset=True) + X = check_cudf(X, input_name="X") + X_list = [X.iloc[:, i] for i in range(X.shape[1])] + return self._fit(X_list) + def _fit(self, X_list, unique=False): categories = _compute_categories( - X, categories=self.categories, handle_unknown=self.handle_unknown + X_list, + unique=unique, + categories=self.categories, + handle_unknown=self.handle_unknown, ) # Compute drop_idx_ @@ -400,7 +429,7 @@ def transform(self, X): n_samples, n_features = raw_inds.shape - feature_indices = np.cumsum([0] + self._n_features_outs) + feature_indices = np.cumsum([0, *self._n_features_outs]) indices = (raw_inds + cp.asarray(feature_indices[:-1])).ravel() if has_unknown: @@ -655,9 +684,16 @@ def __sklearn_tags__(self): def fit(self, X, y=None) -> "OrdinalEncoder": """Fit OrdinalEncoder to X.""" check_features(self, X, reset=True) + X = check_cudf(X, input_name="X") + X_list = [X.iloc[:, i] for i in range(X.shape[1])] + return self._fit(X_list) + def _fit(self, X_list, unique=False): self.categories_ = _compute_categories( - X, categories=self.categories, handle_unknown=self.handle_unknown + X_list, + unique=unique, + categories=self.categories, + handle_unknown=self.handle_unknown, ) self._missing_indices = { From 262b13054c1b80e509109b249cabc2a98c5de7b9 Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Tue, 18 Aug 2026 20:49:03 -0500 Subject: [PATCH 04/14] Update dask implementations - Removes `OneHotEncoderMG` and `OrdinalEncoderMG`. These `*MG` implementations are no longer needed. They also didn't match our previous `*MG` conventions in that they required `dask` to work, rather than being agnostic to the distributed framework used. Since these classes weren't public (and there are no known downstream consumers) it's fine to remove them completely. - Fixes the `sparse_output` parameter support for `cuml.dask.preprocessing.OneHotEncoder` to work the same as it does in the `cuml.preprocessing.OneHotEncoder` implementation. - Improves the test coverage of both `OneHotEncoder` and `OrdinalEncoder`. --- .../cuml/cuml/dask/preprocessing/encoders.py | 151 ++++++----- .../cuml/preprocessing/onehotencoder_mg.py | 48 ---- .../cuml/preprocessing/ordinalencoder_mg.py | 38 --- .../tests/dask/test_dask_one_hot_encoder.py | 240 +++++------------- .../tests/dask/test_dask_ordinal_encoder.py | 141 +++++----- 5 files changed, 216 insertions(+), 402 deletions(-) delete mode 100644 python/cuml/cuml/preprocessing/onehotencoder_mg.py delete mode 100644 python/cuml/cuml/preprocessing/ordinalencoder_mg.py diff --git a/python/cuml/cuml/dask/preprocessing/encoders.py b/python/cuml/cuml/dask/preprocessing/encoders.py index 8130232099..847e77fcad 100644 --- a/python/cuml/cuml/dask/preprocessing/encoders.py +++ b/python/cuml/cuml/dask/preprocessing/encoders.py @@ -1,17 +1,14 @@ -# SPDX-FileCopyrightText: Copyright (c) 2020-2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # -from collections.abc import Sequence - -from dask_cudf import DataFrame as dcDataFrame -from dask_cudf import Series as dcSeries -from toolz import first +import dask_cudf from cuml.dask.common.base import ( BaseEstimator, DelayedInverseTransformMixin, DelayedTransformMixin, ) +from cuml.dask.common.dask_arr_utils import to_dask_cudf class DelayedFitTransformMixin: @@ -41,55 +38,57 @@ class OneHotEncoder( ): """ Encode categorical features as a one-hot numeric array. - The input to this transformer should be a dask_cuDF.DataFrame or cupy - dask.Array, denoting the values taken on by categorical features. + + The input to this transformer should be an array-like of integers or + strings, denoting the values taken on by categorical (discrete) features. The features are encoded using a one-hot (aka 'one-of-K' or 'dummy') encoding scheme. This creates a binary column for each category and - returns a sparse matrix or dense array (depending on the ``sparse`` + returns a sparse matrix or dense array (depending on the ``sparse_output`` parameter). + By default, the encoder derives the categories based on the unique values in each feature. Alternatively, you can also specify the `categories` manually. Parameters ---------- - categories : 'auto', cupy.ndarray or cudf.DataFrame, default='auto' - Categories (unique values) per feature. All categories are expected to - fit on one GPU. + categories : 'auto' or a list of array-like, default='auto' + Categories (unique values) per feature: - 'auto' : Determine categories automatically from the training data. + - list : ``categories[i]`` holds the categories expected in the ith + column. - - DataFrame/ndarray : ``categories[col]`` holds the categories expected - in the feature col. - - drop : 'first', None or a dict, default=None + drop : 'first', None, or array-like of shape (n_features,), default=None Specifies a methodology to use to drop one of the categories per feature. This is useful in situations where perfectly collinear features cause problems, such as when feeding the resulting data - into a neural network or an unregularized regression. + into an unregularized linear regression model. - - None : retain all features (the default). + However, dropping one category breaks the symmetry of the original + representation and can therefore induce a bias in downstream models, + for instance for penalized linear classification or regression models. + - None : retain all features (the default). - 'first' : drop the first category in each feature. If only one category is present, the feature will be dropped entirely. - - - Dict : ``drop[col]`` is the category in feature col that + - array : ``drop[i]`` is the category in feature ``X[:, i]`` that should be dropped. - sparse : bool, default=False - This feature was deactivated and will give an exception when True. - The reason is because sparse matrix are not fully supported by cupy - yet, causing incorrect values when computing one hot encodings. - See https://github.com/cupy/cupy/issues/3223 - dtype : number type, default=np.float - Desired datatype of transform's output. + sparse_output : bool, default=True + When ``True``, transform returns a sparse matrix/array in CSR format. + + dtype : dtype, default=np.float32 + Desired dtype of transformed output. + handle_unknown : {'error', 'ignore'}, default='error' - Whether to raise an error or ignore if an unknown categorical feature - is present during transform (default is to raise). When this parameter - is set to 'ignore' and an unknown category is encountered during - transform, the resulting one-hot encoded columns for this feature - will be all zeros. In the inverse transform, an unknown category - will be denoted as None. + Specifies the way unknown categories are handled during :meth:`transform`. + + - 'error' : Raise an error if an unknown category is present during transform. + - 'ignore' : When an unknown category is encountered during + transform, the resulting one-hot encoded columns for this feature + will be all zeros. In the inverse transform, an unknown category + will be denoted as None. """ def fit(self, X): @@ -104,14 +103,22 @@ def fit(self, X): ------- self """ - from cuml.preprocessing.onehotencoder_mg import OneHotEncoderMG + from cuml.preprocessing import OneHotEncoder - el = first(X) if isinstance(X, Sequence) else X - self.datatype = ( - "cudf" if isinstance(el, (dcDataFrame, dcSeries)) else "cupy" - ) + model = OneHotEncoder(**self.kwargs) - self._set_internal_model(OneHotEncoderMG(**self.kwargs).fit(X)) + if isinstance(X, dask_cudf.DataFrame): + self.datatype = model._input_type = model.output_type = "cudf" + else: + self.datatype = model._input_type = model.output_type = "cupy" + X = to_dask_cudf(X, client=self.client) + + X_list = self.client.compute( + [X.iloc[:, i].drop_duplicates() for i in range(X.shape[1])], + sync=True, + ) + model._fit(X_list, unique=True) + self._set_internal_model(model) return self @@ -130,18 +137,19 @@ def transform(self, X, delayed=True): out : Dask cuDF DataFrame or CuPy backed Dask Array Distributed object containing the transformed input. """ + output_collection_type = ( + "cupy" if self.kwargs.get("sparse_output", True) else self.datatype + ) return self._transform( X, n_dims=2, delayed=delayed, output_dtype=self._get_internal_model().dtype, - output_collection_type="cupy", + output_collection_type=output_collection_type, ) def inverse_transform(self, X, delayed=True): - """Convert the data back to the original representation. In case unknown - categories are encountered (all zeros in the one-hot encoding), ``None`` is used - to represent this category. + """Convert the data back to the original representation. Parameters ---------- @@ -173,29 +181,30 @@ class OrdinalEncoder( ): """Encode categorical features as an integer array. - The input to this transformer should be an :py:class:`dask_cudf.DataFrame` or a - :py:class:`dask.array.Array` backed by cupy, denoting the unique values taken on by - categorical (discrete) features. The features are converted to ordinal - integers. This results in a single column of integers (0 to n_categories - 1) per - feature. + The input to this transformer should be an array-like of integers or + strings, denoting the values taken on by categorical (discrete) features. + The features are converted to ordinal integers. This results in + a single column of integers (0 to n_categories - 1) per feature. Parameters ---------- - categories : :py:class:`cupy.ndarray` or :py:class`cudf.DataFrameq, default='auto' - Categories (unique values) per feature. All categories are expected to - fit on one GPU. + categories : 'auto' or a list of array-like, default='auto' + Categories (unique values) per feature: + - 'auto' : Determine categories automatically from the training data. - - DataFrame/ndarray : ``categories[col]`` holds the categories expected - in the feature col. + - list : ``categories[i]`` holds the categories expected in the ith + column. + + The used categories can be found in the ``categories_`` attribute. + + dtype : number type, default=np.float64 + Desired dtype of output. + handle_unknown : {'error', 'ignore'}, default='error' - Whether to raise an error or ignore if an unknown categorical feature is - present during transform (default is to raise). When this parameter is set - to 'ignore' and an unknown category is encountered during transform, the - resulting encoded value would be null when output type is cudf - dataframe. - verbose : int or boolean, default=False - Sets logging level. It must be one of `cuml.common.logger.level_*`. See - :ref:`verbosity-levels` for more info. + When set to 'error' an error will be raised in case an unknown + categorical feature is present during transform. When set to 'ignore', + the encoded value of unknown categories will be set to NaN. In + :meth:`inverse_transform`, an unknown category will be denoted as None. """ def fit(self, X): @@ -211,14 +220,22 @@ def fit(self, X): ------- self """ - from cuml.preprocessing.ordinalencoder_mg import OrdinalEncoderMG + from cuml.preprocessing import OrdinalEncoder - el = first(X) if isinstance(X, Sequence) else X - self.datatype = ( - "cudf" if isinstance(el, (dcDataFrame, dcSeries)) else "cupy" - ) + model = OrdinalEncoder(**self.kwargs) - self._set_internal_model(OrdinalEncoderMG(**self.kwargs).fit(X)) + if isinstance(X, dask_cudf.DataFrame): + self.datatype = model._input_type = model.output_type = "cudf" + else: + self.datatype = model._input_type = model.output_type = "cupy" + X = to_dask_cudf(X, client=self.client) + + X_list = self.client.compute( + [X.iloc[:, i].drop_duplicates() for i in range(X.shape[1])], + sync=True, + ) + model._fit(X_list, unique=True) + self._set_internal_model(model) return self diff --git a/python/cuml/cuml/preprocessing/onehotencoder_mg.py b/python/cuml/cuml/preprocessing/onehotencoder_mg.py deleted file mode 100644 index 4f99ee58a8..0000000000 --- a/python/cuml/cuml/preprocessing/onehotencoder_mg.py +++ /dev/null @@ -1,48 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 -# - -import cupy as cp -from cudf import DataFrame - -from cuml.preprocessing.encoders import OneHotEncoder - - -class OneHotEncoderMG(OneHotEncoder): - """ - A Multi-Node Multi-GPU implementation of OneHotEncoder - - Refer to the Dask OneHotEncoder implementation - in `cuml.dask.preprocessing.encoders`. - """ - - def __init__(self, *, client=None, **kwargs): - super().__init__(**kwargs) - self.client = client - - def _check_input_fit(self, X, is_categories=False): - """Helper function to check input of fit within the multi-gpu model""" - import dask.array - - from cuml.dask.common.dask_arr_utils import to_dask_cudf - - if isinstance(X, (dask.array.core.Array, cp.ndarray)): - self._set_input_type("array") - if is_categories: - X = X.transpose() - if isinstance(X, cp.ndarray): - return DataFrame(X) - else: - return to_dask_cudf(X, client=self.client) - else: - self._set_input_type("df") - return X - - def _unique(self, inp): - return inp.unique().compute() - - def _has_unknown(self, X_cat, encoder_cat): - if X_cat.dtype != encoder_cat.dtype: - encoder_cat = encoder_cat.astype(X_cat.dtype) - return not X_cat.isin(encoder_cat).all().compute() diff --git a/python/cuml/cuml/preprocessing/ordinalencoder_mg.py b/python/cuml/cuml/preprocessing/ordinalencoder_mg.py deleted file mode 100644 index f76744b469..0000000000 --- a/python/cuml/cuml/preprocessing/ordinalencoder_mg.py +++ /dev/null @@ -1,38 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 -# -import cupy as cp -from cudf import DataFrame - -from cuml.preprocessing.encoders import OrdinalEncoder - - -class OrdinalEncoderMG(OrdinalEncoder): - def __init__(self, *, client=None, **kwargs): - super().__init__(**kwargs) - self.client = client - - def _check_input_fit(self, X, is_categories=False): - """Helper function to check input of fit within the multi-gpu model""" - import dask.array - - from cuml.dask.common.dask_arr_utils import to_dask_cudf - - if isinstance(X, (dask.array.core.Array, cp.ndarray)): - self._set_input_type("array") - if is_categories: - X = X.transpose() - if isinstance(X, cp.ndarray): - return DataFrame(X) - else: - return to_dask_cudf(X, client=self.client) - else: - self._set_input_type("df") - return X - - def _unique(self, inp): - return inp.unique().compute() - - def _has_unknown(self, X_cat, encoder_cat): - return not X_cat.isin(encoder_cat).all().compute() diff --git a/python/cuml/tests/dask/test_dask_one_hot_encoder.py b/python/cuml/tests/dask/test_dask_one_hot_encoder.py index 8a05937021..5372a58b48 100644 --- a/python/cuml/tests/dask/test_dask_one_hot_encoder.py +++ b/python/cuml/tests/dask/test_dask_one_hot_encoder.py @@ -1,223 +1,111 @@ -# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # +import cudf import cupy as cp import dask.array as da import dask_cudf import numpy as np +import pandas as pd import pytest -from cudf import DataFrame, Series -from pandas.testing import assert_frame_equal -from sklearn.preprocessing import OneHotEncoder as SkOneHotEncoder +import sklearn.preprocessing from cuml.dask.preprocessing import OneHotEncoder -from cuml.testing.utils import ( - assert_inverse_equal, - from_df_to_numpy, - generate_inputs_from_categories, - stress_param, -) @pytest.mark.mg -def test_onehot_vs_skonehot(client): - X = DataFrame({"gender": ["Male", "Female", "Female"], "int": [1, 3, 2]}) - skX = from_df_to_numpy(X) - X = dask_cudf.from_cudf(X, npartitions=2) - - enc = OneHotEncoder(sparse_output=False) - skohe = SkOneHotEncoder(sparse_output=False) - - ohe = enc.fit_transform(X) - ref = skohe.fit_transform(skX) +@pytest.mark.parametrize("array_input", [False, True]) +@pytest.mark.parametrize("sparse_output", [False, True]) +@pytest.mark.parametrize("drop", [None, "first"]) +def test_onehot_encoder(client, array_input, sparse_output, drop): + if array_input: + data = cp.array([[10, 20, 20], [1, 3, 2]]).T + X1 = da.from_array(data, chunks=(2, 2)) + X2 = data.get() + else: + data = cudf.DataFrame( + {"gender": ["Male", "Female", "Female"], "int": [1, 3, 2]} + ) + X1 = dask_cudf.from_cudf(data, npartitions=2) + X2 = data.to_numpy() + + cu_enc = OneHotEncoder(sparse_output=sparse_output, drop=drop) + sk_enc = sklearn.preprocessing.OneHotEncoder(drop=drop) + + res = cu_enc.fit_transform(X1).compute() + sol = sk_enc.fit_transform(X2).toarray() + + if sparse_output: + res = res.toarray().get() + elif array_input: + res = res.get() + else: + res = res.to_numpy() - cp.testing.assert_array_equal(ohe.compute(), ref) + np.testing.assert_array_equal(res, sol) @pytest.mark.mg -@pytest.mark.parametrize( - "drop", [None, "first", {"g": Series("F"), "i": Series(3)}] -) +@pytest.mark.parametrize("drop", [None, "first", ["F", 3]]) def test_onehot_inverse_transform(client, drop): - df = DataFrame({"g": ["M", "F", "F"], "i": [1, 3, 2]}) + df = cudf.DataFrame({0: ["M", "F", "F"], 1: [1, 3, 2]}) X = dask_cudf.from_cudf(df, npartitions=2) enc = OneHotEncoder(drop=drop) ohe = enc.fit_transform(X) - inv = enc.inverse_transform(ohe) - assert_frame_equal( - inv.compute().to_pandas().reset_index(drop=True), - X.compute().to_pandas().reset_index(drop=True), - check_dtype=False, + res = ( + enc.inverse_transform(ohe).compute().to_pandas().reset_index(drop=True) ) + sol = X.compute().to_pandas().reset_index(drop=True) + pd.testing.assert_frame_equal(res, sol, check_dtype=False) @pytest.mark.mg def test_onehot_categories(client): - X = DataFrame({"chars": ["a", "b"], "int": [0, 2]}) + X = cudf.DataFrame({"chars": ["a", "b"], "int": [0, 2]}) X = dask_cudf.from_cudf(X, npartitions=2) - cats = DataFrame({"chars": ["a", "b", "c"], "int": [0, 1, 2]}) - enc = OneHotEncoder(categories=cats, sparse_output=False) - ref = cp.array( + enc = OneHotEncoder(categories=[["a", "b", "c"], [0, 1, 2]]) + sol = np.array( [[1.0, 0.0, 0.0, 1.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0, 0.0, 1.0]] ) - res = enc.fit_transform(X) - cp.testing.assert_array_equal(res.compute(), ref) + res = enc.fit_transform(X).compute().toarray().get() + np.testing.assert_array_equal(res, sol) @pytest.mark.mg def test_onehot_fit_handle_unknown(client): - X = DataFrame({"chars": ["a", "b"], "int": [0, 2]}) - Y = DataFrame({"chars": ["c", "b"], "int": [0, 2]}) + X = cudf.DataFrame({"chars": ["a", "b"], "int": [0, 2]}) X = dask_cudf.from_cudf(X, npartitions=2) + categories = [["c", "b"], [0, 2]] - enc = OneHotEncoder(handle_unknown="error", categories=Y) - with pytest.raises(KeyError): + enc = OneHotEncoder(handle_unknown="error", categories=categories) + with pytest.raises( + ValueError, + match="Found unknown categories \\['a'\\] in column 0 during fit", + ): enc.fit(X) - enc = OneHotEncoder(handle_unknown="ignore", categories=Y) + enc = OneHotEncoder(handle_unknown="ignore", categories=categories) enc.fit(X) @pytest.mark.mg def test_onehot_transform_handle_unknown(client): - X = DataFrame({"chars": ["a", "b"], "int": [0, 2]}) - Y = DataFrame({"chars": ["c", "b"], "int": [0, 2]}) - X = dask_cudf.from_cudf(X, npartitions=2) - Y = dask_cudf.from_cudf(Y, npartitions=2) + X1 = cudf.DataFrame({"chars": ["a", "b"], "int": [0, 2]}) + X1 = dask_cudf.from_cudf(X1, npartitions=2) + X2 = cudf.DataFrame({"chars": ["c", "b"], "int": [0, 2]}) + X2 = dask_cudf.from_cudf(X2, npartitions=2) - enc = OneHotEncoder(handle_unknown="error", sparse_output=False) - enc = enc.fit(X) + enc = OneHotEncoder(handle_unknown="error") + enc = enc.fit(X1) with pytest.raises( - ValueError, match="y contains previously unseen labels" + ValueError, + match="Found unknown categories \\['c'\\] in column 0 during transform", ): - enc.transform(Y).compute() - - enc = OneHotEncoder(handle_unknown="ignore", sparse_output=False) - enc = enc.fit(X) - ohe = enc.transform(Y) - ref = cp.array([[0.0, 0.0, 1.0, 0.0], [0.0, 1.0, 0.0, 1.0]]) - cp.testing.assert_array_equal(ohe.compute(), ref) - - -@pytest.mark.mg -def test_onehot_inverse_transform_handle_unknown(client): - X = DataFrame({"chars": ["a", "b"], "int": [0, 2]}) - X = dask_cudf.from_cudf(X, npartitions=2) - Y_ohe = cp.array([[0.0, 0.0, 1.0, 0.0], [0.0, 1.0, 0.0, 1.0]]) - Y_ohe = da.from_array(Y_ohe) + enc.transform(X2).compute() enc = OneHotEncoder(handle_unknown="ignore") - enc = enc.fit(X) - df = enc.inverse_transform(Y_ohe) - ref = DataFrame({"chars": [None, "b"], "int": [0, 2]}) - ref = dask_cudf.from_cudf(ref, npartitions=1).compute().to_pandas() - assert_frame_equal(df.compute().to_pandas(), ref, check_dtype=False) - - -@pytest.mark.mg -@pytest.mark.parametrize("drop", [None, "first"]) -@pytest.mark.parametrize("as_array", [True, False], ids=["cupy", "cudf"]) -@pytest.mark.parametrize("sparse", [True, False], ids=["sparse", "dense"]) -@pytest.mark.parametrize("n_samples", [10, 1000, stress_param(50000)]) -def test_onehot_random_inputs(client, drop, as_array, sparse, n_samples): - X, ary = generate_inputs_from_categories( - n_samples=n_samples, as_array=as_array - ) - if as_array: - dX = da.from_array(X) - else: - dX = dask_cudf.from_cudf(X, npartitions=1) - - enc = OneHotEncoder(sparse_output=sparse, drop=drop, categories="auto") - sk_enc = SkOneHotEncoder( - sparse_output=sparse, drop=drop, categories="auto" - ) - ohe = enc.fit_transform(dX) - ref = sk_enc.fit_transform(ary) - if sparse: - cp.testing.assert_array_equal(ohe.compute().toarray(), ref.toarray()) - else: - cp.testing.assert_array_equal(ohe.compute(), ref) - - inv_ohe = enc.inverse_transform(ohe) - assert_inverse_equal(inv_ohe.compute(), dX.compute(), check_dtype=False) - - -@pytest.mark.mg -def test_onehot_drop_idx_first(client): - X_ary = [["c", 2, "a"], ["b", 2, "b"]] - X = DataFrame({"chars": ["c", "b"], "int": [2, 2], "letters": ["a", "b"]}) - ddf = dask_cudf.from_cudf(X, npartitions=2) - - enc = OneHotEncoder(sparse_output=False, drop="first") - sk_enc = SkOneHotEncoder(sparse_output=False, drop="first") - ohe = enc.fit_transform(ddf) - ref = sk_enc.fit_transform(X_ary) - cp.testing.assert_array_equal(ohe.compute(), ref) - inv = enc.inverse_transform(ohe) - assert_frame_equal( - inv.compute().to_pandas().reset_index(drop=True), - ddf.compute().to_pandas().reset_index(drop=True), - check_dtype=False, - ) - - -@pytest.mark.mg -def test_onehot_drop_one_of_each(client): - X_ary = [["c", 2, "a"], ["b", 2, "b"]] - X = DataFrame({"chars": ["c", "b"], "int": [2, 2], "letters": ["a", "b"]}) - ddf = dask_cudf.from_cudf(X, npartitions=2) - - drop = dict({"chars": "b", "int": 2, "letters": "b"}) - enc = OneHotEncoder(sparse_output=False, drop=drop) - sk_enc = SkOneHotEncoder(sparse_output=False, drop=["b", 2, "b"]) - ohe = enc.fit_transform(ddf) - ref = sk_enc.fit_transform(X_ary) - cp.testing.assert_array_equal(ohe.compute(), ref) - inv = enc.inverse_transform(ohe) - assert_frame_equal( - inv.compute().to_pandas().reset_index(drop=True), - ddf.compute().to_pandas().reset_index(drop=True), - check_dtype=False, - ) - - -@pytest.mark.mg -@pytest.mark.parametrize( - "drop, pattern", - [ - [dict({"chars": "b"}), "`drop` should have as many columns"], - [ - dict({"chars": "b", "int": [2, 0]}), - "Trying to drop multiple values", - ], - [ - dict({"chars": "b", "int": 3}), - "Some categories [a-zA-Z, ]* were not found", - ], - [ - DataFrame({"chars": ["b"], "int": [3]}), - "Wrong input for parameter `drop`.", - ], - ], -) -def test_onehot_drop_exceptions(client, drop, pattern): - X = DataFrame({"chars": ["c", "b", "d"], "int": [2, 1, 0]}) - X = dask_cudf.from_cudf(X, npartitions=2) - - with pytest.raises(ValueError, match=pattern): - OneHotEncoder(sparse_output=False, drop=drop).fit(X) - - -@pytest.mark.mg -def test_onehot_get_categories(client): - X = DataFrame({"chars": ["c", "b", "d"], "ints": [2, 1, 0]}) - X = dask_cudf.from_cudf(X, npartitions=2) - - ref = [np.array(["b", "c", "d"]), np.array([0, 1, 2])] - enc = OneHotEncoder().fit(X) - cats = enc.categories_ - - for i in range(len(ref)): - np.testing.assert_array_equal(ref[i], cats[i]) + enc = enc.fit(X1) + res = enc.transform(X2).compute().toarray().get() + sol = np.array([[0.0, 0.0, 1.0, 0.0], [0.0, 1.0, 0.0, 1.0]]) + np.testing.assert_array_equal(res, sol) diff --git a/python/cuml/tests/dask/test_dask_ordinal_encoder.py b/python/cuml/tests/dask/test_dask_ordinal_encoder.py index 5dc3d66c63..ecb8462dfb 100644 --- a/python/cuml/tests/dask/test_dask_ordinal_encoder.py +++ b/python/cuml/tests/dask/test_dask_ordinal_encoder.py @@ -1,109 +1,104 @@ -# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # +import cudf import cupy as cp +import dask.array as da import dask_cudf import numpy as np import pandas as pd import pytest -from cudf import DataFrame -from distributed import Client +import sklearn.preprocessing from cuml.dask.preprocessing import OrdinalEncoder @pytest.mark.mg -def test_ordinal_encoder_df(client: Client) -> None: - X = DataFrame({"cat": ["M", "F", "F"], "int": [1, 3, 2]}) - X = dask_cudf.from_cudf(X, npartitions=2) - - enc = OrdinalEncoder() - enc.fit(X) - Xt = enc.transform(X) - - X_1 = DataFrame({"cat": ["F", "F"], "int": [1, 2]}) - X_1 = dask_cudf.from_cudf(X_1, npartitions=2) - - enc = OrdinalEncoder(client=client) - enc.fit(X) - Xt_1 = enc.transform(X_1) - - Xt_r = Xt.compute() - Xt_1_r = Xt_1.compute() - assert Xt_1_r.iloc[0, 0] == Xt_r.iloc[1, 0] - assert Xt_1_r.iloc[1, 0] == Xt_r.iloc[1, 0] - assert Xt_1_r.iloc[0, 1] == Xt_r.iloc[0, 1] - assert Xt_1_r.iloc[1, 1] == Xt_r.iloc[2, 1] +@pytest.mark.parametrize("array_input", [False, True]) +def test_ordinal_encoder(client, array_input): + if array_input: + data = cp.array([[10, 20, 20], [1, 3, 2]]).T + X1 = da.from_array(data, chunks=(2, 2)) + X2 = data.get() + else: + data = cudf.DataFrame( + {"gender": ["Male", "Female", "Female"], "int": [1, 3, 2]} + ) + X1 = dask_cudf.from_cudf(data, npartitions=2) + X2 = data.to_numpy() - # Turn Int64Index to RangeIndex for testing equality - inv_Xt = enc.inverse_transform(Xt).compute().reset_index(drop=True) - inv_Xt_1 = enc.inverse_transform(Xt_1).compute().reset_index(drop=True) + cu_enc = OrdinalEncoder() + sk_enc = sklearn.preprocessing.OrdinalEncoder() - X_r = X.compute() - X_1_r = X_1.compute() + res = cu_enc.fit_transform(X1).compute() + sol = sk_enc.fit_transform(X2) - assert inv_Xt.equals(X_r) - assert inv_Xt_1.equals(X_1_r) + if array_input: + res = res.get() + else: + res = res.to_numpy() - assert enc.n_features_in_ == 2 + np.testing.assert_array_equal(res, sol) @pytest.mark.mg -def test_ordinal_encoder_array(client: Client) -> None: - X = DataFrame({"A": [4, 1, 1], "B": [1, 3, 2]}) - X = dask_cudf.from_cudf(X, npartitions=2).values +def test_ordinal_encoder_inverse_transform(client): + df = cudf.DataFrame({0: ["M", "F", "F"], 1: [1, 3, 2]}) + X = dask_cudf.from_cudf(df, npartitions=2) enc = OrdinalEncoder() - enc.fit(X) - Xt = enc.transform(X) + Xt = enc.fit_transform(X) + res = ( + enc.inverse_transform(Xt).compute().to_pandas().reset_index(drop=True) + ) + sol = X.compute().to_pandas().reset_index(drop=True) + pd.testing.assert_frame_equal(res, sol, check_dtype=False) - X_1 = DataFrame({"A": [1, 1], "B": [1, 2]}) - X_1 = dask_cudf.from_cudf(X_1, npartitions=2).values - enc = OrdinalEncoder(client=client) - enc.fit(X) - Xt_1 = enc.transform(X_1) +@pytest.mark.mg +def test_ordinal_encoder_explicit_categories(client): + X = cudf.DataFrame({"chars": ["b", "a"], "int": [20, 10]}) + X = dask_cudf.from_cudf(X, npartitions=2) + enc = OrdinalEncoder(categories=[["a", "b", "c"], [5, 10, 20]]) + sol = np.array([[1, 2], [0, 1]]) + res = enc.fit_transform(X).compute().to_numpy() + np.testing.assert_array_equal(res, sol) - Xt_r = Xt.compute() - Xt_1_r = Xt_1.compute() - assert Xt_1_r[0, 0] == Xt_r[1, 0] - assert Xt_1_r[1, 0] == Xt_r[1, 0] - assert Xt_1_r[0, 1] == Xt_r[0, 1] - assert Xt_1_r[1, 1] == Xt_r[2, 1] - inv_Xt = enc.inverse_transform(Xt) - inv_Xt_1 = enc.inverse_transform(Xt_1) +@pytest.mark.mg +def test_ordinal_fit_handle_unknown(client): + X = cudf.DataFrame({"chars": ["a", "b"], "int": [0, 2]}) + X = dask_cudf.from_cudf(X, npartitions=2) + categories = [["c", "b"], [0, 2]] - cp.testing.assert_allclose(X.compute(), inv_Xt.compute()) - cp.testing.assert_allclose(X_1.compute(), inv_Xt_1.compute()) + enc = OrdinalEncoder(handle_unknown="error", categories=categories) + with pytest.raises( + ValueError, + match="Found unknown categories \\['a'\\] in column 0 during fit", + ): + enc.fit(X) - assert enc.n_features_in_ == 2 + enc = OrdinalEncoder(handle_unknown="ignore", categories=categories) + enc.fit(X) @pytest.mark.mg -@pytest.mark.parametrize("as_array", [True, False], ids=["cupy", "cudf"]) -def test_handle_unknown(client, as_array: bool) -> None: - X = DataFrame({"data": [0, 1]}) - Y = DataFrame({"data": [3, 1]}) - - X = dask_cudf.from_cudf(X, npartitions=2) - Y = dask_cudf.from_cudf(Y, npartitions=2) - - if as_array: - X = X.values - Y = Y.values +def test_ordinal_transform_handle_unknown(client): + X1 = cudf.DataFrame({"chars": ["a", "b"], "int": [0, 2]}) + X1 = dask_cudf.from_cudf(X1, npartitions=2) + X2 = cudf.DataFrame({"chars": ["c", "b"], "int": [0, 2]}) + X2 = dask_cudf.from_cudf(X2, npartitions=2) enc = OrdinalEncoder(handle_unknown="error") - enc = enc.fit(X) + enc = enc.fit(X1) with pytest.raises( - ValueError, match="y contains previously unseen labels" + ValueError, + match="Found unknown categories \\['c'\\] in column 0 during transform", ): - enc.transform(Y).compute() + enc.transform(X2).compute() enc = OrdinalEncoder(handle_unknown="ignore") - enc = enc.fit(X) - encoded = enc.transform(Y).compute() - if as_array: - np.isnan(encoded[0, 0]) - else: - assert pd.isna(encoded.iloc[0, 0]) + enc = enc.fit(X1) + res = enc.transform(X2).compute().to_numpy() + sol = np.array([[np.nan, 0], [1, 1]]) + np.testing.assert_array_equal(res, sol) From 47d676840b75aabc720836861dd4742e266d33e1 Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Tue, 18 Aug 2026 22:05:12 -0500 Subject: [PATCH 05/14] Respond to feedback --- python/cuml/cuml/preprocessing/encoders.py | 64 ++++++++++------------ python/cuml/tests/test_one_hot_encoder.py | 28 +++++----- python/cuml/tests/test_ordinal_encoder.py | 13 +---- 3 files changed, 46 insertions(+), 59 deletions(-) diff --git a/python/cuml/cuml/preprocessing/encoders.py b/python/cuml/cuml/preprocessing/encoders.py index 1c989d97f9..e6f94d9671 100644 --- a/python/cuml/cuml/preprocessing/encoders.py +++ b/python/cuml/cuml/preprocessing/encoders.py @@ -112,8 +112,12 @@ def _compute_categories( else np.asarray(cats, dtype=dtype) ) - # `nan` must be the last stated category - if cats.dtype.kind == "f" and np.isnan(cats[:-1]).any(): + # `nan` may only exist in floating or object dtypes, and must be + # the last stated category + if (cats.dtype.kind == "f" and np.isnan(cats[:-1]).any()) or ( + cats.dtype.kind == "O" + and any(_safe_is_nan(c) for c in cats[:-1]) + ): raise ValueError( "Nan should be the last element in user" f" provided categories, see categories {cats}" @@ -235,27 +239,19 @@ class OneHotEncoder(DeprecatedGetFeatureNamesMixin, Base): Examples -------- - Given a dataset with two features, we let the encoder find the unique - values per feature and transform the data to a binary one-hot encoding. - - >>> from sklearn.preprocessing import OneHotEncoder - - One can discard categories not seen during `fit`: - - >>> enc = OneHotEncoder(handle_unknown='ignore') - >>> X = [['Male', 1], ['Female', 3], ['Female', 2]] - >>> enc.fit(X) - OneHotEncoder(handle_unknown='ignore') + >>> import cudf + >>> from cuml.preprocessing import OneHotEncoder + >>> X = cudf.DataFrame({"fruit": ["apple", "banana", "apple"], "group": [1, 3, 2]}) + >>> enc = OneHotEncoder().fit(X) >>> enc.categories_ - [array(['Female', 'Male'], dtype=object), array([1, 2, 3], dtype=object)] - >>> enc.transform([['Female', 1], ['Male', 4]]).toarray() + [array(['apple', 'banana'], dtype=object), array([1, 2, 3])] + >>> enc.transform(X).toarray() array([[1., 0., 1., 0., 0.], - [0., 1., 0., 0., 0.]]) - >>> enc.inverse_transform([[0, 1, 1, 0, 0], [0, 0, 0, 1, 0]]) - array([['Male', 1], - [None, 2]], dtype=object) - >>> enc.get_feature_names_out(['gender', 'group']) - array(['gender_Female', 'gender_Male', 'group_1', 'group_2', 'group_3'], ...) + [0., 1., 0., 0., 1.], + [1., 0., 0., 1., 0.]], dtype=float32) + >>> enc.inverse_transform([[0, 1, 1, 0, 0], [1, 0, 0, 1, 0]]) + array([['banana', 1], + ['apple', 2]], dtype=object) """ def __init__( @@ -631,23 +627,19 @@ class OrdinalEncoder(Base): Examples -------- - Given a dataset with two features, we let the encoder find the unique - values per feature and transform the data to an ordinal encoding. - - >>> from sklearn.preprocessing import OrdinalEncoder - >>> enc = OrdinalEncoder() - >>> X = [['Male', 1], ['Female', 3], ['Female', 2]] - >>> enc.fit(X) - OrdinalEncoder() + >>> import cudf + >>> from cuml.preprocessing import OrdinalEncoder + >>> X = cudf.DataFrame({"fruit": ["apple", "banana", "apple"], "group": [1, 3, 2]}) + >>> enc = OrdinalEncoder(output_type="numpy").fit(X) >>> enc.categories_ - [array(['Female', 'Male'], dtype=object), array([1, 2, 3], dtype=object)] - >>> enc.transform([['Female', 3], ['Male', 1]]) - array([[0., 2.], - [1., 0.]]) - + [array(['apple', 'banana'], dtype=object), array([1, 2, 3])] + >>> enc.transform(X) + array([[0., 0.], + [1., 2.], + [0., 1.]]) >>> enc.inverse_transform([[1, 0], [0, 1]]) - array([['Male', 1], - ['Female', 2]], dtype=object) + array([['banana', 1], + ['apple', 2]], dtype=object) """ def __init__( diff --git a/python/cuml/tests/test_one_hot_encoder.py b/python/cuml/tests/test_one_hot_encoder.py index 9429aaa7ed..089188d747 100644 --- a/python/cuml/tests/test_one_hot_encoder.py +++ b/python/cuml/tests/test_one_hot_encoder.py @@ -58,8 +58,7 @@ def test_onehot_encoder(kind, drop, dtype, sparse_output): @pytest.mark.parametrize( "drop", [None, "first", [True, 2, 2, float("nan"), 2, "banana", "b"]] ) -@pytest.mark.parametrize("handle_unknown", ["error", "ignore"]) -def test_onehot_encoder_all_dtypes(drop, handle_unknown): +def test_onehot_encoder_all_dtypes(drop): X = pd.DataFrame( { "bool": pd.Series([False, True, False, True, False], dtype="bool"), @@ -73,22 +72,14 @@ def test_onehot_encoder_all_dtypes(drop, handle_unknown): ), } ) - kwargs = {"drop": drop, "handle_unknown": handle_unknown} - cu_enc = OneHotEncoder(**kwargs).fit(X) - sk_enc = sklearn.preprocessing.OneHotEncoder(**kwargs).fit(X) + cu_enc = OneHotEncoder(drop=drop).fit(X) + sk_enc = sklearn.preprocessing.OneHotEncoder(drop=drop).fit(X) # Check fitted attributes assert len(cu_enc.categories_) == len(sk_enc.categories_) for res, sol in zip(cu_enc.categories_, sk_enc.categories_): assert res.dtype == sol.dtype - # XXX: assert_array_equal doesn't compar NaN == NaN, we need to handle - # this case manually. Only need to check last element since NaN should - # always be last. - if res.dtype == "O" and isinstance(res[-1], float): - assert np.isnan(res[-1]) - assert np.isnan(sol[-1]) - res, sol = res[:-1], sol[:-1] - np.testing.assert_array_equal(res, sol) + pd.testing.assert_series_equal(pd.Series(res), pd.Series(sol)) if drop is not None: np.testing.assert_array_equal(cu_enc.drop_idx_, sk_enc.drop_idx_) @@ -194,6 +185,17 @@ def test_onehot_encoder_invalid_parameters(): with pytest.raises(ValueError, match="Nan should be the last element"): OneHotEncoder(categories=[[1, 2], [1, 2, 3], [float("nan"), 2]]).fit(X) + X2 = pd.DataFrame( + { + "x": [1.0, 2.0, 1.0, 2.0], + "y": ["a", None, "b", None], + } + ) + with pytest.raises(ValueError, match="Nan should be the last element"): + OneHotEncoder(categories=[[1, 2], ["a", float("nan"), "b"]]).fit(X2) + with pytest.raises(ValueError, match="Nan should be the last element"): + OneHotEncoder(categories=[[1, 2], ["a", float("nan"), "b"]]).fit(X2) + with pytest.raises(ValueError, match="In column 1, .* duplicate elements"): OneHotEncoder( categories=[[1, 2], [1, 2, 3, 3], [2, float("nan")]] diff --git a/python/cuml/tests/test_ordinal_encoder.py b/python/cuml/tests/test_ordinal_encoder.py index 2fdd00bbf4..eab315b8c4 100644 --- a/python/cuml/tests/test_ordinal_encoder.py +++ b/python/cuml/tests/test_ordinal_encoder.py @@ -41,8 +41,7 @@ def test_ordinal_encoder(kind, dtype): pd.testing.assert_frame_equal(res, sol) -@pytest.mark.parametrize("handle_unknown", ["error", "ignore"]) -def test_ordinal_encoder_all_dtypes(handle_unknown): +def test_ordinal_encoder_all_dtypes(): X = pd.DataFrame( { "bool": pd.Series([False, True, False, True, False], dtype="bool"), @@ -56,14 +55,8 @@ def test_ordinal_encoder_all_dtypes(handle_unknown): ), } ) - cu_enc = OrdinalEncoder(output_type="numpy", handle_unknown=handle_unknown) - if handle_unknown == "ignore": - sk_enc = sklearn.preprocessing.OrdinalEncoder() - else: - sk_enc = sklearn.preprocessing.OrdinalEncoder( - handle_unknown="use_encoded_value", - unknown_value=np.nan, - ) + cu_enc = OrdinalEncoder(output_type="numpy") + sk_enc = sklearn.preprocessing.OrdinalEncoder() cu_enc.fit(X) sk_enc.fit(X) From 7dba783e839774d0f1af8dda9540d39e3eafea9f Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Tue, 18 Aug 2026 22:07:01 -0500 Subject: [PATCH 06/14] Fix `KBinsDiscretizer` tests --- .../cuml/_thirdparty/sklearn/preprocessing/_discretization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_discretization.py b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_discretization.py index 0e389876d8..7d60bc1096 100644 --- a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_discretization.py +++ b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_discretization.py @@ -231,7 +231,7 @@ def fit(self, X, y=None) -> "KBinsDiscretizer": if 'onehot' in self.encode: self._encoder = OneHotEncoder( - categories=np.array([np.arange(i) for i in self.n_bins_]), + categories=[np.arange(i) for i in self.n_bins_], sparse_output=self.encode == 'onehot', output_type='cupy') # Fit the OneHotEncoder with toy datasets # so that it's ready for use after the KBinsDiscretizer is fitted From 6ecba6c5e3e854b90464c1fd8bae4b766e015edf Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Tue, 18 Aug 2026 22:20:46 -0500 Subject: [PATCH 07/14] Some cudf.pandas compat fixes --- python/cuml/cuml/preprocessing/encoders.py | 7 +++++-- python/cuml/tests/test_one_hot_encoder.py | 3 +-- python/cuml/tests/test_ordinal_encoder.py | 1 - 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/python/cuml/cuml/preprocessing/encoders.py b/python/cuml/cuml/preprocessing/encoders.py index e6f94d9671..33e2686c46 100644 --- a/python/cuml/cuml/preprocessing/encoders.py +++ b/python/cuml/cuml/preprocessing/encoders.py @@ -399,7 +399,7 @@ def transform(self, X): # cudf's CategoricalDtype doesn't allow encoding null values, # we have to handle these manually. codes = Xi.astype(cudf.CategoricalDtype(cats[:-1])).cat.codes - if Xi.has_nulls: + if Xi.has_nulls or Xi.hasnans: codes[Xi.isnull()] = len(cats) - 1 else: codes = Xi.astype(cudf.CategoricalDtype(cats)).cat.codes @@ -724,7 +724,10 @@ def transform(self, X): if ( self.handle_unknown == "error" and codes.has_nulls - and (not Xi.has_nulls or codes[Xi.notnull()].has_nulls) + and ( + (not Xi.has_nulls and not Xi.hasnans) + or codes[Xi.notnull()].has_nulls + ) ): present = ( Xi.drop_duplicates().dropna().sort_values().to_numpy() diff --git a/python/cuml/tests/test_one_hot_encoder.py b/python/cuml/tests/test_one_hot_encoder.py index 089188d747..e5bcc47441 100644 --- a/python/cuml/tests/test_one_hot_encoder.py +++ b/python/cuml/tests/test_one_hot_encoder.py @@ -56,12 +56,11 @@ def test_onehot_encoder(kind, drop, dtype, sparse_output): @pytest.mark.parametrize( - "drop", [None, "first", [True, 2, 2, float("nan"), 2, "banana", "b"]] + "drop", [None, "first", [2, 2, float("nan"), 2, "banana", "b"]] ) def test_onehot_encoder_all_dtypes(drop): X = pd.DataFrame( { - "bool": pd.Series([False, True, False, True, False], dtype="bool"), "int32": pd.Series([1, 2, 1, 2, 1], dtype="int32"), "int64": pd.Series([1, 2, 1, 2, 1], dtype="int64"), "float32": pd.Series([1, 2, float("nan"), 2, 1], dtype="float32"), diff --git a/python/cuml/tests/test_ordinal_encoder.py b/python/cuml/tests/test_ordinal_encoder.py index eab315b8c4..47d6576113 100644 --- a/python/cuml/tests/test_ordinal_encoder.py +++ b/python/cuml/tests/test_ordinal_encoder.py @@ -44,7 +44,6 @@ def test_ordinal_encoder(kind, dtype): def test_ordinal_encoder_all_dtypes(): X = pd.DataFrame( { - "bool": pd.Series([False, True, False, True, False], dtype="bool"), "int32": pd.Series([1, 2, 1, 2, 1], dtype="int32"), "int64": pd.Series([1, 2, 1, 2, 1], dtype="int64"), "float32": pd.Series([1, 2, float("nan"), 2, 1], dtype="float32"), From 4f459b71280ad618a7a567e5bde485cf82f94a9a Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Fri, 21 Aug 2026 08:12:18 -0500 Subject: [PATCH 08/14] Clean out unused testing utils --- python/cuml/cuml/testing/utils.py | 106 ------------------------------ 1 file changed, 106 deletions(-) diff --git a/python/cuml/cuml/testing/utils.py b/python/cuml/cuml/testing/utils.py index 3cb591665b..6b93664e5a 100644 --- a/python/cuml/cuml/testing/utils.py +++ b/python/cuml/cuml/testing/utils.py @@ -1,16 +1,11 @@ # SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 import inspect -from copy import deepcopy -from itertools import dropwhile from textwrap import dedent, indent -import cudf import cupy as cp import numpy as np -import pandas as pd import pytest -from cudf.pandas import LOADED as cudf_pandas_active from numba import cuda from cuml.internals.base import Base @@ -131,20 +126,6 @@ def assert_array_equal(a, b, unit_tol=1e-4, total_tol=1e-4, with_sign=True): raise AssertionError(assertion_error_msg) -def normalize_clusters(a0, b0, n_clusters): - a = as_numpy(a0) - b = as_numpy(b0) - - c = deepcopy(b) - - for i in range(n_clusters): - (idx,) = np.where(a == i) - a_to_b = c[idx[0]] - b[c == a_to_b] = i - - return a, b - - def as_type(output_type, *arrays): """Convert input arrays (cupy or numpy) to the requested output type""" out = convert_arrays( @@ -183,11 +164,6 @@ def as_cupy(*arrays): return out[0] if len(out) == 1 else tuple(out) -def clusters_equal(a0, b0, n_clusters, tol=1e-4): - a, b = normalize_clusters(a0, b0, n_clusters) - return array_equal(a, b, total_tol=tol) - - def assert_dbscan_equal(ref, actual, X, core_indices, eps): """ Utility function to compare two numpy label arrays. @@ -385,18 +361,6 @@ def generate_random_labels(random_generation_lambda, seed=1234, as_cupy=False): return cuda.to_device(a), cuda.to_device(b), a, b -def get_number_positional_args(func, default=2): - # function to return number of positional arguments in func - if hasattr(func, "__code__"): - all_args = func.__code__.co_argcount - if func.__defaults__ is not None: - kwargs = len(func.__defaults__) - else: - kwargs = 0 - return all_args - kwargs - return default - - def get_shap_values( model, explainer, @@ -416,55 +380,6 @@ def get_shap_values( return explainer, shap_values -def generate_inputs_from_categories( - categories=None, n_samples=10, seed=5060, as_array=False -): - if categories is None: - if as_array: - categories = { - "strings": list(range(1000, 4000, 3)), - "integers": list(range(1000)), - } - else: - categories = { - "strings": ["Foo", "Bar", "Baz"], - "integers": list(range(1000)), - } - - rd = np.random.RandomState(seed) - pandas_df = pd.DataFrame( - {name: rd.choice(cat, n_samples) for name, cat in categories.items()} - ) - ary = from_df_to_numpy(pandas_df) - if as_array: - inp_ary = cp.array(ary) - return inp_ary, ary - else: - if cudf_pandas_active: - df = pandas_df - else: - df = cudf.DataFrame(pandas_df) - return df, ary - - -def assert_inverse_equal(ours, ref, **kwargs): - if isinstance(ours, cp.ndarray): - cp.testing.assert_array_equal(ours, ref) - else: - if hasattr(ours, "to_pandas"): - ours = ours.to_pandas() - if hasattr(ref, "to_pandas"): - ref = ref.to_pandas() - pd.testing.assert_frame_equal(ours, ref, **kwargs) - - -def from_df_to_numpy(df): - if isinstance(df, pd.DataFrame): - return list(zip(*[df[feature] for feature in df.columns])) - else: - return list(zip(*[df[feature].to_numpy() for feature in df.columns])) - - def compare_svm( svm1, svm2, @@ -627,24 +542,3 @@ def svm_array_equal(a, b, tol=1e-6, relative_diff=True, report_summary=False): np.mean(b), ) return equal - - -def normalized_shape(shape): - """Normalize shape to tuple.""" - return (shape,) if isinstance(shape, int) else shape - - -def squeezed_shape(shape): - """Remove all trailing axes of length 1 from shape. - - Similar to, but not exactly like np.squeeze(). - """ - return tuple(reversed(list(dropwhile(lambda d: d == 1, reversed(shape))))) - - -def series_squeezed_shape(shape): - """Remove all but one axes of length 1 from shape.""" - if shape: - return tuple([d for d in normalized_shape(shape) if d != 1]) or (1,) - else: - return () From 37933281b7747425344aa8f964adb1ce04030d42 Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Fri, 21 Aug 2026 08:29:56 -0500 Subject: [PATCH 09/14] Fix `fit_transform` reflection handling in encoders --- python/cuml/cuml/preprocessing/encoders.py | 92 +++++++++++----------- python/cuml/tests/test_one_hot_encoder.py | 23 ++++++ python/cuml/tests/test_ordinal_encoder.py | 23 ++++++ 3 files changed, 94 insertions(+), 44 deletions(-) diff --git a/python/cuml/cuml/preprocessing/encoders.py b/python/cuml/cuml/preprocessing/encoders.py index 33e2686c46..eba10a3176 100644 --- a/python/cuml/cuml/preprocessing/encoders.py +++ b/python/cuml/cuml/preprocessing/encoders.py @@ -50,14 +50,14 @@ def _cats_to_series(cats): def _compute_categories( - X_list, unique=False, categories="auto", handle_unknown="error" + X, unique=False, categories="auto", handle_unknown="error" ): """Compute `categories_` for an encoder. Parameters ---------- - X_list : list[cudf.Series] - A list of columns in the input X. + X : cudf.DataFrame or list[cudf.Series] + A cudf.DataFrame or list[cudf.Series] of the input X. unique : bool, default=False Whether the columns already are reduced to only their unique entries. categories : 'auto' or list[array-like], default='auto' @@ -72,6 +72,10 @@ def _compute_categories( categories_ : list[numpy.ndarray] A list of the categories determined per-column. """ + if isinstance(X, list): + X_list = X + else: + X_list = [X.iloc[:, i] for i in range(X.shape[1])] n_features = len(X_list) if handle_unknown not in ("ignore", "error"): @@ -295,12 +299,30 @@ def fit(self, X, y=None) -> "OneHotEncoder": """Fit OneHotEncoder to X.""" check_features(self, X, reset=True) X = check_cudf(X, input_name="X") - X_list = [X.iloc[:, i] for i in range(X.shape[1])] - return self._fit(X_list) + return self._fit(X) - def _fit(self, X_list, unique=False): + @mlfunc(set_input_type=True, preserve_index=True) + @generate_docstring( + y=None, + return_values={ + "name": "X_out", + "description": ( + "Transformed input. A sparse matrix if ``sparse_output=True``, " + "dense otherwise." + ), + "type": "dense_sparse", + "shape": "(n_samples, n_encoded_features)", + }, + ) + def fit_transform(self, X, y=None): + """Fit OneHotEncoder to X, then transform X.""" + check_features(self, X, reset=True) + X = check_cudf(X, input_name="X") + return self._fit(X).transform(X) + + def _fit(self, X, unique=False): categories = _compute_categories( - X_list, + X, unique=unique, categories=self.categories, handle_unknown=self.handle_unknown, @@ -451,24 +473,6 @@ def transform(self, X): return out return out.toarray() - @mlfunc(preserve_index=True) - @generate_docstring( - y=None, - return_values={ - "name": "X_out", - "description": ( - "Transformed input. A sparse matrix if ``sparse_output=True``, " - "dense otherwise." - ), - "type": "dense_sparse", - "shape": "(n_samples, n_encoded_features)", - }, - ) - def fit_transform(self, X, y=None): - """Fit OneHotEncoder to X, then transform X.""" - X = check_cudf(X, input_name="X") - return self.fit(X).transform(X) - @mlfunc(preserve_index=True) def inverse_transform(self, X): """Convert the data back to the original representation. @@ -677,12 +681,27 @@ def fit(self, X, y=None) -> "OrdinalEncoder": """Fit OrdinalEncoder to X.""" check_features(self, X, reset=True) X = check_cudf(X, input_name="X") - X_list = [X.iloc[:, i] for i in range(X.shape[1])] - return self._fit(X_list) + return self._fit(X) - def _fit(self, X_list, unique=False): + @mlfunc(set_input_type=True, preserve_index=True) + @generate_docstring( + y=None, + return_values={ + "name": "X_out", + "description": "Transformed input.", + "type": "dense", + "shape": "(n_samples, n_features)", + }, + ) + def fit_transform(self, X, y=None): + """Fit OrdinalEncoder to X, then transform X.""" + check_features(self, X, reset=True) + X = check_cudf(X, input_name="X") + return self._fit(X).transform(X) + + def _fit(self, X, unique=False): self.categories_ = _compute_categories( - X_list, + X, unique=unique, categories=self.categories, handle_unknown=self.handle_unknown, @@ -744,21 +763,6 @@ def transform(self, X): return out - @mlfunc(preserve_index=True) - @generate_docstring( - y=None, - return_values={ - "name": "X_out", - "description": "Transformed input.", - "type": "dense", - "shape": "(n_samples, n_features)", - }, - ) - def fit_transform(self, X, y=None): - """Fit OrdinalEncoder to X, then transform X.""" - X = check_cudf(X, input_name="X") - return self.fit(X).transform(X) - @mlfunc(preserve_index=True) def inverse_transform(self, X): """Convert the data back to the original representation. diff --git a/python/cuml/tests/test_one_hot_encoder.py b/python/cuml/tests/test_one_hot_encoder.py index e5bcc47441..7338eb2f2a 100644 --- a/python/cuml/tests/test_one_hot_encoder.py +++ b/python/cuml/tests/test_one_hot_encoder.py @@ -55,6 +55,29 @@ def test_onehot_encoder(kind, drop, dtype, sparse_output): pd.testing.assert_frame_equal(res, sol) +@pytest.mark.parametrize("kind", ["numpy", "pandas"]) +def test_onehot_encoder_fit_transform(kind): + X = np.array( + [ + [2, 2, 2, 2], + [1, 2, 1, 2], + [3, 2, 1, 1], + ] + ).T + if kind == "pandas": + X = pd.DataFrame(X, columns=["a", "b", "c"]) + enc1 = OneHotEncoder(sparse_output=False).fit(X) + Xt1 = enc1.transform(X) + enc2 = OneHotEncoder(sparse_output=False) + Xt2 = enc2.fit_transform(X) + assert enc1._input_type == kind + assert enc2._input_type == kind + if kind == "pandas": + pd.testing.assert_frame_equal(Xt1, Xt2) + else: + np.testing.assert_array_equal(Xt1, Xt2) + + @pytest.mark.parametrize( "drop", [None, "first", [2, 2, float("nan"), 2, "banana", "b"]] ) diff --git a/python/cuml/tests/test_ordinal_encoder.py b/python/cuml/tests/test_ordinal_encoder.py index 47d6576113..1ca0ec5e10 100644 --- a/python/cuml/tests/test_ordinal_encoder.py +++ b/python/cuml/tests/test_ordinal_encoder.py @@ -41,6 +41,29 @@ def test_ordinal_encoder(kind, dtype): pd.testing.assert_frame_equal(res, sol) +@pytest.mark.parametrize("kind", ["numpy", "pandas"]) +def test_ordinal_encoder_fit_transform(kind): + X = np.array( + [ + [2, 2, 2, 2], + [1, 2, 1, 2], + [3, 2, 1, 1], + ] + ).T + if kind == "pandas": + X = pd.DataFrame(X, columns=["a", "b", "c"]) + enc1 = OrdinalEncoder().fit(X) + Xt1 = enc1.transform(X) + enc2 = OrdinalEncoder() + Xt2 = enc2.fit_transform(X) + assert enc1._input_type == kind + assert enc2._input_type == kind + if kind == "pandas": + pd.testing.assert_frame_equal(Xt1, Xt2) + else: + np.testing.assert_array_equal(Xt1, Xt2) + + def test_ordinal_encoder_all_dtypes(): X = pd.DataFrame( { From 4f6e1efea39b0e3cdf2bb5a9b748e6e2ecddb984 Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Fri, 21 Aug 2026 08:59:12 -0500 Subject: [PATCH 10/14] Error in `OrdinalEncoder` for non-float dtype if NaN needed --- python/cuml/cuml/preprocessing/encoders.py | 14 ++++++++++++ python/cuml/tests/test_ordinal_encoder.py | 25 ++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/python/cuml/cuml/preprocessing/encoders.py b/python/cuml/cuml/preprocessing/encoders.py index eba10a3176..6ff56feedf 100644 --- a/python/cuml/cuml/preprocessing/encoders.py +++ b/python/cuml/cuml/preprocessing/encoders.py @@ -707,12 +707,26 @@ def _fit(self, X, unique=False): handle_unknown=self.handle_unknown, ) + out_dtype = np.dtype(self.dtype) + if self.handle_unknown == "ignore" and out_dtype.kind != "f": + raise ValueError( + f"When handle_unknown='ignore', the dtype parameter " + f"should be a float dtype. Got {self.dtype}." + ) + self._missing_indices = { i: len(cats) - 1 for i, cats in enumerate(self.categories_) if _safe_is_nan(cats[-1]) } + if self._missing_indices and out_dtype.kind != "f": + raise ValueError( + "There are missing values in features " + f"{list(self._missing_indices)}. Please " + "set dtype to a float." + ) + return self @mlfunc(preserve_index=True) diff --git a/python/cuml/tests/test_ordinal_encoder.py b/python/cuml/tests/test_ordinal_encoder.py index 1ca0ec5e10..22de61f823 100644 --- a/python/cuml/tests/test_ordinal_encoder.py +++ b/python/cuml/tests/test_ordinal_encoder.py @@ -156,6 +156,15 @@ def test_ordinal_encoder_invalid_parameters(): ): OrdinalEncoder(handle_unknown="bad").fit(X) + with pytest.raises( + ValueError, + match=( + "When handle_unknown='ignore', the dtype parameter should be a " + "float dtype. Got int32." + ), + ): + OrdinalEncoder(dtype="int32", handle_unknown="ignore").fit(X) + # Invalid `categories` errors with pytest.raises(ValueError, match="Expected `categories` .* got 'bad'"): OrdinalEncoder(categories="bad").fit(X) @@ -174,6 +183,22 @@ def test_ordinal_encoder_invalid_parameters(): ).fit(X) +def test_ordinal_encoder_int_dtype(): + X1 = pd.DataFrame({"x": [1, 2, np.nan]}) + X2 = pd.DataFrame({"x": [1, 2, 1], "y": [1, 3, 2]}) + + with pytest.raises( + ValueError, + match="There are missing values in features \\[0\\].", + ): + OrdinalEncoder(dtype="int32").fit(X1) + + res = OrdinalEncoder(dtype="int32").fit_transform(X2) + np.testing.assert_array_equal( + res, np.array([[0, 0], [1, 2], [0, 1]], dtype="int32") + ) + + def test_ordinal_encoder_unknown_categories_in_fit(): X = np.array([[1, 2, float("nan"), 2]]).T From d887cc4761c698fd3c73d97424fe503eaebe3b22 Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Fri, 21 Aug 2026 11:15:03 -0500 Subject: [PATCH 11/14] Fixup unknown value errors in OrdinalEncoder --- python/cuml/cuml/preprocessing/encoders.py | 35 +++++++++++----------- python/cuml/tests/test_ordinal_encoder.py | 20 +++++++++++++ 2 files changed, 37 insertions(+), 18 deletions(-) diff --git a/python/cuml/cuml/preprocessing/encoders.py b/python/cuml/cuml/preprocessing/encoders.py index 6ff56feedf..ec939762f9 100644 --- a/python/cuml/cuml/preprocessing/encoders.py +++ b/python/cuml/cuml/preprocessing/encoders.py @@ -421,7 +421,7 @@ def transform(self, X): # cudf's CategoricalDtype doesn't allow encoding null values, # we have to handle these manually. codes = Xi.astype(cudf.CategoricalDtype(cats[:-1])).cat.codes - if Xi.has_nulls or Xi.hasnans: + if Xi.hasnans: codes[Xi.isnull()] = len(cats) - 1 else: codes = Xi.astype(cudf.CategoricalDtype(cats)).cat.codes @@ -754,25 +754,24 @@ def transform(self, X): cats = cats[:-1] codes = Xi.astype(cudf.CategoricalDtype(cats)).cat.codes - if ( - self.handle_unknown == "error" - and codes.has_nulls - and ( - (not Xi.has_nulls and not Xi.hasnans) - or codes[Xi.notnull()].has_nulls - ) - ): - present = ( - Xi.drop_duplicates().dropna().sort_values().to_numpy() - ) - diff = _get_diff(present, self.categories_[i]) - raise ValueError( - f"Found unknown categories {diff} in column {i}" - " during transform" - ) - if codes.has_nulls: + if self.handle_unknown == "error": + # If NaN is a known category and all nulls map to NaN in + # the input then there's no need to error. Otherwise error. + if not ( + i in self._missing_indices + and Xi.hasnans + and not codes[Xi.notnull()].has_nulls + ): + present = Xi.drop_duplicates().sort_values().to_numpy() + diff = _get_diff(present, self.categories_[i]) + raise ValueError( + f"Found unknown categories {diff} in column {i}" + " during transform" + ) + codes = codes.to_cupy() + out[:, i] = codes return out diff --git a/python/cuml/tests/test_ordinal_encoder.py b/python/cuml/tests/test_ordinal_encoder.py index 22de61f823..f7541a3e8b 100644 --- a/python/cuml/tests/test_ordinal_encoder.py +++ b/python/cuml/tests/test_ordinal_encoder.py @@ -226,6 +226,26 @@ def test_ordinal_encoder_transform_missing(): np.testing.assert_array_equal(res.to_numpy(), sol) +def test_ordinal_encoder_transform_missing_unknown(): + """Check error raised if unknown category is NaN""" + X1 = pd.DataFrame({"x": ["a", "b", "a"], "y": [1, 2, 1]}) + X2 = pd.DataFrame({"x": ["b", None], "y": [2, 1]}) + X3 = pd.DataFrame({"x": ["b", "a"], "y": [2, np.nan]}) + + enc = OrdinalEncoder().fit(X1) + + with pytest.raises( + ValueError, + match="Found unknown categories \\[nan\\] in column 0 during transform", + ): + enc.transform(X2) + with pytest.raises( + ValueError, + match="Found unknown categories \\[nan\\] in column 1 during transform", + ): + enc.transform(X3) + + def test_ordinal_encoder_transform_unknown(): X1 = pd.DataFrame({"x": ["a", "b", "a"]}) X2 = pd.DataFrame({"x": ["b", "c"]}) From a22412b6a2b50c24531f18d8b43ee6abd85ade8c Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Fri, 21 Aug 2026 13:05:08 -0500 Subject: [PATCH 12/14] Update umap xfail list --- python/cuml/cuml_accel_tests/upstream/umap/xfail-list.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/python/cuml/cuml_accel_tests/upstream/umap/xfail-list.yaml b/python/cuml/cuml_accel_tests/upstream/umap/xfail-list.yaml index 742e2a3d15..0741dabf4b 100644 --- a/python/cuml/cuml_accel_tests/upstream/umap/xfail-list.yaml +++ b/python/cuml/cuml_accel_tests/upstream/umap/xfail-list.yaml @@ -51,4 +51,5 @@ marker: cuml_accel_sklearn_pin condition: umap-learn<=0.5.8 and scikit-learn>=1.6 tests: + - "umap.tests.test_umap_metrics::test_sokalmichener" - "umap.tests.test_umap_validation_params::test_umap_custom_distance_w_grad" From f2bf7ec2a7e090fce19582da692910a1e95689da Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Fri, 21 Aug 2026 14:01:08 -0500 Subject: [PATCH 13/14] maybe now --- python/cuml/cuml_accel_tests/upstream/umap/xfail-list.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/cuml/cuml_accel_tests/upstream/umap/xfail-list.yaml b/python/cuml/cuml_accel_tests/upstream/umap/xfail-list.yaml index 0741dabf4b..212d30854c 100644 --- a/python/cuml/cuml_accel_tests/upstream/umap/xfail-list.yaml +++ b/python/cuml/cuml_accel_tests/upstream/umap/xfail-list.yaml @@ -51,5 +51,8 @@ marker: cuml_accel_sklearn_pin condition: umap-learn<=0.5.8 and scikit-learn>=1.6 tests: - - "umap.tests.test_umap_metrics::test_sokalmichener" - "umap.tests.test_umap_validation_params::test_umap_custom_distance_w_grad" +- reason: Test fails with old scikit-learn OR old scipy, and doesn't test our umap anyway + strict: false + tests: + - "umap.tests.test_umap_metrics::test_sokalmichener" From 9fc9991e7b331757d8a4c927147c74c00996b76c Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Fri, 21 Aug 2026 16:24:54 -0500 Subject: [PATCH 14/14] wtf --- python/cuml/cuml_accel_tests/upstream/umap/xfail-list.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/python/cuml/cuml_accel_tests/upstream/umap/xfail-list.yaml b/python/cuml/cuml_accel_tests/upstream/umap/xfail-list.yaml index 212d30854c..7fe2cf0f05 100644 --- a/python/cuml/cuml_accel_tests/upstream/umap/xfail-list.yaml +++ b/python/cuml/cuml_accel_tests/upstream/umap/xfail-list.yaml @@ -56,3 +56,4 @@ strict: false tests: - "umap.tests.test_umap_metrics::test_sokalmichener" + - "umap.tests.test_umap_metrics::test_sparse_sokalmichener"