diff --git a/python/cuml/cuml/feature_extraction/_vectorizers.py b/python/cuml/cuml/feature_extraction/_vectorizers.py index 8f1376a02b..5ec3747ddd 100644 --- a/python/cuml/cuml/feature_extraction/_vectorizers.py +++ b/python/cuml/cuml/feature_extraction/_vectorizers.py @@ -214,11 +214,15 @@ def get_char_ngrams(self, ngram_size, str_series, doc_id_sr): tokens = str_series.str.tokenize(self.delimiter) del str_series - padding = Series(self.delimiter).repeat(len(tokens)) + # tokens keeps the original per-document index (repeated per + # token); reset both to a plain range first so the two str.cat() + # calls below align positionally instead of by that index. + tokens = tokens.reset_index(drop=True) + padding = Series(self.delimiter).repeat(len(tokens)).reset_index( + drop=True + ) tokens = tokens.str.cat(padding) - padding = padding.reset_index(drop=True) tokens = padding.str.cat(tokens) - tokens = tokens.reset_index(drop=True) ngram_sr = tokens.str.character_ngrams(n=ngram_size) diff --git a/python/cuml/tests/test_text_feature_extraction.py b/python/cuml/tests/test_text_feature_extraction.py index 54a864fd43..d3bd3c4d72 100644 --- a/python/cuml/tests/test_text_feature_extraction.py +++ b/python/cuml/tests/test_text_feature_extraction.py @@ -432,6 +432,23 @@ def test_vectorizer_get_feature_names_deprecated(cls): np.testing.assert_array_equal(res, model.get_feature_names_out()) +def test_tfidf_vectorizer_char_wb_ngrams(): + # Regression test for #8416: get_char_ngrams misaligned padded tokens + # across documents once index alignment relied on the original + # per-document index instead of a reset range index. + vectorizer = TfidfVectorizer(analyzer="char_wb", ngram_range=(2, 6)) + tfidf_mat = vectorizer.fit_transform(DOCS_GPU) + + ref_vectorizer = SkTfidfVect(analyzer="char_wb", ngram_range=(2, 6)) + ref = ref_vectorizer.fit_transform(DOCS) + + cp.testing.assert_array_almost_equal(tfidf_mat.todense(), ref.toarray()) + assert_array_equal( + vectorizer.get_feature_names_out(), + ref_vectorizer.get_feature_names_out(), + ) + + # ---------------------------------------------------------------- # HashingVectorizer tests # ----------------------------------------------------------------