From 051402bfa334642eaed51ac44a5d7e6dd68efdf4 Mon Sep 17 00:00:00 2001 From: Jerry Zeng Date: Tue, 11 Aug 2026 18:32:04 +0000 Subject: [PATCH 01/25] SNOW-3923354 decouple pandas version from python connector and import pandas 3.x --- setup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 25d8590197..b55f6d70da 100644 --- a/setup.py +++ b/setup.py @@ -43,8 +43,9 @@ REQUIRED_PYTHON_VERSION = ">=3.10" PANDAS_REQUIREMENTS = [ - f"snowflake-connector-python[pandas]{CONNECTOR_DEPENDENCY_VERSION}", - "pandas<3.0.0", + f"snowflake-connector-python{CONNECTOR_DEPENDENCY_VERSION}", + "pandas<4.0.0", + "pyarrow", ] MODIN_REQUIREMENTS = [ *PANDAS_REQUIREMENTS, From 115ef720adb99092d94c8953bf8525a472edf1e7 Mon Sep 17 00:00:00 2001 From: Jerry Zeng Date: Thu, 13 Aug 2026 19:13:42 +0000 Subject: [PATCH 02/25] SNOW-3923354 preserve Series name on ColumnEmulator copies under pandas 3 ColumnEmulator replaced pandas.Series._metadata (['_name']) instead of extending it. NDFrame.__finalize__ propagates only the intersection of the two _metadata lists, so every ColumnEmulator copy silently dropped _name and .name became None. The defect is pre-existing and version-independent -- it reproduces on pandas 2.3.1 in isolation. What changed is that pandas 3.0.5 added "grouper = grouper.copy(deep=False)" to Grouping.__init__, which forces that copy on every group-by, so the pivot and group-by family started failing. Upstream: pandas-dev/pandas#61491. Beyond pivot, a nulled .name silently corrupts mock_count_distinct (_functions.py:444 keys a TableEmulator on cols[i].name, collapsing every column onto a single None key with no exception) and _functions.py:962. No test covers either, which is why this is a product bug and not a pivot quirk. TableEmulator._metadata is deliberately left alone: pandas.DataFrame._metadata is empty, so it omits nothing, and adding "_name" there makes the DataFrame->Series __finalize__ intersection non-empty and clobbers correct column labels. tests/integ/scala --local_testing_mode on pandas 3.0.5: 22 failed/595 passed -> 9 failed/608 passed, zero regressions. tests/mock unchanged at 2/462. pandas 2.3.1 stays fully green at 617 passed. --- src/snowflake/snowpark/mock/_snowflake_data_type.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/snowflake/snowpark/mock/_snowflake_data_type.py b/src/snowflake/snowpark/mock/_snowflake_data_type.py index b588de7b2e..9d9cecc45b 100644 --- a/src/snowflake/snowpark/mock/_snowflake_data_type.py +++ b/src/snowflake/snowpark/mock/_snowflake_data_type.py @@ -506,7 +506,7 @@ def broadcast_value(value: Any, len: int) -> "ColumnEmulator": class ColumnEmulator(PandasSeriesType): - _metadata = ["sf_type", "_null_rows_idxs"] + _metadata = ["_name", "sf_type", "_null_rows_idxs"] @property def _constructor(self): From de8567e1f7006387e8aca7be09ad6ecc24458d85 Mon Sep 17 00:00:00 2001 From: Jerry Zeng Date: Thu, 13 Aug 2026 21:57:45 +0000 Subject: [PATCH 03/25] SNOW-3923354 keep sort keys at object dtype so NULL survives as None DataFrame.sort_values re-wraps each sort column as pandas.Series(ndarray) before handing it to key= (pandas/core/frame.py; the line is identical in pandas 2 and 3). What changed is Series.__init__ inference: pandas 3 infers the dedicated str dtype for an object array of strings, and str's NA sentinel is nan. So a SQL NULL that is a real None in the frame reached custom_comparator as nan, `value_a is None` stopped firing, and the comparator fell through to a mixed float/str comparison. --- .../snowpark/mock/_snowflake_data_type.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/snowflake/snowpark/mock/_snowflake_data_type.py b/src/snowflake/snowpark/mock/_snowflake_data_type.py index 9d9cecc45b..5686d97561 100644 --- a/src/snowflake/snowpark/mock/_snowflake_data_type.py +++ b/src/snowflake/snowpark/mock/_snowflake_data_type.py @@ -446,7 +446,39 @@ def __setitem__(self, key, value): self.sf_types[key] = value.sf_type self._null_rows_idxs_map[key] = value._null_rows_idxs + def _object_dtype_sort_key(self, key, by): + """Wrap a ``sort_values(key=...)`` callable so it sees the untouched column. + + pandas re-wraps each sort column as ``pandas.Series(ndarray)``, which under + pandas 3 infers ``str`` dtype, whose NA sentinel is ``nan`` -- so a NULL + stored as ``None`` reaches the comparator as ``nan``. Restore ``object`` + dtype here rather than in the comparator: ``NaN`` sorts largest while NULL + obeys ``nulls_first``. + """ + labels = list(by) if isinstance(by, (list, tuple)) else [by] + + def wrapper(series): + # One sort column means we know the label outright; only a multi-column + # sort has to fall back on the name pandas set on the re-wrapped Series. + label = labels[0] if len(labels) == 1 else getattr(series, "name", None) + if label is not None: + try: + original = PandasDataframeType.__getitem__(self, label) + except (KeyError, IndexError): + original = None + if isinstance(original, PandasSeriesType) and len(original) == len( + series + ): + series = pd.Series( + original.to_numpy(dtype=object), name=label, dtype=object + ) + return key(series) + + return wrapper + def sort_values(self, by, **kwargs): + if kwargs.get("key") is not None and kwargs.get("axis", 0) in (0, "index"): + kwargs["key"] = self._object_dtype_sort_key(kwargs["key"], by) result = super().sort_values(by, **kwargs) result.sf_types = self.sf_types return result From e4bda40395dc13248fc9512c0de9948148902995 Mon Sep 17 00:00:00 2001 From: Jerry Zeng Date: Thu, 13 Aug 2026 23:23:58 +0000 Subject: [PATCH 04/25] SNOW-3923354 keep SQL NULL as None across pandas-3 dtype re-inference MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pandas 3 infers the dedicated str dtype for an object array of strings and None, and str's NA sentinel is nan—so seven mock-layer sites that deliberately produced None had it silently converted on the way out of apply, combine, DataFrame.T.apply, iterrows, and a bare ColumnEmulator built from a list. Rebuild each result at explicit object dtype; gate the variant sink on isna_helper, since iterrows re-infers per row and has no upstream dtype to preserve. --- src/snowflake/snowpark/mock/_connection.py | 8 +++- src/snowflake/snowpark/mock/_functions.py | 43 ++++++++++++++++------ src/snowflake/snowpark/mock/_plan.py | 14 +++++-- tests/mock/test_functions.py | 19 ++++++++++ 4 files changed, 66 insertions(+), 18 deletions(-) diff --git a/src/snowflake/snowpark/mock/_connection.py b/src/snowflake/snowpark/mock/_connection.py index 92fea43002..0451462e25 100644 --- a/src/snowflake/snowpark/mock/_connection.py +++ b/src/snowflake/snowpark/mock/_connection.py @@ -38,7 +38,11 @@ from snowflake.snowpark.exceptions import SnowparkSessionException from snowflake.snowpark.mock._options import pandas from snowflake.snowpark.mock._plan import MockExecutionPlan, execute_mock_plan -from snowflake.snowpark.mock._snowflake_data_type import ColumnEmulator, TableEmulator +from snowflake.snowpark.mock._snowflake_data_type import ( + ColumnEmulator, + TableEmulator, + isna_helper, +) from snowflake.snowpark.mock._stage_registry import StageEntityRegistry from snowflake.snowpark.mock._telemetry import LocalTestOOBTelemetryService from snowflake.snowpark.mock._util import get_fully_qualified_name @@ -653,7 +657,7 @@ def execute( from snowflake.snowpark.mock import CUSTOM_JSON_ENCODER for idx, row in res.iterrows(): - if row[col] is not None: + if not isna_helper(row[col]): # Snowflake sorts maps by key before serializing if isinstance(row[col], dict): row[col] = dict(sorted(row[col].items())) diff --git a/src/snowflake/snowpark/mock/_functions.py b/src/snowflake/snowpark/mock/_functions.py index 12e8d23d76..74f7db38aa 100644 --- a/src/snowflake/snowpark/mock/_functions.py +++ b/src/snowflake/snowpark/mock/_functions.py @@ -1303,7 +1303,11 @@ def convert_char(data, _fmt): raise_error=NotImplementedError, ) - res = column.combine(fmt, convert_char) + res = ColumnEmulator( + [convert_char(v, f) for v, f in zip(column, fmt)], + index=column.index, + dtype=object, + ).__finalize__(column) res.sf_type = ColumnType(StringType(), column.sf_type.nullable) return res @@ -1553,11 +1557,14 @@ def mock_parse_json(expr: ColumnEmulator): from snowflake.snowpark.mock import CUSTOM_JSON_DECODER if isinstance(expr.sf_type.datatype, StringType): - res = expr.apply( - lambda x: try_convert( - partial(json.loads, cls=CUSTOM_JSON_DECODER), False, x - ) - ) + res = ColumnEmulator( + [ + try_convert(partial(json.loads, cls=CUSTOM_JSON_DECODER), False, x) + for x in expr + ], + index=expr.index, + dtype=object, + ).__finalize__(expr) else: res = expr.copy() res.sf_type = ColumnType(VariantType(), expr.sf_type.nullable) @@ -1597,6 +1604,8 @@ def convert_variant_to_array(val): def mock_strip_null_value(expr: ColumnEmulator): return ColumnEmulator( [None if x == "null" else x for x in expr], + index=expr.index, + dtype=object, sf_type=ColumnType(expr.sf_type.datatype, True), ) @@ -2149,8 +2158,14 @@ def mock_concat(*columns: ColumnEmulator) -> ColumnEmulator: ValueError("concat expects one or more column(s) to be passed in.") ) pdf = pandas.concat(columns, axis=1) - result = pdf.T.apply( - lambda c: None if c.isnull().values.any() else c.astype(str).str.cat() + transposed = pdf.T + result = ColumnEmulator( + [ + None if c.isnull().values.any() else c.astype(str).str.cat() + for _, c in transposed.items() + ], + index=transposed.columns, + dtype=object, ) result.sf_type = ColumnType(StringType(), result.hasnans) return result @@ -2165,10 +2180,14 @@ def mock_concat_ws(*columns: ColumnEmulator) -> ColumnEmulator: ) ) pdf = pandas.concat(columns, axis=1) - result = pdf.T.apply( - lambda c: None - if c.isnull().values.any() - else c[1:].astype(str).str.cat(sep=c[0]) + transposed = pdf.T + result = ColumnEmulator( + [ + None if c.isnull().values.any() else c[1:].astype(str).str.cat(sep=c[0]) + for _, c in transposed.items() + ], + index=transposed.columns, + dtype=object, ) result.sf_type = ColumnType(StringType(), result.hasnans) return result diff --git a/src/snowflake/snowpark/mock/_plan.py b/src/snowflake/snowpark/mock/_plan.py index f794f4f72f..358470edf9 100644 --- a/src/snowflake/snowpark/mock/_plan.py +++ b/src/snowflake/snowpark/mock/_plan.py @@ -2966,14 +2966,20 @@ def get_bound(bound): and field in col[index] and col[index][field] is None ] - res = col.apply(lambda x: None if x is None or field not in x else x[field]) + res = ColumnEmulator( + [None if x is None or field not in x else x[field] for x in col], + index=col.index, + dtype=object, + ).__finalize__(col) res.sf_type = ColumnType(VariantType(), col.sf_type.nullable) return res elif isinstance(exp, SubfieldInt): col = calculate_expression(exp.child, input_data, analyzer, expr_to_alias) - res = col.apply( - lambda x: None if x is None or exp.field >= len(x) else x[exp.field] - ) + res = ColumnEmulator( + [None if x is None or exp.field >= len(x) else x[exp.field] for x in col], + index=col.index, + dtype=object, + ).__finalize__(col) res.sf_type = ColumnType(VariantType(), col.sf_type.nullable) return res elif isinstance(exp, SnowflakeUDF): diff --git a/tests/mock/test_functions.py b/tests/mock/test_functions.py index 84409240be..63c06ed95a 100644 --- a/tests/mock/test_functions.py +++ b/tests/mock/test_functions.py @@ -36,6 +36,7 @@ sum, to_char, to_date, + udf, ) from snowflake.snowpark.mock._functions import MockedFunctionRegistry, patch from snowflake.snowpark.mock._snowflake_data_type import ColumnEmulator, ColumnType @@ -884,3 +885,21 @@ def test_save_as_table_column_order_name_array_type(session): assert len(result) == 1 assert result[0]["ID"] == 1 assert result[0]["TAGS"] is None + + +def test_concat_null_does_not_reach_strict_udf(session): + # SNOW-3923354: pandas 3 re-inference leaked the NULL into the UDF as nan. + seen = [] + + @udf(strict=True, return_type=StringType(), input_types=[StringType()]) + def probe(s): + seen.append(s) + return "CALLED" + + df = session.create_dataframe([["a"], [None], ["c"]], schema=["v"]) + assert df.select(probe(concat(col("v"), col("v")))).collect() == [ + Row("CALLED"), + Row(None), + Row("CALLED"), + ] + assert sorted(seen) == ["aa", "cc"] From 24384714e8481c8eec04ce1cd0cb088fccc1b49d Mon Sep 17 00:00:00 2001 From: Jerry Zeng Date: Thu, 13 Aug 2026 23:39:42 +0000 Subject: [PATCH 05/25] SNOW-3923354 fix MERGE-INSERT default NULLs under pandas 3 Copy-on-Write MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A chained inplace replace mutates a temporary, which pandas 2 still wrote through to the parent while warning. pandas 3 is Copy-on-Write only, so the write is discarded and only announced via ChainedAssignmentError—a Warning subclass, so nothing raises. MERGE-INSERT columns omitted from the insert clause kept nan instead of None. Assign the result back. --- src/snowflake/snowpark/mock/_plan.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/snowflake/snowpark/mock/_plan.py b/src/snowflake/snowpark/mock/_plan.py index 358470edf9..0c2b549545 100644 --- a/src/snowflake/snowpark/mock/_plan.py +++ b/src/snowflake/snowpark/mock/_plan.py @@ -2005,9 +2005,9 @@ def flatten_object_cell_func(cell): for unspecified_col in set(rows_to_insert.columns).difference( inserted_columns ): - rows_to_insert[unspecified_col].replace( - np.nan, None, inplace=True - ) + rows_to_insert[unspecified_col] = rows_to_insert[ + unspecified_col + ].replace(np.nan, None) else: if len(clause.values) != len(rows_to_insert.columns): From 3c4e30f7bacd7084de5d3a6f6ac864ab94ed2554 Mon Sep 17 00:00:00 2001 From: Jerry Zeng Date: Thu, 13 Aug 2026 23:49:08 +0000 Subject: [PATCH 06/25] SNOW-3923354 derive the generated-import expectation from read_sql.__module__ pandas 3 re-homed read_sql from pandas.io.sql to the top-level pandas namespace, and code_generation routes "from X import Y" by Y.module, so the generated source legitimately changed. Product code is correct; only the expectation was stale. Interpolate the module rather than hard-coding the pandas-3 spelling, which would move the failure to the py310 job that still resolves pandas 2. --- tests/unit/test_code_generation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_code_generation.py b/tests/unit/test_code_generation.py index 52c7e54769..93eaaed5c6 100644 --- a/tests/unit/test_code_generation.py +++ b/tests/unit/test_code_generation.py @@ -592,7 +592,7 @@ def func(): assert ( generate_source_code(func) - == """\ + == f"""\ # The following comment contains the source code generated by snowpark-python for explanatory purposes. # import io # import pandas @@ -601,7 +601,7 @@ def func(): # import pandas.io.sql # import random as rd # from datetime import date as d -# from pandas.io.sql import read_sql +# from {read_sql.__module__} import read_sql # def func(): # pandas.io # io From e577c4153b084971cf3d7845b892cffa89b48bcc Mon Sep 17 00:00:00 2001 From: Jerry Zeng Date: Fri, 14 Aug 2026 07:28:34 +0000 Subject: [PATCH 07/25] SNOW-3923354 fix pandas-3 positional accessors in vectorized UDF tests pandas 3 removed the integer-as-position fallback on Series.getitem, so iloc[0][0], dtypes[0] and row[0] from iterrows became label lookups that raise KeyError. Test-only: the vectorized UDFs run fine server-side, and product code's row[0] indexes a Snowpark Row, which is a tuple subclass. The accessor fix alone is not sufficient -- it unmasks two expectations that were only hidden because the KeyError fired first. VARCHAR-transported types (STRING, ARRAY, GEOGRAPHY, GEOMETRY, MAP) now arrive as str dtype rather than object, and pandas.Timestamp.module moved to the top-level namespace. Both accept either value, matching the spelling test_pandas_udf_input_types already uses, so the py310 job on pandas 2 keeps passing. --- tests/integ/test_udf.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/tests/integ/test_udf.py b/tests/integ/test_udf.py index 7acee3c1c4..737e98421d 100644 --- a/tests/integ/test_udf.py +++ b/tests/integ/test_udf.py @@ -2374,7 +2374,7 @@ def return_type_in_dataframe(x): ("", ""), ("float64",), ), - (StringType, [["1"]], ("",), ("object",)), + (StringType, [["1"]], ("",), ("str", "object")), ( BooleanType, [[True]], @@ -2397,7 +2397,7 @@ def return_type_in_dataframe(x): ("",), ("object",), ), - (ArrayType, [[[1]]], ("",), ("object",)), + (ArrayType, [[[1]]], ("",), ("str", "object")), ( TimeType, [[datetime.time(1, 1, 1)]], @@ -2407,12 +2407,15 @@ def return_type_in_dataframe(x): ( TimestampType, [[datetime.datetime(2016, 3, 13, 5, tzinfo=datetime.timezone.utc)]], - ("",), + ( + "", + "", + ), ("datetime64[ns]",), ), - (GeographyType, [["POINT(30 10)"]], ("",), ("object",)), - (GeometryType, [["POINT(30 10)"]], ("",), ("object",)), - (MapType, [[{1: 2}]], ("",), ("object",)), + (GeographyType, [["POINT(30 10)"]], ("",), ("str", "object")), + (GeometryType, [["POINT(30 10)"]], ("",), ("str", "object")), + (MapType, [[{1: 2}]], ("",), ("str", "object")), ], ) def test_pandas_udf_return_types(session, _type, data, expected_types, expected_dtypes): @@ -2429,7 +2432,7 @@ def test_pandas_udf_return_types(session, _type, data, expected_types, expected_ immutable=True, ) result_df = df.select(series_udf("a")).to_pandas() - result_val = result_df.iloc[0][0] + result_val = result_df.iloc[0, 0] if _type in (ArrayType, MapType, GeographyType, GeometryType): # TODO: SNOW-573478 result_val = json.loads(result_val) @@ -2437,8 +2440,8 @@ def test_pandas_udf_return_types(session, _type, data, expected_types, expected_ str(type(result_val)) in expected_types ), f"returned type is {type(result_val)} instead of {expected_types}" assert ( - result_df.dtypes[0] in expected_dtypes - ), f"returned dtype is {result_df.dtypes[0]} instead of {expected_dtypes}" + result_df.dtypes.iloc[0] in expected_dtypes + ), f"returned dtype is {result_df.dtypes.iloc[0]} instead of {expected_dtypes}" @pytest.mark.skipif( @@ -2481,13 +2484,14 @@ def test_pandas_udf_return_variant(session): input_types=[PandasSeriesType(VariantType())], ) temp = df.select(series_udf("a")).to_pandas() - assert ( - temp.dtypes[0] == object - ), f"returned dtype is {temp.dtypes[0]} instead of object" + assert temp.dtypes.iloc[0] in ( + "str", + "object", + ), f"returned dtype is {temp.dtypes.iloc[0]} instead of str or object" for i, row in temp.iterrows(): assert isinstance( - row[0], expected_types[i] - ), f"returned type is {type(row[0])} instead of {expected_types[i]}" + row.iloc[0], expected_types[i] + ), f"returned type is {type(row.iloc[0])} instead of {expected_types[i]}" @pytest.mark.skipif( From 06f3cc3fac89713b02cdbc150ab5af498c9fa265 Mon Sep 17 00:00:00 2001 From: Jerry Zeng Date: Fri, 14 Aug 2026 17:40:37 +0000 Subject: [PATCH 08/25] SNOW-3923354 accept pandas-3 dtype labels in to_pandas cast expectations Pandas 3 renames two dtypes this test pins by string: uncast VARCHAR comes back as str instead of object, and the local-testing timestamp default resolution moved from ns to us. Labels only; values and instants are unchanged, so accept either spelling rather than branching on version. The live path already gets its expected timestamp dtype from pyarrow, so only local testing was hit. Verified on pandas 2.3.1 and 3.0.5, offline and against a live account. --- tests/integ/test_df_to_pandas.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/integ/test_df_to_pandas.py b/tests/integ/test_df_to_pandas.py index 862af71b90..a9dedb118c 100644 --- a/tests/integ/test_df_to_pandas.py +++ b/tests/integ/test_df_to_pandas.py @@ -104,9 +104,10 @@ def test_to_pandas_cast_integer(session, to_pandas_api, local_testing_mode): assert ( str(pandas_df.dtypes.iloc[4]) == "int64" ) # When limits are not explicitly defined, rely on metadata information from GS. - assert ( - str(pandas_df.dtypes.iloc[5]) == "object" - ) # No cast so it's a string. dtype is "object". + assert str(pandas_df.dtypes.iloc[5]) in ( + "object", + "str", + ) # No cast so it's a string: object on pandas 2, str on pandas 3. assert ( str(pandas_df.dtypes.iloc[6]) == "float64" ) # A 20-digit number is over int64 max. Convert to float64 in pandas. @@ -134,7 +135,11 @@ def test_to_pandas_cast_integer(session, to_pandas_api, local_testing_mode): assert str(timestamp_pandas_df.dtypes.iloc[0]) == expected_dtype else: # TODO: mock the non-nanosecond unit pyarrow+pandas behavior in local test - assert str(timestamp_pandas_df.dtypes.iloc[0]) == "datetime64[ns]" + # The mock layer follows pandas' own default resolution: ns on pandas 2, us on 3. + assert str(timestamp_pandas_df.dtypes.iloc[0]) in ( + "datetime64[ns]", + "datetime64[us]", + ) def test_to_pandas_precision_for_number_38_0(session): From 975ddf6b24ab14a8f88340e14c3fff47f3cd1d11 Mon Sep 17 00:00:00 2001 From: Jerry Zeng Date: Fri, 14 Aug 2026 18:15:50 +0000 Subject: [PATCH 09/25] SNOW-3923354 normalize timedelta64 to ns on the pandas write path write_pandas serializes to parquet, where a timedelta64 column becomes a duration Snowflake does not read back as a duration. The raw tick count lands in a NUMBER, so the unit is baked into the stored value. Pandas 2 inferred ns; pandas 3 infers us. The same timedelta(days=1) therefore started storing 86400000000 instead of 86400000000000 into the same column, with no warning. Nanoseconds is the contract the local testing emulator already enforces via Timedelta.value, and what every table written by an older client holds. The live path only matched that by accident, because ns was pandas' default. Normalizing at the single connector call site covers both write_pandas and create_dataframe(pdf). The to_pandas expectations go the other direction. The live path pins TIMESTAMP_NTZ to datetime64[ns] via pyarrow, while the emulator returns pandas' own default, so they cannot share a literal; the expectation is derived from local_testing_mode. String columns drop dtype=object, since plain inference now matches on both versions and both paths. Only the all-NULL column still needs an explicit dtype, because pd.Series([None]) still infers object on pandas 3. --- src/snowflake/snowpark/session.py | 13 +++++++++++++ tests/integ/test_df_to_pandas.py | 31 +++++++++++++++++++++++-------- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/src/snowflake/snowpark/session.py b/src/snowflake/snowpark/session.py index da0f4a409f..a03b21a4e8 100644 --- a/src/snowflake/snowpark/session.py +++ b/src/snowflake/snowpark/session.py @@ -3610,6 +3610,19 @@ def write_pandas( # TODO: Implement here write_pandas correctly. success, ci_output = True, [] else: + # write_pandas stores timedelta as a NUMBER of ticks with + # no unit. Normalize to ns so pandas 3's us default does + # not shrink values 1000x versus existing tables. + timedelta_columns = [ + col + for col in df.columns + if pandas.api.types.is_timedelta64_dtype(df[col].dtype) + and str(df[col].dtype) != "timedelta64[ns]" + ] + if timedelta_columns: + df = df.copy(deep=False) + for col in timedelta_columns: + df[col] = df[col].astype("timedelta64[ns]") success, _, _, ci_output = write_pandas( self._conn._conn, df, diff --git a/tests/integ/test_df_to_pandas.py b/tests/integ/test_df_to_pandas.py index a9dedb118c..516b10d9cd 100644 --- a/tests/integ/test_df_to_pandas.py +++ b/tests/integ/test_df_to_pandas.py @@ -275,7 +275,21 @@ def test_to_pandas_batches(session, local_testing_mode): @pytest.mark.skipif( IS_IN_STORED_PROC, reason="SNOW-1362480, backend optimization in different reg env" ) -def test_df_to_pandas_df(session): +def test_df_to_pandas_df(session, local_testing_mode): + # The live path builds the frame through pyarrow, which pins a TIMESTAMP_NTZ + # column to datetime64[ns]. The local testing emulator returns pandas' own + # default resolution instead, and that default moved from ns to us in pandas + # 3, so only the live expectation can be pinned. + timestamp_dtype = None if local_testing_mode else "datetime64[ns]" + # From pandas 3 on, pyarrow maps an Arrow string column onto the dedicated + # str dtype, so an all-NULL column arrives as str rather than object on the + # live path. The local testing emulator never goes through pyarrow. + null_dtype = ( + object + if local_testing_mode or int(pd.__version__.split(".")[0]) < 3 + else pd.Series([None], dtype="str").dtype + ) + df = session.create_dataframe( [ [ @@ -324,7 +338,8 @@ def test_df_to_pandas_df(session): minute=12, second=12, ) - ] + ], + dtype=timestamp_dtype, ), } ) @@ -381,7 +396,7 @@ def test_df_to_pandas_df(session): pandas_df = pd.DataFrame( { - "A": pd.Series(["[\n 1,\n 2,\n 3,\n 4\n]"], dtype=object), + "A": pd.Series(["[\n 1,\n 2,\n 3,\n 4\n]"]), "B": pd.Series([b"123"], dtype=object), "C": pd.Series([True], dtype=bool), "D": pd.Series( @@ -397,17 +412,17 @@ def test_df_to_pandas_df(session): "J": pd.Series( [100], dtype=np.int64 ), # in reg env, there can be backend optimization resulting in np.int8 - "K": pd.Series([None], dtype=object), + "K": pd.Series([None], dtype=null_dtype), "L": pd.Series( [100], dtype=np.int64 ), # in reg env, there can be backend optimization resulting in np.int8 - "M": pd.Series(["abc"], dtype=object), + "M": pd.Series(["abc"]), "N": pd.Series( - [datetime.datetime(2023, 10, 30, 12, 12, 12)], dtype="datetime64[ns]" + [datetime.datetime(2023, 10, 30, 12, 12, 12)], dtype=timestamp_dtype ), "O": pd.Series([datetime.time(12, 12, 12)], dtype=object), - "P": pd.Series(['{\n "a": "b"\n}'], dtype=object), - "Q": pd.Series(['{\n "a": "b"\n}'], dtype=object), + "P": pd.Series(['{\n "a": "b"\n}']), + "Q": pd.Series(['{\n "a": "b"\n}']), } ) assert_frame_equal(df.to_pandas(), pandas_df) From f978573ed8ac5e2c0fbfae32fa38fc1ca4eeb006 Mon Sep 17 00:00:00 2001 From: Jerry Zeng Date: Fri, 14 Aug 2026 18:24:41 +0000 Subject: [PATCH 10/25] SNOW-3923354 make the structured-type to_json expectation version-aware MAP(STRING, INT) arrives as Arrow map, so to_pandas() yields an object column of Decimal map values. Pandas 2 serializes that Decimal as a JSON number; pandas 3 emits a JSON string. Only one token changes: 1.0 becomes "1". Despite the test name, this is not a dtype change. The column is object on both versions, verified directly, so the dtype half of the expectation is left alone, as is the non-structured branch, whose values travel as VARIANT text and are byte-identical across versions. We never call DataFrame.to_json in product code; this only affects users who do. --- tests/integ/scala/test_datatype_suite.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/integ/scala/test_datatype_suite.py b/tests/integ/scala/test_datatype_suite.py index 950e05a060..221e632dd0 100644 --- a/tests/integ/scala/test_datatype_suite.py +++ b/tests/integ/scala/test_datatype_suite.py @@ -504,13 +504,19 @@ def test_structured_dtypes_select( reason="FEAT: SNOW-1372813 Cast to StructType not supported", ) def test_structured_dtypes_pandas(structured_type_session, structured_type_support): + import pandas as pd + pdf = _create_test_dataframe( structured_type_session, structured_type_support ).to_pandas() if structured_type_support: - assert ( - pdf.to_json() - == '{"MAP":{"0":[["k1",1.0]]},"OBJ":{"0":{"A":"foo","b":0.05}},"ARR":{"0":[1.0,3.1,4.5]}}' + # The MAP value arrives as a decimal.Decimal on every pandas version, and + # pandas 3 serializes Decimal to a JSON string where pandas 2 emitted a number. + map_value = '"1"' if int(pd.__version__.split(".")[0]) >= 3 else "1.0" + assert pdf.to_json() == ( + '{"MAP":{"0":[["k1",' + map_value + "]]}," + '"OBJ":{"0":{"A":"foo","b":0.05}},' + '"ARR":{"0":[1.0,3.1,4.5]}}' ) else: assert ( From 85bdce984efb96da62b12d87561a9378c94517b3 Mon Sep 17 00:00:00 2001 From: Jerry Zeng Date: Fri, 14 Aug 2026 18:45:51 +0000 Subject: [PATCH 11/25] SNOW-3923354 centralize pandas major version detection in utils Add `pandas_major_version` to `_internal/utils.py` --- src/snowflake/snowpark/_internal/utils.py | 6 ++++++ tests/integ/scala/test_datatype_suite.py | 5 ++--- tests/integ/test_df_to_pandas.py | 5 ++--- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/snowflake/snowpark/_internal/utils.py b/src/snowflake/snowpark/_internal/utils.py index 99dc105b37..f2d92c38c5 100644 --- a/src/snowflake/snowpark/_internal/utils.py +++ b/src/snowflake/snowpark/_internal/utils.py @@ -1937,6 +1937,12 @@ def get_sorted_key_for_version(version_str): ) +# Use this for pandas 2 vs 3 branches instead of parsing __version__ inline. +pandas_major_version = ( + get_sorted_key_for_version(str(pandas.__version__))[0] if installed_pandas else 0 +) + + def ttl_cache(ttl_seconds: float): """ A decorator that caches function results with a time-to-live (TTL) expiration. diff --git a/tests/integ/scala/test_datatype_suite.py b/tests/integ/scala/test_datatype_suite.py index 221e632dd0..68604e7c99 100644 --- a/tests/integ/scala/test_datatype_suite.py +++ b/tests/integ/scala/test_datatype_suite.py @@ -14,6 +14,7 @@ import snowflake.snowpark.context as context from snowflake.connector.options import installed_pandas from snowflake.snowpark import Row +from snowflake.snowpark._internal.utils import pandas_major_version from snowflake.snowpark.dataframe import DataFrame from snowflake.snowpark.exceptions import SnowparkSQLException from snowflake.snowpark.functions import ( @@ -504,15 +505,13 @@ def test_structured_dtypes_select( reason="FEAT: SNOW-1372813 Cast to StructType not supported", ) def test_structured_dtypes_pandas(structured_type_session, structured_type_support): - import pandas as pd - pdf = _create_test_dataframe( structured_type_session, structured_type_support ).to_pandas() if structured_type_support: # The MAP value arrives as a decimal.Decimal on every pandas version, and # pandas 3 serializes Decimal to a JSON string where pandas 2 emitted a number. - map_value = '"1"' if int(pd.__version__.split(".")[0]) >= 3 else "1.0" + map_value = '"1"' if pandas_major_version >= 3 else "1.0" assert pdf.to_json() == ( '{"MAP":{"0":[["k1",' + map_value + "]]}," '"OBJ":{"0":{"A":"foo","b":0.05}},' diff --git a/tests/integ/test_df_to_pandas.py b/tests/integ/test_df_to_pandas.py index 516b10d9cd..117a716c47 100644 --- a/tests/integ/test_df_to_pandas.py +++ b/tests/integ/test_df_to_pandas.py @@ -27,7 +27,7 @@ import pytest from unittest import mock -from snowflake.snowpark._internal.utils import TempObjectType +from snowflake.snowpark._internal.utils import TempObjectType, pandas_major_version from snowflake.snowpark.session import write_pandas, WRITE_PANDAS_CHUNK_SIZE from snowflake.snowpark.functions import col, div0, round, to_timestamp from snowflake.snowpark.types import ( @@ -126,7 +126,6 @@ def test_to_pandas_cast_integer(session, to_pandas_api, local_testing_mode): # Starting from pyarrow 13, pyarrow no longer coerces non-nanosecond to nanosecond for pandas >=2.0 # https://arrow.apache.org/release/13.0.0.html and https://github.com/apache/arrow/issues/33321 pyarrow_major_version = int(pa.__version__.split(".")[0]) - pandas_major_version = int(pd.__version__.split(".")[0]) expected_dtype = ( "datetime64[s]" if pyarrow_major_version >= 13 and pandas_major_version >= 2 @@ -286,7 +285,7 @@ def test_df_to_pandas_df(session, local_testing_mode): # live path. The local testing emulator never goes through pyarrow. null_dtype = ( object - if local_testing_mode or int(pd.__version__.split(".")[0]) < 3 + if local_testing_mode or pandas_major_version < 3 else pd.Series([None], dtype="str").dtype ) From 9b5a9bc94f4f6086c2453c37b573a7fe422daa94 Mon Sep 17 00:00:00 2001 From: Jerry Zeng Date: Fri, 14 Aug 2026 19:02:34 +0000 Subject: [PATCH 12/25] SNOW-3923354 update CHANGELOG with pandas 3 support notes --- CHANGELOG.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea750f0639..73c60c4f91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,24 @@ - Added interval type support for Python UDFs and stored procedures. Use `datetime.timedelta` as the type annotation for day-time interval (`DayTimeIntervalType`) parameters and return values, and `YearMonthInterval` (a type annotation sentinel from `snowflake.snowpark.types`) for year-month interval (`YearMonthIntervalType`) parameters and return values. +#### Dependency Updates + +- Added support for pandas 3, lifting the `pandas<3.0.0` cap added to the `[pandas]` extra in 1.53.0. `snowflake-snowpark-python[pandas]` now declares its own `pandas<4.0.0` and `pyarrow` requirements instead of depending on `snowflake-connector-python[pandas]`. pandas 2 remains fully supported. Upgrading Snowpark does not upgrade an existing pandas 2 install: + - pandas 3 requires Python 3.11 or later, so Python 3.10 environments continue to resolve pandas 2. + - `snowflake-snowpark-python[modin]` continues to resolve pandas 2, because modin requires `pandas<2.4`. + +#### Behavior Changes + +- When running with pandas 3, `DataFrame.to_pandas()` and `DataFrame.to_pandas_batches()` return pandas' `str` dtype for string columns (`VARCHAR`, `VARIANT`, `ARRAY`) instead of `object`. NULLs in those columns arrive as `nan` rather than `None`, so `value is None` no longer matches; use `pandas.isna(value)` instead. This is pandas 3's default for string data in both live sessions and local testing, and it cannot be disabled. `DataFrame.collect()` still returns `None` for NULL. + +#### Bug Fixes + +- Fixed a bug where `Session.write_pandas` and `Session.create_dataframe` stored a `timedelta` 1000 times too small when running with pandas 3. Snowflake stores a timedelta as a raw integer tick count, and parquet does not carry the resolution, so a column left at pandas 3's default microsecond resolution was written as microseconds. `timedelta64` columns are now normalized to nanoseconds before writing, so the stored integer matches pandas 2 even if the caller picked another resolution. +- The following local testing bugs appeared only when running with pandas 3: + - Fixed a bug where a SQL NULL became `nan` instead of `None`. `IS NULL` then evaluated to `False`, `collect()` no longer returned `None` for those NULLs, and `strict=True` UDF handlers received `nan`. This affected `parse_json`, `to_char`, `concat`, `concat_ws`, `strip_null_value`, subfield access, `ORDER BY`, and the default values of columns omitted from a MERGE insert. + - Fixed a bug where `group_by` and `pivot` results lost their column and index names. + - Fixed a bug where `create_dataframe` raised `TypeError` for `VARIANT` columns. + ## 1.54.0 (2026-07-29) ### Snowpark Python API updates From 721986ca24ac9875117399e93e83dc48e376b4ad Mon Sep 17 00:00:00 2001 From: Jerry Zeng Date: Fri, 14 Aug 2026 19:19:29 +0000 Subject: [PATCH 13/25] SNOW-3923354 rename the timedelta loop variable to avoid shadowing col --- src/snowflake/snowpark/session.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/snowflake/snowpark/session.py b/src/snowflake/snowpark/session.py index a03b21a4e8..77710e7526 100644 --- a/src/snowflake/snowpark/session.py +++ b/src/snowflake/snowpark/session.py @@ -3614,15 +3614,15 @@ def write_pandas( # no unit. Normalize to ns so pandas 3's us default does # not shrink values 1000x versus existing tables. timedelta_columns = [ - col - for col in df.columns - if pandas.api.types.is_timedelta64_dtype(df[col].dtype) - and str(df[col].dtype) != "timedelta64[ns]" + td_col + for td_col in df.columns + if pandas.api.types.is_timedelta64_dtype(df[td_col].dtype) + and str(df[td_col].dtype) != "timedelta64[ns]" ] if timedelta_columns: df = df.copy(deep=False) - for col in timedelta_columns: - df[col] = df[col].astype("timedelta64[ns]") + for td_col in timedelta_columns: + df[td_col] = df[td_col].astype("timedelta64[ns]") success, _, _, ci_output = write_pandas( self._conn._conn, df, From 01c2c1793bb4876cd31ab0c9224b03a7a65158d6 Mon Sep 17 00:00:00 2001 From: Jerry Zeng Date: Sun, 16 Aug 2026 06:42:30 +0000 Subject: [PATCH 14/25] SNOW-3923354 run local-testing precommit on pandas 2 and 3 Split the unpinned local-testing job so both majors gate the PR. --- .github/workflows/precommit.yml | 18 ++++++++++++------ tox.ini | 6 ++++++ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/.github/workflows/precommit.yml b/.github/workflows/precommit.yml index ad5cc50be0..e587461fb9 100644 --- a/.github/workflows/precommit.yml +++ b/.github/workflows/precommit.yml @@ -238,15 +238,21 @@ jobs: .tox/coverage.xml test-local-testing: - name: Test Local Testing Module py-${{ matrix.os }}-${{ matrix.python-version }} + name: Test Local Testing Module py-${{ matrix.os }}-${{ matrix.python-version }}-pandas${{ matrix.pandas-major }} needs: build runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - os: [ubuntu-latest] - python-version: ["3.14"] - cloud-provider: [azure] + include: + - os: ubuntu-latest + python-version: "3.14" + cloud-provider: azure + pandas-major: "2" + - os: ubuntu-latest + python-version: "3.14" + cloud-provider: azure + pandas-major: "3" steps: - name: Checkout Code uses: actions/checkout@v4 @@ -286,7 +292,7 @@ jobs: - name: Install tox run: uv pip install tox --system - name: Run tests - run: python -m tox -e "py${PYTHON_VERSION/\./}-local" + run: python -m tox -e "py${PYTHON_VERSION/\./}-local-pandas${{ matrix.pandas-major }}" env: PYTHON_VERSION: ${{ matrix.python-version }} cloud_provider: ${{ matrix.cloud-provider }} @@ -302,7 +308,7 @@ jobs: - uses: actions/upload-artifact@v4 with: include-hidden-files: true - name: coverage_${{ matrix.os }}-${{ matrix.python-version }}-local-testing + name: coverage_${{ matrix.os }}-${{ matrix.python-version }}-pandas${{ matrix.pandas-major }}-local-testing path: | .tox/.coverage .tox/coverage.xml diff --git a/tox.ini b/tox.ini index c77f9710f0..073a992c77 100644 --- a/tox.ini +++ b/tox.ini @@ -57,6 +57,8 @@ deps = {env:SNOWFLAKE_PYTEST_MODIN_DEPS} {env:SNOWFLAKE_PYTEST_MODIN_PIN} {env:SNOWFLAKE_PYTEST_PANDAS_DEPS} + pandas2: pandas>=2.1.2,<3 + pandas3: pandas>=3,<4 install_command = bash ./scripts/tox_install_cmd.sh {opts} {packages} setenv = COVERAGE_FILE = {env:COVERAGE_FILE:{toxworkdir}/.coverage.{envname}} @@ -129,6 +131,10 @@ passenv = GITHUB_ENV SNOWPARK_PYTHON_API_TEST_BUCKET_PATH SNOWPARK_PYTHON_API_S3_STORAGE_INTEGRATION +commands_pre = + pandas2: python -c "import pandas; assert pandas.__version__.startswith('2.'), f'wanted pandas 2, got {pandas.__version__}'" + pandas3: python -c "import pandas; assert pandas.__version__.startswith('3.'), f'wanted pandas 3, got {pandas.__version__}'" + local: python -c "import pandas, pyarrow; print('pandas', pandas.__version__, 'pyarrow', pyarrow.__version__)" commands = notudf: {env:SNOWFLAKE_PYTEST_CMD} -m "{env:SNOWFLAKE_TEST_TYPE} and not udf" {posargs:} {env:RERUN_FLAGS} src/snowflake/snowpark tests udf: {env:SNOWFLAKE_PYTEST_CMD} -m "{env:SNOWFLAKE_TEST_TYPE} or udf" {posargs:} {env:RERUN_FLAGS} src/snowflake/snowpark tests From 8bae44d696ad2bd5c5ec156bc539f309f03b737f Mon Sep 17 00:00:00 2001 From: Jerry Zeng Date: Sun, 16 Aug 2026 07:17:15 +0000 Subject: [PATCH 15/25] SNOW-3923354 normalize Arrow duration to ns on write_arrow write_arrow skipped the pandas write path, so a us duration was stored 1000x too small. Also stop calling the local-testing NULL fix pandas-3-only in the changelog. --- CHANGELOG.md | 6 ++++-- src/snowflake/snowpark/session.py | 21 +++++++++++++++++++++ tests/unit/test_session.py | 26 ++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 73c60c4f91..5638f7140e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,9 +20,11 @@ #### Bug Fixes -- Fixed a bug where `Session.write_pandas` and `Session.create_dataframe` stored a `timedelta` 1000 times too small when running with pandas 3. Snowflake stores a timedelta as a raw integer tick count, and parquet does not carry the resolution, so a column left at pandas 3's default microsecond resolution was written as microseconds. `timedelta64` columns are now normalized to nanoseconds before writing, so the stored integer matches pandas 2 even if the caller picked another resolution. +- Fixed a bug where `Session.write_pandas`, `Session.write_arrow`, and `Session.create_dataframe` stored a `timedelta` 1000 times too small when running with pandas 3. Snowflake stores a timedelta as a raw integer tick count, and parquet does not carry the resolution, so a column left at pandas 3's default microsecond resolution was written as microseconds. Duration columns are now normalized to nanoseconds before writing, so the stored integer matches pandas 2 even if the caller picked another resolution. +- Fixed a local testing bug where a SQL NULL became `nan` instead of `None` after a column was rebuilt through pandas. `IS NULL` then evaluated to `False`, `collect()` no longer returned `None` for those NULLs, and `strict=True` UDF handlers received `nan`. This affected `parse_json`, `to_char`, `concat`, `concat_ws`, `strip_null_value`, and VARIANT/OBJECT/ARRAY/MAP subfield access. The leak existed on pandas 2 as well (a missing key collected as the string `'NaN'`, and a numeric `1` collected as `'1.0'`); pandas 3 additionally did this for string values. - The following local testing bugs appeared only when running with pandas 3: - - Fixed a bug where a SQL NULL became `nan` instead of `None`. `IS NULL` then evaluated to `False`, `collect()` no longer returned `None` for those NULLs, and `strict=True` UDF handlers received `nan`. This affected `parse_json`, `to_char`, `concat`, `concat_ws`, `strip_null_value`, subfield access, `ORDER BY`, and the default values of columns omitted from a MERGE insert. + - Fixed a bug where `ORDER BY` treated SQL NULL as `nan`. + - Fixed a bug where columns omitted from a MERGE insert kept `nan` instead of NULL under Copy-on-Write. - Fixed a bug where `group_by` and `pivot` results lost their column and index names. - Fixed a bug where `create_dataframe` raised `TypeError` for `VARIANT` columns. diff --git a/src/snowflake/snowpark/session.py b/src/snowflake/snowpark/session.py index 77710e7526..973bac4a6e 100644 --- a/src/snowflake/snowpark/session.py +++ b/src/snowflake/snowpark/session.py @@ -3234,6 +3234,26 @@ def get_session_stage( self._session_stage = full_qualified_stage_name return f"{STAGE_PREFIX}{self._session_stage}" + @staticmethod + def _normalize_arrow_duration_to_ns(table: "pyarrow.Table") -> "pyarrow.Table": + # Snowflake stores duration as an integer with no unit. Cast to ns + # so a us column is not written 1000x too small. + arrays = [] + fields = [] + changed = False + for i in range(table.num_columns): + field = table.schema.field(i) + column = table.column(i) + if pyarrow.types.is_duration(field.type) and field.type.unit != "ns": + column = column.cast(pyarrow.duration("ns")) + field = field.with_type(pyarrow.duration("ns")) + changed = True + arrays.append(column) + fields.append(field) + if not changed: + return table + return pyarrow.Table.from_arrays(arrays, schema=pyarrow.schema(fields)) + @experimental(version="1.28.0") @publicapi def write_arrow( @@ -3295,6 +3315,7 @@ def write_arrow( set use_logical_type as True. Set to None to use Snowflakes default. For more information, see: https://docs.snowflake.com/en/sql-reference/sql/create-file-format """ + table = self._normalize_arrow_duration_to_ns(table) cursor = self._conn._conn.cursor() if quote_identifiers: diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index a11c557ea7..b266c5ba1e 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -22,6 +22,13 @@ except ImportError: is_pandas_available = False +try: + import pyarrow as pa + + is_pyarrow_available = True +except ImportError: + is_pyarrow_available = False + from snowflake.snowpark import Session from snowflake.snowpark._internal.server_connection import ServerConnection from snowflake.snowpark._internal.utils import parse_table_name @@ -372,6 +379,25 @@ def test_resolve_packages_optional_artifact_repository(mock_server_connection): ] +@pytest.mark.skipif(not is_pyarrow_available, reason="requires pyarrow") +def test_normalize_arrow_duration_to_ns(): + one_second = 1_000_000_000 + table = pa.table( + { + "s": pa.array([1], type=pa.duration("s")), + "ms": pa.array([1_000], type=pa.duration("ms")), + "us": pa.array([1_000_000], type=pa.duration("us")), + "ns": pa.array([one_second], type=pa.duration("ns")), + "id": pa.array([1], type=pa.int64()), + } + ) + out = Session._normalize_arrow_duration_to_ns(table) + assert out.schema.field("id").type == pa.int64() + for name in ("s", "ms", "us", "ns"): + assert out.schema.field(name).type == pa.duration("ns") + assert out.column(name).cast(pa.int64())[0].as_py() == one_second + + @pytest.mark.skipif(not is_pandas_available, reason="requires pandas for write_pandas") def test_write_pandas_wrong_table_type(mock_server_connection): session = Session(mock_server_connection) From 28972309ea3847918b0942d4590d6bccffbbf5a6 Mon Sep 17 00:00:00 2001 From: Jerry Zeng Date: Sun, 16 Aug 2026 06:42:30 +0000 Subject: [PATCH 16/25] SNOW-3923354 keep NULL as None in mock initcap, greatest, and least These three still used Series.combine after the to_char fix, so a string NULL became nan and IS NULL missed the row. --- CHANGELOG.md | 2 +- src/snowflake/snowpark/mock/_functions.py | 24 +++++++++++++++++++---- tests/mock/test_functions.py | 21 ++++++++++++++++++++ 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5638f7140e..dddd2b220f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,7 @@ #### Bug Fixes - Fixed a bug where `Session.write_pandas`, `Session.write_arrow`, and `Session.create_dataframe` stored a `timedelta` 1000 times too small when running with pandas 3. Snowflake stores a timedelta as a raw integer tick count, and parquet does not carry the resolution, so a column left at pandas 3's default microsecond resolution was written as microseconds. Duration columns are now normalized to nanoseconds before writing, so the stored integer matches pandas 2 even if the caller picked another resolution. -- Fixed a local testing bug where a SQL NULL became `nan` instead of `None` after a column was rebuilt through pandas. `IS NULL` then evaluated to `False`, `collect()` no longer returned `None` for those NULLs, and `strict=True` UDF handlers received `nan`. This affected `parse_json`, `to_char`, `concat`, `concat_ws`, `strip_null_value`, and VARIANT/OBJECT/ARRAY/MAP subfield access. The leak existed on pandas 2 as well (a missing key collected as the string `'NaN'`, and a numeric `1` collected as `'1.0'`); pandas 3 additionally did this for string values. +- Fixed a local testing bug where a SQL NULL became `nan` instead of `None` after a column was rebuilt through pandas. `IS NULL` then evaluated to `False`, `collect()` no longer returned `None` for those NULLs, and `strict=True` UDF handlers received `nan`. This affected `parse_json`, `to_char`, `concat`, `concat_ws`, `strip_null_value`, `initcap`, `greatest`, `least`, and VARIANT/OBJECT/ARRAY/MAP subfield access. The leak existed on pandas 2 as well (a missing key collected as the string `'NaN'`, and a numeric `1` collected as `'1.0'`); pandas 3 additionally did this for string values. - The following local testing bugs appeared only when running with pandas 3: - Fixed a bug where `ORDER BY` treated SQL NULL as `nan`. - Fixed a bug where columns omitted from a MERGE insert kept `nan` instead of NULL under Copy-on-Write. diff --git a/src/snowflake/snowpark/mock/_functions.py b/src/snowflake/snowpark/mock/_functions.py index 74f7db38aa..0dfc6a69c5 100644 --- a/src/snowflake/snowpark/mock/_functions.py +++ b/src/snowflake/snowpark/mock/_functions.py @@ -1990,14 +1990,22 @@ def _greatest(x: CompareType, y: Any) -> Union[CompareType, float]: @patch("greatest") def mock_greatest(*exprs: ColumnEmulator): - result = reduce(lambda x, y: x.combine(y, _greatest), exprs) + result = ColumnEmulator( + [reduce(_greatest, row) for row in zip(*exprs)], + index=exprs[0].index, + dtype=object, + ).__finalize__(exprs[0]) result.sf_type = exprs[0].sf_type return result @patch("least") def mock_least(*exprs: ColumnEmulator): - result = reduce(lambda x, y: x.combine(y, _least), exprs) + result = ColumnEmulator( + [reduce(_least, row) for row in zip(*exprs)], + index=exprs[0].index, + dtype=object, + ).__finalize__(exprs[0]) result.sf_type = exprs[0].sf_type return result @@ -2041,8 +2049,16 @@ def _initcap(value: Optional[str], delimiters: Optional[str]) -> str: @patch("initcap") -def mock_initcap(values: ColumnEmulator, delimiters: ColumnEmulator): - result = values.combine(delimiters, _initcap) +def mock_initcap(values: ColumnEmulator, delimiters: ColumnEmulator = None): + if delimiters is None: + data = [_initcap(v, None) for v in values] + else: + data = [_initcap(v, d) for v, d in zip(values, delimiters)] + result = ColumnEmulator( + data, + index=values.index, + dtype=object, + ).__finalize__(values) result.sf_type = values.sf_type return result diff --git a/tests/mock/test_functions.py b/tests/mock/test_functions.py index 63c06ed95a..f21ede4781 100644 --- a/tests/mock/test_functions.py +++ b/tests/mock/test_functions.py @@ -27,7 +27,10 @@ dense_rank, desc, get, + greatest, + initcap, is_null, + least, lit, max, min, @@ -69,6 +72,24 @@ def test_col(session): assert origin_df.select(col("o")).collect() == [Row(True), Row(False), Row(None)] +def test_initcap_greatest_least_keep_null(session): + df = session.create_dataframe( + [["hello", "a", "b"], [None, None, "b"]], + schema=["name", "a", "b"], + ) + initcap_out = df.select(initcap(col("name")).alias("R")) + assert initcap_out.collect() == [Row("Hello"), Row(None)] + assert initcap_out.filter(col("R").is_null()).count() == 1 + + greatest_out = df.select(greatest(col("a"), col("b")).alias("R")) + assert greatest_out.collect() == [Row("b"), Row(None)] + assert greatest_out.filter(col("R").is_null()).count() == 1 + + least_out = df.select(least(col("a"), col("b")).alias("R")) + assert least_out.collect() == [Row("a"), Row(None)] + assert least_out.filter(col("R").is_null()).count() == 1 + + def test_max(session): origin_df: DataFrame = session.create_dataframe( [ From 8d57a6de737c67211b3a7f7cc92cd97d33e94250 Mon Sep 17 00:00:00 2001 From: Jerry Zeng Date: Sun, 16 Aug 2026 06:42:30 +0000 Subject: [PATCH 17/25] SNOW-3923354 tighten the pandas 3 to_pandas changelog note Name the remaining semi-structured and geo string types, and stop saying the str/nan conversion cannot be disabled. --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dddd2b220f..07bf401ba9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ #### Behavior Changes -- When running with pandas 3, `DataFrame.to_pandas()` and `DataFrame.to_pandas_batches()` return pandas' `str` dtype for string columns (`VARCHAR`, `VARIANT`, `ARRAY`) instead of `object`. NULLs in those columns arrive as `nan` rather than `None`, so `value is None` no longer matches; use `pandas.isna(value)` instead. This is pandas 3's default for string data in both live sessions and local testing, and it cannot be disabled. `DataFrame.collect()` still returns `None` for NULL. +- When running with pandas 3, `DataFrame.to_pandas()` and `DataFrame.to_pandas_batches()` return pandas' `str` dtype for string columns (`VARCHAR`, `VARIANT`, `ARRAY`, `OBJECT`, `MAP`, `GEOGRAPHY`, `GEOMETRY`) instead of `object`. NULLs in those columns arrive as `nan` rather than `None`, so `value is None` no longer matches; use `pandas.isna(value)` instead. The Arrow conversion does this by default, and `pd.set_option("future.infer_string", False)` does not change that path. The JSON result-format fallback still honors the option. `DataFrame.collect()` still returns `None` for NULL. #### Bug Fixes From 9c432190086056e210c3a5f75680e0d96f3164ae Mon Sep 17 00:00:00 2001 From: Jerry Zeng Date: Mon, 17 Aug 2026 00:47:17 +0000 Subject: [PATCH 18/25] SNOW-3923354 cover duration write call sites and initcap delimiters The helper-only Arrow test stays green if write_arrow skips normalize. Pin both write paths and the mock initcap(delimiters) branch. --- tests/mock/test_functions.py | 5 +++ tests/unit/test_session.py | 62 ++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/tests/mock/test_functions.py b/tests/mock/test_functions.py index f21ede4781..3675c9a3be 100644 --- a/tests/mock/test_functions.py +++ b/tests/mock/test_functions.py @@ -89,6 +89,11 @@ def test_initcap_greatest_least_keep_null(session): assert least_out.collect() == [Row("a"), Row(None)] assert least_out.filter(col("R").is_null()).count() == 1 + delim_df = session.create_dataframe([["hello-world"], [None]], schema=["name"]) + delim_out = delim_df.select(initcap(col("name"), lit("-")).alias("R")) + assert delim_out.collect() == [Row("Hello-World"), Row(None)] + assert delim_out.filter(col("R").is_null()).count() == 1 + def test_max(session): origin_df: DataFrame = session.create_dataframe( diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index b266c5ba1e..60e84c7ff7 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -1,6 +1,7 @@ # # Copyright (c) 2012-2025 Snowflake Computing Inc. All rights reserved. # +import datetime import json import logging import os @@ -398,6 +399,67 @@ def test_normalize_arrow_duration_to_ns(): assert out.column(name).cast(pa.int64())[0].as_py() == one_second +@pytest.mark.skipif(not is_pandas_available, reason="requires pandas for write_pandas") +@pytest.mark.parametrize("unit", ["s", "ms", "us", "ns"]) +def test_write_pandas_normalizes_timedelta_to_ns(mock_server_connection, unit): + session = Session(mock_server_connection) + one_day = datetime.timedelta(days=1) + df = pandas.DataFrame( + { + "D": pandas.Series([one_day]).astype(f"timedelta64[{unit}]"), + "N": [1], + } + ) + captured = {} + + def fake_write_pandas(_conn, frame, *args, **kwargs): + captured["frame"] = frame + return True, 1, 1, [] + + with mock.patch.object( + snowflake.snowpark.session, "write_pandas", side_effect=fake_write_pandas + ): + session.write_pandas(df, table_name="t") + + frame = captured["frame"] + assert str(frame["D"].dtype) == "timedelta64[ns]" + # Snowflake stores the raw tick count with no unit, so ns is the contract. + assert frame["D"].to_numpy().astype("int64")[0] == 86_400_000_000_000 + assert str(frame["N"].dtype) == "int64" + assert str(df["D"].dtype) == f"timedelta64[{unit}]" + + +@pytest.mark.skipif(not is_pyarrow_available, reason="requires pyarrow") +@pytest.mark.parametrize( + "unit,ticks", + [("s", 1), ("ms", 1_000), ("us", 1_000_000), ("ns", 1_000_000_000)], +) +def test_write_arrow_normalizes_duration_to_ns(mock_server_connection, unit, ticks): + session = Session(mock_server_connection) + table = pa.table( + { + "D": pa.array([ticks], type=pa.duration(unit)), + "N": pa.array([1], type=pa.int64()), + } + ) + captured = {} + + def fake_write_arrow(*args, **kwargs): + captured["table"] = kwargs["table"] + return True, 1, 1, [] + + with mock.patch.object( + snowflake.snowpark.session, "write_arrow", side_effect=fake_write_arrow + ): + session.write_arrow(table, table_name="t") + + out = captured["table"] + assert out.schema.field("D").type == pa.duration("ns") + assert out.column("D").cast(pa.int64())[0].as_py() == 1_000_000_000 + assert out.schema.field("N").type == pa.int64() + assert table.schema.field("D").type == pa.duration(unit) + + @pytest.mark.skipif(not is_pandas_available, reason="requires pandas for write_pandas") def test_write_pandas_wrong_table_type(mock_server_connection): session = Session(mock_server_connection) From 666a198c2c5780de12b1782e49d9a8572aecf89f Mon Sep 17 00:00:00 2001 From: Jerry Zeng Date: Mon, 17 Aug 2026 06:16:41 +0000 Subject: [PATCH 19/25] SNOW-3923354 correct the timedelta changelog scope and trim the notes The timedelta fix applies to any non-nanosecond column, so scoping the entry to pandas 3 told the affected pandas 2 users they were unaffected, and omitted that rows written earlier need correcting. The pandas cap for modin is declared by our own [modin] extra, not by modin itself. --- CHANGELOG.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07bf401ba9..0c8ac8d06c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,16 +12,16 @@ - Added support for pandas 3, lifting the `pandas<3.0.0` cap added to the `[pandas]` extra in 1.53.0. `snowflake-snowpark-python[pandas]` now declares its own `pandas<4.0.0` and `pyarrow` requirements instead of depending on `snowflake-connector-python[pandas]`. pandas 2 remains fully supported. Upgrading Snowpark does not upgrade an existing pandas 2 install: - pandas 3 requires Python 3.11 or later, so Python 3.10 environments continue to resolve pandas 2. - - `snowflake-snowpark-python[modin]` continues to resolve pandas 2, because modin requires `pandas<2.4`. + - `snowflake-snowpark-python[modin]` continues to resolve pandas 2, because the `[modin]` extra declares `pandas<=2.4`. #### Behavior Changes -- When running with pandas 3, `DataFrame.to_pandas()` and `DataFrame.to_pandas_batches()` return pandas' `str` dtype for string columns (`VARCHAR`, `VARIANT`, `ARRAY`, `OBJECT`, `MAP`, `GEOGRAPHY`, `GEOMETRY`) instead of `object`. NULLs in those columns arrive as `nan` rather than `None`, so `value is None` no longer matches; use `pandas.isna(value)` instead. The Arrow conversion does this by default, and `pd.set_option("future.infer_string", False)` does not change that path. The JSON result-format fallback still honors the option. `DataFrame.collect()` still returns `None` for NULL. +- On pandas 3, `DataFrame.to_pandas()` and `DataFrame.to_pandas_batches()` return the `str` dtype instead of `object` for text columns, including `VARIANT`, `OBJECT`, `ARRAY`, `MAP` and the geospatial types. NULLs in those columns come back as `nan` instead of `None`, so test them with `pandas.isna(value)` rather than `value is None`. Setting `future.infer_string` to `False` does not turn this off. `DataFrame.collect()` is unchanged and still returns `None`. #### Bug Fixes -- Fixed a bug where `Session.write_pandas`, `Session.write_arrow`, and `Session.create_dataframe` stored a `timedelta` 1000 times too small when running with pandas 3. Snowflake stores a timedelta as a raw integer tick count, and parquet does not carry the resolution, so a column left at pandas 3's default microsecond resolution was written as microseconds. Duration columns are now normalized to nanoseconds before writing, so the stored integer matches pandas 2 even if the caller picked another resolution. -- Fixed a local testing bug where a SQL NULL became `nan` instead of `None` after a column was rebuilt through pandas. `IS NULL` then evaluated to `False`, `collect()` no longer returned `None` for those NULLs, and `strict=True` UDF handlers received `nan`. This affected `parse_json`, `to_char`, `concat`, `concat_ws`, `strip_null_value`, `initcap`, `greatest`, `least`, and VARIANT/OBJECT/ARRAY/MAP subfield access. The leak existed on pandas 2 as well (a missing key collected as the string `'NaN'`, and a numeric `1` collected as `'1.0'`); pandas 3 additionally did this for string values. +- Fixed a bug where `Session.write_pandas`, `Session.write_arrow`, and `Session.create_dataframe` stored a `timedelta` at the column's own resolution instead of nanoseconds, so a microsecond column stored a value 1000 times too small. Duration columns are now converted to nanoseconds before writing. pandas 3 defaults to microseconds and is much more likely to hit this, but a non-nanosecond column on pandas 2 was affected too. Rows an earlier release wrote from such a column are too small and need correcting. +- Fixed a local testing bug where a SQL NULL became `nan` instead of `None` after a column was rebuilt through pandas. `IS NULL` then evaluated to `False`, `collect()` no longer returned `None` for those NULLs, and `strict=True` UDF handlers received `nan`. This affected `parse_json`, `to_char`, `concat`, `concat_ws`, `strip_null_value`, `initcap`, `greatest`, `least`, and VARIANT/OBJECT/ARRAY/MAP subfield access. This happened on pandas 2 too: a missing key collected as the string `'NaN'`, and a numeric `1` as `'1.0'`. pandas 3 also did it for string values. - The following local testing bugs appeared only when running with pandas 3: - Fixed a bug where `ORDER BY` treated SQL NULL as `nan`. - Fixed a bug where columns omitted from a MERGE insert kept `nan` instead of NULL under Copy-on-Write. From cf9a8eb1b0974aea85d2ae8a0005de5b5e30fbf8 Mon Sep 17 00:00:00 2001 From: Jerry Zeng Date: Sun, 23 Aug 2026 03:13:18 +0000 Subject: [PATCH 20/25] SNOW-3923354 derive pandas major version locally in tests The stored procedure suite uploads our test files to the server but runs them against the snowpark bundled in the Python UDF sandbox, which ships the released version rather than the branch build. A test module that imports a symbol added on this branch therefore cannot be collected at all, losing the whole file instead of skipping one assertion: ImportError: cannot import name 'pandas_major_version' from 'snowflake.snowpark._internal.utils' Jenkins trigger #103 caught this in PythonStoredProcBuildSnowfortTest. GitHub CI cannot, because there the installed snowpark is the branch itself and the import resolves. test_df_to_pandas.py carried the same import and was a latent second instance, masked by SNOW-3674599 skipping it for lack of pandas on Python 3.14. The constant in _internal/utils.py stays for product code, which always runs against its own tree. --- tests/integ/scala/test_datatype_suite.py | 6 +++++- tests/integ/test_df_to_pandas.py | 5 ++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/integ/scala/test_datatype_suite.py b/tests/integ/scala/test_datatype_suite.py index 68604e7c99..05c38791ab 100644 --- a/tests/integ/scala/test_datatype_suite.py +++ b/tests/integ/scala/test_datatype_suite.py @@ -14,7 +14,6 @@ import snowflake.snowpark.context as context from snowflake.connector.options import installed_pandas from snowflake.snowpark import Row -from snowflake.snowpark._internal.utils import pandas_major_version from snowflake.snowpark.dataframe import DataFrame from snowflake.snowpark.exceptions import SnowparkSQLException from snowflake.snowpark.functions import ( @@ -505,6 +504,11 @@ def test_structured_dtypes_select( reason="FEAT: SNOW-1372813 Cast to StructType not supported", ) def test_structured_dtypes_pandas(structured_type_session, structured_type_support): + # Not imported from _internal.utils: stored procs run the server's released snowpark. + import pandas + + pandas_major_version = int(pandas.__version__.split(".")[0]) + pdf = _create_test_dataframe( structured_type_session, structured_type_support ).to_pandas() diff --git a/tests/integ/test_df_to_pandas.py b/tests/integ/test_df_to_pandas.py index 117a716c47..881bfad663 100644 --- a/tests/integ/test_df_to_pandas.py +++ b/tests/integ/test_df_to_pandas.py @@ -27,7 +27,7 @@ import pytest from unittest import mock -from snowflake.snowpark._internal.utils import TempObjectType, pandas_major_version +from snowflake.snowpark._internal.utils import TempObjectType from snowflake.snowpark.session import write_pandas, WRITE_PANDAS_CHUNK_SIZE from snowflake.snowpark.functions import col, div0, round, to_timestamp from snowflake.snowpark.types import ( @@ -53,6 +53,9 @@ ) from tests.utils import IS_IN_STORED_PROC, Utils +# Not imported from _internal.utils: stored procs run the server's released snowpark. +pandas_major_version = int(pd.__version__.split(".")[0]) + def test_to_pandas_new_df_from_range(session): # Single column From a31458f5752a19f5ccb2fbccd1c094cdc9b42deb Mon Sep 17 00:00:00 2001 From: Jerry Zeng Date: Wed, 26 Aug 2026 21:25:52 +0000 Subject: [PATCH 21/25] SNOW-3923354 drop the unused pandas_major_version helper The two test files that imported this symbol now derive the major version locally, because stored procedures run the server's released snowpark and importing it there fails collection with ImportError. That left the helper with no consumers. Keeping it is not neutral: the per-BCR plan bans version branches in product code wherever a version-agnostic spelling exists, so the helper has no intended caller by design, and anyone who imports it again reintroduces the stored-proc collection failure -- which only surfaces in the sproc suite, against a bundled older snowpark, where regular CI cannot see it. With this removed, _internal/utils.py is untouched by this branch. --- src/snowflake/snowpark/_internal/utils.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/snowflake/snowpark/_internal/utils.py b/src/snowflake/snowpark/_internal/utils.py index f2d92c38c5..99dc105b37 100644 --- a/src/snowflake/snowpark/_internal/utils.py +++ b/src/snowflake/snowpark/_internal/utils.py @@ -1937,12 +1937,6 @@ def get_sorted_key_for_version(version_str): ) -# Use this for pandas 2 vs 3 branches instead of parsing __version__ inline. -pandas_major_version = ( - get_sorted_key_for_version(str(pandas.__version__))[0] if installed_pandas else 0 -) - - def ttl_cache(ttl_seconds: float): """ A decorator that caches function results with a time-to-live (TTL) expiration. From c409453725730918984955c5bbce5653748062db Mon Sep 17 00:00:00 2001 From: Jerry Zeng Date: Wed, 26 Aug 2026 21:26:12 +0000 Subject: [PATCH 22/25] SNOW-3923354 drop the redundant connector requirement from the pandas extra Before this branch the entry was snowflake-connector-python[pandas], and the extra suffix was the point: it is what pulled the connector's pandas and pyarrow in. Decoupling from that extra is the whole purpose of this change, which left the line resolving to exactly the specifier install_requires already declares via CONNECTOR_DEPENDENCY. An extra cannot be installed without the base package, so a [pandas] user always had the connector regardless. It also spelled the requirement out instead of reusing the constant, so it would drift if that constant's shape ever changed. The secure-local-storage extra keeps its full spelling, because that one does request an extra install_requires does not provide. --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index b55f6d70da..5cfaf033fc 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,6 @@ REQUIRED_PYTHON_VERSION = ">=3.10" PANDAS_REQUIREMENTS = [ - f"snowflake-connector-python{CONNECTOR_DEPENDENCY_VERSION}", "pandas<4.0.0", "pyarrow", ] From 8aa19c023e40c5613e240488b13e0bbf1b5d3bb1 Mon Sep 17 00:00:00 2001 From: Jerry Zeng Date: Wed, 26 Aug 2026 21:26:29 +0000 Subject: [PATCH 23/25] SNOW-3923354 keep Arrow schema metadata when normalizing duration to ns pyarrow.schema() builds a schema with no metadata, so rebuilding the table after casting a non-nanosecond duration column silently discarded whatever the caller's schema carried -- for a table produced from a pandas DataFrame that includes the pandas dtype and index information. Only the converting branch was affected; a table with no duration column returns early and was never touched. Found while reviewing this branch, in code the branch itself introduces, so there is nothing shipped to note. --- src/snowflake/snowpark/session.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/snowflake/snowpark/session.py b/src/snowflake/snowpark/session.py index 973bac4a6e..27228b25c9 100644 --- a/src/snowflake/snowpark/session.py +++ b/src/snowflake/snowpark/session.py @@ -3252,7 +3252,10 @@ def _normalize_arrow_duration_to_ns(table: "pyarrow.Table") -> "pyarrow.Table": fields.append(field) if not changed: return table - return pyarrow.Table.from_arrays(arrays, schema=pyarrow.schema(fields)) + # Carry the original schema metadata over: pyarrow.schema() drops it otherwise. + return pyarrow.Table.from_arrays( + arrays, schema=pyarrow.schema(fields, metadata=table.schema.metadata) + ) @experimental(version="1.28.0") @publicapi From 4c14a3c6111fafde473036d9fcd94352c125078a Mon Sep 17 00:00:00 2001 From: Jerry Zeng Date: Wed, 26 Aug 2026 21:26:45 +0000 Subject: [PATCH 24/25] SNOW-3923354 tighten the pandas 3 changelog entries Measured against the 1.53.0 and 1.54.0 sections, repo bullets run a median of 25 words with a maximum of 77. Two of ours exceeded that maximum at 78 and 84, so they are cut to 52 and 56 by dropping mechanism rather than consequence -- the data-correction warning and the affected-function list both stay, since those are what a reader acts on. The behaviour-change entry also asserted that DataFrame.collect() is unchanged. A "Behavior Changes" section describing something that did not change invites the reader to look for a relationship that is not there, and it read as contradicting the local-testing bug fix a few lines below, which is on the mock path. Removed, and the entry now states the dtype change as the cause of the NULL change rather than listing them as two separate facts. --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c8ac8d06c..71647f9bea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,12 +16,12 @@ #### Behavior Changes -- On pandas 3, `DataFrame.to_pandas()` and `DataFrame.to_pandas_batches()` return the `str` dtype instead of `object` for text columns, including `VARIANT`, `OBJECT`, `ARRAY`, `MAP` and the geospatial types. NULLs in those columns come back as `nan` instead of `None`, so test them with `pandas.isna(value)` rather than `value is None`. Setting `future.infer_string` to `False` does not turn this off. `DataFrame.collect()` is unchanged and still returns `None`. +- On pandas 3, `DataFrame.to_pandas()` and `DataFrame.to_pandas_batches()` return text columns as the `str` dtype instead of `object`, so a SQL NULL in them arrives as `nan` instead of `None`. Test for NULL with `pandas.isna(value)`, not `value is None`. This also covers `VARIANT`, `OBJECT`, `ARRAY`, `MAP` and the geospatial types, and the `future.infer_string` option does not opt out of it. #### Bug Fixes -- Fixed a bug where `Session.write_pandas`, `Session.write_arrow`, and `Session.create_dataframe` stored a `timedelta` at the column's own resolution instead of nanoseconds, so a microsecond column stored a value 1000 times too small. Duration columns are now converted to nanoseconds before writing. pandas 3 defaults to microseconds and is much more likely to hit this, but a non-nanosecond column on pandas 2 was affected too. Rows an earlier release wrote from such a column are too small and need correcting. -- Fixed a local testing bug where a SQL NULL became `nan` instead of `None` after a column was rebuilt through pandas. `IS NULL` then evaluated to `False`, `collect()` no longer returned `None` for those NULLs, and `strict=True` UDF handlers received `nan`. This affected `parse_json`, `to_char`, `concat`, `concat_ws`, `strip_null_value`, `initcap`, `greatest`, `least`, and VARIANT/OBJECT/ARRAY/MAP subfield access. This happened on pandas 2 too: a missing key collected as the string `'NaN'`, and a numeric `1` as `'1.0'`. pandas 3 also did it for string values. +- Fixed a bug where `Session.write_pandas`, `Session.write_arrow`, and `Session.create_dataframe` stored a `timedelta` at the column's own resolution instead of nanoseconds, so a microsecond column was written 1000 times too small. Duration columns are now converted to nanoseconds first. pandas 2 was affected too, and rows written by an earlier release need correcting. +- Fixed a local testing bug where a SQL NULL became `nan` instead of `None` after a column was rebuilt through pandas, so `IS NULL` evaluated to `False` and `strict=True` UDF handlers received `nan`. This affected `parse_json`, `to_char`, `concat`, `concat_ws`, `strip_null_value`, `initcap`, `greatest`, `least`, and VARIANT/OBJECT/ARRAY/MAP subfield access, on pandas 2 as well as pandas 3. - The following local testing bugs appeared only when running with pandas 3: - Fixed a bug where `ORDER BY` treated SQL NULL as `nan`. - Fixed a bug where columns omitted from a MERGE insert kept `nan` instead of NULL under Copy-on-Write. From 76bc2a9e92ba1d951742e2e6fc3c4faa232ba227 Mon Sep 17 00:00:00 2001 From: Jerry Zeng Date: Wed, 26 Aug 2026 22:25:45 +0000 Subject: [PATCH 25/25] SNOW-3923354 bound pyarrow in the pandas extra, mirroring the connector Dropping connector[pandas] also dropped its pyarrow bounds, which we had been inheriting: a floor of >=14.0.1 since connector 4.5.0, plus a <24 ceiling on Python 3.14 added in 4.7.2. A bare pyarrow left both off. The floor mattered because our supported connector range starts at 3.17.0, whose own pyarrow requirement is unbounded, so nothing stopped a very old pyarrow from resolving. The ceiling mattered more: measured on CPython 3.14, snowpark[pandas] resolved pyarrow 25.0.1, and the connector then warned at import that this is a version it declares incompatible with its own Arrow code. With these bounds it resolves 23.0.1 and the warning is gone. pandas still resolves 3.0.5 on 3.12 through 3.14, so the point of the change is intact. These are deliberately a line-for-line mirror of the connector's own two pyarrow entries rather than a shorter equivalent spelling, so that diffing ours against theirs stays trivial when they move the bound -- they have moved it several times. The same marker-split idiom is already used two entries above for protobuf, for the same Python 3.14 reason. Only the pyarrow lines are mirrored. The connector's two pandas lines carry the <3.0.0 cap this change exists to lift, so those are deliberately not copied. --- setup.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5cfaf033fc..298cc162f7 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,11 @@ PANDAS_REQUIREMENTS = [ "pandas<4.0.0", - "pyarrow", + # Mirrors the pyarrow bounds in snowflake-connector-python's [pandas] extra + # (its setup.cfg). We no longer request that extra, but the connector's Arrow + # code still needs this range and warns at import when it is not met. + "pyarrow>=14.0.1,<24; python_version >= '3.14'", + "pyarrow>=14.0.1; python_version < '3.14'", ] MODIN_REQUIREMENTS = [ *PANDAS_REQUIREMENTS,