dependencies: Allow Pandas 2.x (#2210)

* Allow Pandas 2.x

* Minor test fixes

* Update extension types mappings

* Revert "Update extension types mappings"

This reverts commit 6fbd99b.

* Fix tests

* Add warning for Pandas 2.x

* Improve wwarning text

Author: kukushking

awswrangler/pandas/__init__.py

@@ -1,6 +1,10 @@
 """Pandas "proxy" package."""
+import logging
 from typing import TYPE_CHECKING
 
+from packaging import version
+from pandas import __version__ as _PANDAS_VERSION
+
 from awswrangler._distributed import MemoryFormatEnum, memory_format
 
 if TYPE_CHECKING or memory_format.get() == MemoryFormatEnum.PANDAS:
@@ -38,6 +42,13 @@
 else:
     raise ImportError(f"Unknown memory format {memory_format}")
 
+_logger: logging.Logger = logging.getLogger(__name__)
+
+if version.parse(_PANDAS_VERSION) >= version.parse("2.0.0"):
+    _logger.warning(
+        "Pandas version 2.x was detected. Please note awswrangler currently does not support pyarrow-backed ArrowDtype "
+        "DataFrames."
+    )
 
 __all__ = [
     "DataFrame",

poetry.lock

@@ -30,7 +30,7 @@ python = ">=3.8, <4.0"
 # Required
 boto3 = "^1.20.32"
 botocore = "^1.23.32"
-pandas = "^1.2.0, !=1.5.0" # Exclusion per: https://github.com/aws/aws-sdk-pandas/issues/1678
+pandas = ">=1.2.0,!=1.5.0,<3.0.0" # Exclusion per: https://github.com/aws/aws-sdk-pandas/issues/1678
 numpy = "^1.18"
 pyarrow = ">=7.0.0"
 typing-extensions = "^4.4.0"

pyproject.toml

@@ -10,6 +10,7 @@
 
 import boto3
 import botocore.exceptions
+from packaging import version
 from pandas import DataFrame as PandasDataFrame
 from pandas import Series as PandasSeries
 from pytest import FixtureRequest
@@ -19,6 +20,7 @@
 from awswrangler._utils import try_it
 
 is_ray_modin = wr.engine.get() == EngineEnum.RAY and wr.memory_format.get() == MemoryFormatEnum.MODIN
+is_pandas_2_x = False
 
 if is_ray_modin:
     import modin.pandas as pd
@@ -27,6 +29,9 @@
 else:
     import pandas as pd
 
+    if version.parse(pd.__version__) >= version.parse("2.0.0"):
+        is_pandas_2_x = True
+
 
 CFN_VALID_STATUS = ["CREATE_COMPLETE", "ROLLBACK_COMPLETE", "UPDATE_COMPLETE", "UPDATE_ROLLBACK_COMPLETE"]
 

tests/_utils.py

@@ -1386,7 +1386,7 @@ def test_athena_date_recovery(path, glue_database, glue_table):
             "date3": [datetime.date(3099, 1, 3), datetime.date(3099, 1, 4), datetime.date(4080, 1, 5)],
         }
     )
-    df["date1"] = df["date1"].astype("datetime64")
+    df["date1"] = df["date1"].astype("datetime64[ns]")
     wr.s3.to_parquet(
         df=df,
         path=path,

tests/unit/test_athena.py

@@ -82,8 +82,14 @@ def test_recommendation_ruleset(df, path, name, glue_database, glue_table, glue_
         iam_role_arn=glue_data_quality_role,
         number_of_workers=2,
     )
-    df_rules = df_recommended_ruleset.append(
-        {"rule_type": "ColumnValues", "parameter": '"c2"', "expression": "in [0, 1, 2]"}, ignore_index=True
+    df_rules = pd.concat(
+        [
+            df_recommended_ruleset,
+            pd.DataFrame(
+                [{"rule_type": "ColumnValues", "parameter": '"c2"', "expression": "in [0, 1, 2]"}],
+            ),
+        ],
+        ignore_index=True,
     )
     wr.data_quality.create_ruleset(
         name=glue_ruleset,
@@ -179,9 +185,14 @@ def test_update_ruleset(df: pd.DataFrame, glue_database: str, glue_table: str, g
         table=glue_table,
         df_rules=df_rules,
     )
-
-    df_rules = df_rules.append(
-        {"rule_type": "ColumnValues", "parameter": '"c2"', "expression": "in [0, 1, 2]"}, ignore_index=True
+    df_rules = pd.concat(
+        [
+            df_rules,
+            pd.DataFrame(
+                [{"rule_type": "ColumnValues", "parameter": '"c2"', "expression": "in [0, 1, 2]"}],
+            ),
+        ],
+        ignore_index=True,
     )
 
     wr.data_quality.update_ruleset(name=glue_ruleset, df_rules=df_rules)

tests/unit/test_data_quality.py

@@ -27,9 +27,7 @@
 def test_csv_encoding(path, encoding, strings, wrong_encoding, exception, line_terminator, chunksize, use_threads):
     file_path = f"{path}0.csv"
     df = pd.DataFrame({"c0": [1, 2, 3], "c1": strings})
-    wr.s3.to_csv(
-        df, file_path, index=False, encoding=encoding, line_terminator=line_terminator, use_threads=use_threads
-    )
+    wr.s3.to_csv(df, file_path, index=False, encoding=encoding, lineterminator=line_terminator, use_threads=use_threads)
     df2 = wr.s3.read_csv(
         file_path, encoding=encoding, lineterminator=line_terminator, use_threads=use_threads, chunksize=chunksize
     )
@@ -347,11 +345,11 @@ def test_csv_additional_kwargs(path, kms_key_id, s3_additional_kwargs, use_threa
         assert desc.get("ServerSideEncryption") == "AES256"
 
 
-@pytest.mark.parametrize("line_terminator", ["\n", "\r", "\n\r"])
+@pytest.mark.parametrize("line_terminator", ["\n", "\r", "\r\n"])
 def test_csv_line_terminator(path, line_terminator):
     file_path = f"{path}0.csv"
     df = pd.DataFrame(data={"reading": ["col1", "col2"], "timestamp": [1601379427618, 1601379427625], "value": [1, 2]})
-    wr.s3.to_csv(df=df, path=file_path, index=False, line_terminator=line_terminator)
+    wr.s3.to_csv(df=df, path=file_path, index=False, lineterminator=line_terminator)
     df2 = wr.s3.read_csv(file_path)
     assert df.equals(df2)