diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 2524fa1..897fc80 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 8.5.0 +current_version = 8.6.1 commit = True tag = True tag_name = {new_version} diff --git a/AUTHORS.md b/AUTHORS.md index eabda43..ef0d5a7 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -75,3 +75,4 @@ Authors in order of the timeline of their contributions: - [dtorres-sf](https://github.com/dtorres-sf) for the fix for moving nested tables when using iterable_compare_func. - [Jim Cipar](https://github.com/jcipar) for the fix recursion depth limit when hashing numpy.datetime64 - [Enji Cooper](https://github.com/ngie-eign) for converting legacy setuptools use to pyproject.toml +- [Diogo Correia](https://github.com/diogotcorreia) for reporting security vulnerability in Delta and DeepDiff that could allow remote code execution. diff --git a/CHANGELOG.md b/CHANGELOG.md index 3935d83..605d83f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # DeepDiff Change log +- v8-6-1 + - Patched security vulnerability in the Delta class which was vulnerable to class pollution via its constructor, and when combined with a gadget available in DeltaDiff itself, it could lead to Denial of Service and Remote Code Execution (via insecure Pickle deserialization). + + - v8-6-0 - Added Colored View thanks to @mauvilsa - Added support for applying deltas to NamedTuple thanks to @paulsc diff --git a/CITATION.cff b/CITATION.cff index 4311a4f..9312277 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -5,6 +5,6 @@ authors: given-names: "Sep" orcid: "https://orcid.org/0009-0009-5828-4345" title: "DeepDiff" -version: 8.5.0 +version: 8.6.1 date-released: 2024 url: "https://github.com/seperman/deepdiff" diff --git a/README.md b/README.md index 9ef1f79..65215f4 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# DeepDiff v 8.5.0 +# DeepDiff v 8.6.1 ![Downloads](https://img.shields.io/pypi/dm/deepdiff.svg?style=flat) ![Python Versions](https://img.shields.io/pypi/pyversions/deepdiff.svg?style=flat) @@ -17,12 +17,15 @@ Tested on Python 3.9+ and PyPy3. -- **[Documentation](https://zepworks.com/deepdiff/8.5.0/)** +- **[Documentation](https://zepworks.com/deepdiff/8.6.1/)** ## What is new? Please check the [ChangeLog](CHANGELOG.md) file for the detailed information. +DeepDiff 8-6-1 +- Patched security vulnerability in the Delta class which was vulnerable to class pollution via its constructor, and when combined with a gadget available in DeltaDiff itself, it could lead to Denial of Service and Remote Code Execution (via insecure Pickle deserialization). + DeepDiff 8-6-0 - Added Colored View thanks to @mauvilsa diff --git a/deepdiff/__init__.py b/deepdiff/__init__.py index 6f0240a..ea6c5b9 100644 --- a/deepdiff/__init__.py +++ b/deepdiff/__init__.py @@ -1,6 +1,6 @@ """This module offers the DeepDiff, DeepSearch, grep, Delta and DeepHash classes.""" # flake8: noqa -__version__ = '8.5.0' +__version__ = '8.6.1' import logging if __name__ == '__main__': diff --git a/deepdiff/delta.py b/deepdiff/delta.py index 065f540..0774502 100644 --- a/deepdiff/delta.py +++ b/deepdiff/delta.py @@ -17,7 +17,7 @@ ) from deepdiff.path import ( _path_to_elements, _get_nested_obj, _get_nested_obj_and_force, - GET, GETATTR, parse_path, stringify_path, + GET, GETATTR, check_elem, parse_path, stringify_path, ) from deepdiff.anyset import AnySet from deepdiff.summarize import summarize @@ -237,6 +237,11 @@ def _get_elem_and_compare_to_old_value( forced_old_value=None, next_element=None, ): + try: + check_elem(elem) + except ValueError as error: + self._raise_or_log(UNABLE_TO_GET_ITEM_MSG.format(path_for_err_reporting, error)) + return not_found # if forced_old_value is not None: try: if action == GET: @@ -536,6 +541,7 @@ def _get_elements_and_details(self, path): obj = self # obj = self.get_nested_obj(obj=self, elements=elements[:-1]) elem, action = elements[-1] # type: ignore + check_elem(elem) except Exception as e: self._raise_or_log(UNABLE_TO_GET_ITEM_MSG.format(path, e)) return None diff --git a/deepdiff/helper.py b/deepdiff/helper.py index 1b01931..8960936 100644 --- a/deepdiff/helper.py +++ b/deepdiff/helper.py @@ -739,8 +739,8 @@ def detailed__dict__(obj: Any, ignore_private_variables: bool = True, ignore_key else: result = obj.__dict__.copy() # A shallow copy private_var_prefix = f"_{obj.__class__.__name__}__" # The semi private variables in Python get this prefix - for key in ignore_keys: - if key in result or ( + for key in obj.__dict__: + if key in ignore_keys or ( ignore_private_variables and key.startswith('__') and not key.startswith(private_var_prefix) ): del result[key] diff --git a/deepdiff/path.py b/deepdiff/path.py index ee63b5b..e5b64c7 100644 --- a/deepdiff/path.py +++ b/deepdiff/path.py @@ -117,6 +117,7 @@ def _path_to_elements(path, root_element=DEFAULT_FIRST_ELEMENT): def _get_nested_obj(obj, elements, next_element=None): for (elem, action) in elements: + check_elem(elem) if action == GET: obj = obj[elem] elif action == GETATTR: @@ -134,11 +135,17 @@ def _guess_type(elements, elem, index, next_element): return {} +def check_elem(elem): + if isinstance(elem, str) and elem.startswith("__") and elem.endswith("__"): + raise ValueError("traversing dunder attributes is not allowed") + + def _get_nested_obj_and_force(obj, elements, next_element=None): prev_elem = None prev_action = None prev_obj = obj for index, (elem, action) in enumerate(elements): + check_elem(elem) _prev_obj = obj if action == GET: try: diff --git a/deepdiff/serialization.py b/deepdiff/serialization.py index ed48959..b773413 100644 --- a/deepdiff/serialization.py +++ b/deepdiff/serialization.py @@ -59,7 +59,7 @@ class UnsupportedFormatErr(TypeError): DELTA_IGNORE_ORDER_NEEDS_REPETITION_REPORT = 'report_repetition must be set to True when ignore_order is True to create the delta object.' DELTA_ERROR_WHEN_GROUP_BY = 'Delta can not be made when group_by is used since the structure of data is modified from the original form.' -SAFE_TO_IMPORT = { +SAFE_TO_IMPORT = frozenset({ 'builtins.range', 'builtins.complex', 'builtins.set', @@ -95,7 +95,7 @@ class UnsupportedFormatErr(TypeError): 'ipaddress.IPv4Address', 'ipaddress.IPv6Address', 'collections.abc.KeysView', -} +}) TYPE_STR_TO_TYPE = { diff --git a/docs/authors.rst b/docs/authors.rst index 9353130..9a70f9f 100644 --- a/docs/authors.rst +++ b/docs/authors.rst @@ -117,6 +117,7 @@ and polars support. limit when hashing numpy.datetime64 - `Enji Cooper `__ for converting legacy setuptools use to pyproject.toml +- `Diogo Correia `__ for reporting security vulnerability in Delta and DeepDiff that could allow remote code execution. .. _Sep Dehpour (Seperman): http://www.zepworks.com diff --git a/docs/changelog.rst b/docs/changelog.rst index 0197362..ce0b7d4 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,9 @@ Changelog DeepDiff Changelog +- v8-6-1 + - Patched security vulnerability in the Delta class which was vulnerable to class pollution via its constructor, and when combined with a gadget available in DeltaDiff itself, it could lead to Denial of Service and Remote Code Execution (via insecure Pickle deserialization). + - v8-6-0 - Added Colored View thanks to @mauvilsa - Added support for applying deltas to NamedTuple thanks to @paulsc diff --git a/docs/conf.py b/docs/conf.py index ae6dcec..0999906 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -64,9 +64,9 @@ # built documents. # # The short X.Y version. -version = '8.5.0' +version = '8.6.1' # The full version, including alpha/beta/rc tags. -release = '8.5.0' +release = '8.6.1' load_dotenv(override=True) DOC_VERSION = os.environ.get('DOC_VERSION', version) diff --git a/docs/index.rst b/docs/index.rst index 9b0e68d..135db1a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -4,7 +4,7 @@ contain the root `toctree` directive. -DeepDiff 8.5.0 documentation! +DeepDiff 8.6.1 documentation! ============================= ******* @@ -31,6 +31,40 @@ The DeepDiff library includes the following modules: What Is New *********** +DeepDiff 8-6-1 +-------------- + + - Patched security vulnerability in the Delta class which was vulnerable to class pollution via its constructor, and when combined with a gadget available in DeltaDiff itself, it could lead to Denial of Service and Remote Code Execution (via insecure Pickle deserialization). + + +DeepDiff 8-6-0 +-------------- + + - Added Colored View thanks to @mauvilsa + - Added support for applying deltas to NamedTuple thanks to @paulsc + - Fixed test_delta.py with Python 3.14 thanks to @Romain-Geissler-1A + - Added python property serialization to json + - Added ip address serialization + - Switched to UV from pip + - Added Claude.md + - Added uuid hashing thanks to @akshat62 + - Added ``ignore_uuid_types`` flag to DeepDiff to avoid type reports + when comparing UUID and string. + - Added comprehensive type hints across the codebase (multiple commits + for better type safety) + - Added support for memoryview serialization + - Added support for bytes serialization (non-UTF8 compatible) + - Fixed bug where group_by with numbers would leak type info into group + path reports + - Fixed bug in ``_get_clean_to_keys_mapping without`` explicit + significant digits + - Added support for python dict key serialization + - Enhanced support for IP address serialization with safe module imports + - Added development tooling improvements (pyright config, .envrc + example) + - Updated documentation and development instructions + + DeepDiff 8-5-0 -------------- diff --git a/pyproject.toml b/pyproject.toml index dff03fa..4a906b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "flit_core.buildapi" [project] name = "deepdiff" -version = "8.5.0" +version = "8.6.1" dependencies = [ "orderly-set>=5.4.1,<6", ] diff --git a/tests/test_security.py b/tests/test_security.py new file mode 100644 index 0000000..e221018 --- /dev/null +++ b/tests/test_security.py @@ -0,0 +1,133 @@ +import os +import pickle +import pytest +from deepdiff import Delta +from deepdiff.helper import Opcode +from deepdiff.serialization import ForbiddenModule + + +class TestDeltaClassPollution: + + def test_builtins_int(self): + + pollute_int = pickle.dumps( + { + "values_changed": {"root['tmp']": {"new_value": Opcode("", 0, 0, 0, 0)}}, + "dictionary_item_added": { + ( + ("root", "GETATTR"), + ("tmp", "GET"), + ("__repr__", "GETATTR"), + ("__globals__", "GETATTR"), + ("__builtins__", "GET"), + ("int", "GET"), + ): "no longer a class" + }, + } + ) + + assert isinstance(pollute_int, bytes) + + # ------------[ Exploit ]------------ + # This could be some example, vulnerable, application. + # The inputs above could be sent via HTTP, for example. + + + # Existing dictionary; it is assumed that it contains + # at least one entry, otherwise a different Delta needs to be + # applied first, adding an entry to the dictionary. + mydict = {"tmp": "foobar"} + + # Before pollution + assert 42 == int("41") + 1 + + # Apply Delta to mydict + result = mydict + Delta(pollute_int) + + assert 1337 == int("1337") + + def test_remote_code_execution(self): + if os.path.exists('/tmp/pwned'): + os.remove('/tmp/pwned') + + pollute_safe_to_import = pickle.dumps( + { + "values_changed": {"root['tmp']": {"new_value": Opcode("", 0, 0, 0, 0)}}, + "set_item_added": { + ( + ("root", "GETATTR"), + ("tmp", "GET"), + ("__repr__", "GETATTR"), + ("__globals__", "GETATTR"), + ("sys", "GET"), + ("modules", "GETATTR"), + ("deepdiff.serialization", "GET"), + ("SAFE_TO_IMPORT", "GETATTR"), + ): set(["posix.system"]) + }, + } + ) + + # From https://davidhamann.de/2020/04/05/exploiting-python-pickle/ + class RCE: + def __reduce__(self): + cmd = "id > /tmp/pwned" + return os.system, (cmd,) + + # Wrap object with dictionary so that Delta does not crash + rce_pickle = pickle.dumps({"_": RCE()}) + + assert isinstance(pollute_safe_to_import, bytes) + assert isinstance(rce_pickle, bytes) + + # ------------[ Exploit ]------------ + # This could be some example, vulnerable, application. + # The inputs above could be sent via HTTP, for example. + + # Existing dictionary; it is assumed that it contains + # at least one entry, otherwise a different Delta needs to be + # applied first, adding an entry to the dictionary. + mydict = {"tmp": "foobar"} + + # Apply Delta to mydict + with pytest.raises(ValueError) as exc_info: + mydict + Delta(pollute_safe_to_import) + assert "traversing dunder attributes is not allowed" == str(exc_info.value) + + with pytest.raises(ForbiddenModule) as exc_info: + Delta(rce_pickle) # no need to apply this Delta + assert "Module 'posix.system' is forbidden. You need to explicitly pass it by passing a safe_to_import parameter" == str(exc_info.value) + + assert not os.path.exists('/tmp/pwned'), "We should not have created this file" + + def test_delta_should_not_access_globals(self): + + pollute_global = pickle.dumps( + { + "dictionary_item_added": { + ( + ("root", "GETATTR"), + ("myfunc", "GETATTR"), + ("__globals__", "GETATTR"), + ("PWNED", "GET"), + ): 1337 + } + } + ) + + + # demo application + class Foo: + def __init__(self): + pass + + def myfunc(self): + pass + + + PWNED = False + delta = Delta(pollute_global) + assert PWNED is False + b = Foo() + delta + + assert PWNED is False diff --git a/uv.lock b/uv.lock index adac038..50beb1d 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.9" resolution-markers = [ "python_full_version >= '3.12'", @@ -273,7 +273,7 @@ wheels = [ [[package]] name = "deepdiff" -version = "8.5.0" +version = "8.6.1" source = { editable = "." } dependencies = [ { name = "orderly-set" },