From 18fec3a92cec447c32ac62b4e570c8037c0dbf77 Mon Sep 17 00:00:00 2001 From: LmeSzinc <37934724+LmeSzinc@users.noreply.github.com> Date: Wed, 6 May 2026 05:21:25 +0800 Subject: [PATCH 1/6] Fix: minor import syntax mistake --- tests/test_speed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_speed.py b/tests/test_speed.py index 3a303b9d0..303d8590d 100644 --- a/tests/test_speed.py +++ b/tests/test_speed.py @@ -1,7 +1,7 @@ # Tests from tests.base import BaseTestZSTD,tDATA,log,zstd,platform,raise_skip -from.get_memory_usage import get_real_memory_usage +from tests.get_memory_usage import get_real_memory_usage from time import time import os import platform From edc79b0d7cb97fdba7dc30f987144e33209ec418 Mon Sep 17 00:00:00 2001 From: LmeSzinc <37934724+LmeSzinc@users.noreply.github.com> Date: Wed, 6 May 2026 07:44:41 +0800 Subject: [PATCH 2/6] Fix: update get-pip url since python3.9 is EOL --- .github/workflows/Build_wheels_for_cpython39_x86.yml | 2 +- .../workflows/Build_wheels_for_cpython39_x86_64_u24.yml | 2 +- .github/workflows/Build_wheels_for_pypy39_x64_u16.yml | 2 +- .github/workflows/debug_build-wheels.yml | 8 ++++---- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/Build_wheels_for_cpython39_x86.yml b/.github/workflows/Build_wheels_for_cpython39_x86.yml index f2ae642cc..e356c7076 100644 --- a/.github/workflows/Build_wheels_for_cpython39_x86.yml +++ b/.github/workflows/Build_wheels_for_cpython39_x86.yml @@ -23,7 +23,7 @@ jobs: mirror: http://azure.archive.ubuntu.com/ubuntu #version: 1.5.6.7 #pyver: "3.9" - getpipurl: https://bootstrap.pypa.io/pip/get-pip.py + getpipurl: https://bootstrap.pypa.io/pip/3.9/get-pip.py pypkg: python3.9 pypkgadd: python3.9-distutils pyengine_tag: cp39-cp39 diff --git a/.github/workflows/Build_wheels_for_cpython39_x86_64_u24.yml b/.github/workflows/Build_wheels_for_cpython39_x86_64_u24.yml index 7be8b8df3..6e4ca2c29 100644 --- a/.github/workflows/Build_wheels_for_cpython39_x86_64_u24.yml +++ b/.github/workflows/Build_wheels_for_cpython39_x86_64_u24.yml @@ -23,7 +23,7 @@ jobs: mirror: http://azure.archive.ubuntu.com/ubuntu #version: 1.5.6.7 # pyver: "3.7" - getpipurl: https://bootstrap.pypa.io/pip/get-pip.py + getpipurl: https://bootstrap.pypa.io/pip/3.9/get-pip.py pypkg: python3.9 pypkgadd: python3.9-distutils pyengine_tag: cp39-cp39 diff --git a/.github/workflows/Build_wheels_for_pypy39_x64_u16.yml b/.github/workflows/Build_wheels_for_pypy39_x64_u16.yml index cce57a9d4..9dfd4da89 100644 --- a/.github/workflows/Build_wheels_for_pypy39_x64_u16.yml +++ b/.github/workflows/Build_wheels_for_pypy39_x64_u16.yml @@ -21,7 +21,7 @@ jobs: tag_arch: x86_64 release: xenial mirror: http://azure.archive.ubuntu.com/ubuntu - getpipurl: https://bootstrap.pypa.io/pip/get-pip.py + getpipurl: https://bootstrap.pypa.io/pip/3.9/get-pip.py pypkg: pypy3.9 pypkgdev: pypy3.9-bin-dev pyengine_tag: pp39-pypy39_pp73 diff --git a/.github/workflows/debug_build-wheels.yml b/.github/workflows/debug_build-wheels.yml index acd27e04f..b64971f59 100644 --- a/.github/workflows/debug_build-wheels.yml +++ b/.github/workflows/debug_build-wheels.yml @@ -90,7 +90,7 @@ jobs: mirror: http://azure.archive.ubuntu.com/ubuntu version: 1.5.6.3 pyver: "3.8" - getpipurl: https://bootstrap.pypa.io/pip/get-pip.py + getpipurl: https://bootstrap.pypa.io/pip/3.8/get-pip.py pypkg: python3.8 pypkgadd: python3.8-distutils pyengine_tag: cp38-cp38 @@ -104,7 +104,7 @@ jobs: mirror: http://azure.archive.ubuntu.com/ubuntu version: 1.5.6.3 pyver: "3.9" - getpipurl: https://bootstrap.pypa.io/pip/get-pip.py + getpipurl: https://bootstrap.pypa.io/pip/3.9/get-pip.py pypkg: python3.9 pypkgadd: python3.9-distutils pyengine_tag: cp39-cp39 @@ -215,7 +215,7 @@ jobs: pypkg: python3.8 pypkgadd: python3.8-distutils pyver: "3.8" - getpipurl: https://bootstrap.pypa.io/pip/get-pip.py + getpipurl: https://bootstrap.pypa.io/pip/3.8/get-pip.py pyengine_tag: cp38-cp38 libc_tag: manylinux_2_17 - name: Ubuntu 24+16 amd64 CPython3.9 @@ -229,7 +229,7 @@ jobs: pypkg: python3.9 pypkgadd: python3.9-distutils pyver: "3.9" - getpipurl: https://bootstrap.pypa.io/pip/get-pip.py + getpipurl: https://bootstrap.pypa.io/pip/3.9/get-pip.py pyengine_tag: cp39-cp39 libc_tag: manylinux_2_17 - name: Ubuntu 24+20 amd64 CPython3.10 From 6d9b29dd1d067de3b4ca5af2cda180ef2eecabd0 Mon Sep 17 00:00:00 2001 From: LmeSzinc <37934724+LmeSzinc@users.noreply.github.com> Date: Wed, 6 May 2026 07:45:31 +0800 Subject: [PATCH 3/6] Fix: trying to fix byte resizing --- src/python-zstd.c | 25 +++++++++++---------- tests/test_resize_and_formatting.py | 34 +++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 11 deletions(-) create mode 100644 tests/test_resize_and_formatting.py diff --git a/src/python-zstd.c b/src/python-zstd.c index 0c0c73247..a18eda3cc 100644 --- a/src/python-zstd.c +++ b/src/python-zstd.c @@ -155,7 +155,9 @@ static PyObject *py_zstd_compress_mt(PyObject* self, PyObject *args) Py_CLEAR(result); return NULL; } - Py_SET_SIZE(result, cSize); + if (cSize < (size_t)dest_size) { + _PyBytes_Resize(&result, (Py_ssize_t)cSize); + } } return result; } @@ -284,7 +286,9 @@ static PyObject *py_zstd_compress_mt2(PyObject* self, PyObject *args) Py_CLEAR(result); return NULL; } - Py_SET_SIZE(result, cSize); + if (cSize < (size_t)dest_size) { + _PyBytes_Resize(&result, (Py_ssize_t)cSize); + } } return result; } @@ -381,14 +385,11 @@ static PyObject *py_zstd_uncompress(PyObject* self, PyObject *args) PyErr_Format(ZstdError, "Decompression error: %s", errStr); error = 1; // } - } else if (cSize != dest_size) { - //if (sizeof(uint64_t)==sizeof(unsigned long)) { - PyErr_Format(ZstdError, "Decompression error: length mismatch -> decomp %lu != %lu [header]", (unsigned long)cSize, (unsigned long)dest_size); - //} - //else if (sizeof(uint64_t)==sizeof(unsigned long long)) - //{ //unsigned long long?! x86?! - // PyErr_Format(ZstdError, "Decompression error: length mismatch -> decomp %llu != %llu [header]", (uint64_t)cSize, (uint64_t)dest_size); - //} + } else if (cSize != (size_t)dest_size) { + /* Using %llu is safe for Python 3.3+ on all platforms (Win/Linux/32/64) + as it is handled by Python's own internal formatter. */ + PyErr_Format(ZstdError, "Decompression error: length mismatch -> decomp %llu != %llu [header]", + (unsigned long long)cSize, (unsigned long long)dest_size); error = 1; } } else { @@ -401,7 +402,9 @@ static PyObject *py_zstd_uncompress(PyObject* self, PyObject *args) } if (!error) { - Py_SET_SIZE(result, cSize); + if (cSize < dest_size) { + _PyBytes_Resize(&result, (Py_ssize_t)cSize); + } } return result; diff --git a/tests/test_resize_and_formatting.py b/tests/test_resize_and_formatting.py new file mode 100644 index 000000000..e40a68bbf --- /dev/null +++ b/tests/test_resize_and_formatting.py @@ -0,0 +1,34 @@ +import unittest + +import zstd + + +class TestFixes(unittest.TestCase): + def test_resize_memory_integrity(self): + # Construct data with high compression ratio to trigger buffer resizing logic. + # In this case, cSize will be significantly smaller than initial dest_size. + data = b"0" * 1024 * 1024 # 1MB of repetitive data + compressed = zstd.compress(data) + + # Verify if decompression results in identical data. + # This checks if _PyBytes_Resize preserved data integrity and null-termination. + decompressed = zstd.decompress(compressed) + self.assertEqual(data, decompressed) + + # Verify the bytes object properties in Python. + self.assertEqual(len(decompressed), 1024 * 1024) + + # Append data to check if the internal buffer is correctly null-terminated. + test_str = decompressed + b"check" + self.assertTrue(test_str.endswith(b"check")) + + def test_exception_path_safety(self): + # Verify that the exception handling path is safe and doesn't crash the interpreter. + # Providing invalid data will trigger the error reporting logic in C. + invalid_data = b"NOT_A_ZSTD_FRAME_DATA_BUT_LONG_ENOUGH_FOR_VALIDATION_FAILURE" + with self.assertRaises(zstd.Error) as cm: + zstd.decompress(invalid_data) + + # The specific message might vary depending on where zstd library stops validation, + # but it must be a valid zstd.Error instance. + self.assertTrue(len(str(cm.exception)) > 0) From 1d6de14a18790d8aa3d1f9a1fbd4f4731a3e3cfe Mon Sep 17 00:00:00 2001 From: LmeSzinc <37934724+LmeSzinc@users.noreply.github.com> Date: Wed, 6 May 2026 08:22:31 +0800 Subject: [PATCH 4/6] Fix: byte resizing on 32bit --- src/python-zstd.c | 125 ++++++++++++++++++++++++---------------------- 1 file changed, 66 insertions(+), 59 deletions(-) diff --git a/src/python-zstd.c b/src/python-zstd.c index a18eda3cc..21d986197 100644 --- a/src/python-zstd.c +++ b/src/python-zstd.c @@ -156,7 +156,9 @@ static PyObject *py_zstd_compress_mt(PyObject* self, PyObject *args) return NULL; } if (cSize < (size_t)dest_size) { - _PyBytes_Resize(&result, (Py_ssize_t)cSize); + if (_PyBytes_Resize(&result, (Py_ssize_t)cSize) < 0) { + return NULL; + } } } return result; @@ -287,7 +289,9 @@ static PyObject *py_zstd_compress_mt2(PyObject* self, PyObject *args) return NULL; } if (cSize < (size_t)dest_size) { - _PyBytes_Resize(&result, (Py_ssize_t)cSize); + if (_PyBytes_Resize(&result, (Py_ssize_t)cSize) < 0) { + return NULL; + } } } return result; @@ -305,10 +309,10 @@ static PyObject *py_zstd_uncompress(PyObject* self, PyObject *args) PyObject *result; const char *source, *src; - Py_ssize_t source_size, ss, seek_frame; - uint64_t dest_size, frame_size; + Py_ssize_t source_size; + uint64_t dest_size, frame_size, allocated_size; char error = 0, streamed = 0; - size_t cSize = 0, processed = 0; + size_t total_output = 0; #if PY_MAJOR_VERSION >= 3 if (!PyArg_ParseTuple(args, "y#", &source, &source_size)) @@ -330,50 +334,45 @@ static PyObject *py_zstd_uncompress(PyObject* self, PyObject *args) // known block // Find real dest_size across multiple frames - ss = source_size; - seek_frame = ss - 1; + size_t ss = (size_t)source_size; + size_t seek_frame; src = source; - while (seek_frame < ss) { - seek_frame = ZSTD_findFrameCompressedSize(src, ss); - if (ZSTD_isError(seek_frame)) break; + seek_frame = ZSTD_findFrameCompressedSize(src, ss); + while (!ZSTD_isError(seek_frame) && seek_frame <= ss) { src += seek_frame; ss -= seek_frame; - if (ss <=0) break; + if (ss == 0) break; frame_size = (uint64_t) ZSTD_getFrameContentSize(src, ss); if (ZSTD_isError(frame_size)) break; dest_size += frame_size; + seek_frame = ZSTD_findFrameCompressedSize(src, ss); } } - result = PyBytes_FromStringAndSize(NULL, dest_size); + + allocated_size = dest_size; + result = PyBytes_FromStringAndSize(NULL, (Py_ssize_t)allocated_size); if (result != NULL) { char *dest = PyBytes_AS_STRING(result); + size_t cSize = 0; Py_BEGIN_ALLOW_THREADS - if (streamed) { - ZSTD_DStream* zds; - zds = ZSTD_createDStream(); - // buffers create and decompress - ZSTD_initDStream(zds); - ZSTD_outBuffer out; - ZSTD_inBuffer in; - in.src = source; - in.pos = 0; - in.size = source_size; - out.dst = dest; - out.pos = 0; - out.size = dest_size; - processed = ZSTD_decompressStream(zds, &out, &in); - if (processed==0) { - cSize=out.pos; - if (cSize) dest_size=cSize; - } - ZSTD_freeDStream(zds); - } - else { - cSize = ZSTD_decompress(dest, dest_size, source, source_size); - } - + if (streamed) { + ZSTD_DStream* zds = ZSTD_createDStream(); + ZSTD_initDStream(zds); + ZSTD_outBuffer out = { dest, (size_t)allocated_size, 0 }; + ZSTD_inBuffer in = { source, (size_t)source_size, 0 }; + + size_t processed = ZSTD_decompressStream(zds, &out, &in); + if (ZSTD_isError(processed)) { + cSize = processed; + } else { + cSize = out.pos; + } + ZSTD_freeDStream(zds); + } else { + cSize = ZSTD_decompress(dest, (size_t)allocated_size, source, (size_t)source_size); + } Py_END_ALLOW_THREADS if (ZSTD_isError(cSize)) { @@ -385,27 +384,30 @@ static PyObject *py_zstd_uncompress(PyObject* self, PyObject *args) PyErr_Format(ZstdError, "Decompression error: %s", errStr); error = 1; // } - } else if (cSize != (size_t)dest_size) { - /* Using %llu is safe for Python 3.3+ on all platforms (Win/Linux/32/64) - as it is handled by Python's own internal formatter. */ - PyErr_Format(ZstdError, "Decompression error: length mismatch -> decomp %llu != %llu [header]", - (unsigned long long)cSize, (unsigned long long)dest_size); - error = 1; + } else { + total_output = cSize; + // Using %llu is safe for Python 3.3+ on all platforms (Win/Linux/32/64) + // as it is handled by Python's own internal formatter. + if (!streamed && total_output != (size_t)dest_size) { + PyErr_Format(ZstdError, "Decompression error: length mismatch -> decomp %llu != %llu [header]", + (unsigned long long)total_output, (unsigned long long)dest_size); + error = 1; + } } } else { - error = 1; + error = 1; } if (error) { Py_CLEAR(result); - result = NULL; + return NULL; } - if (!error) { - if (cSize < dest_size) { - _PyBytes_Resize(&result, (Py_ssize_t)cSize); + if (total_output < (size_t)allocated_size) { + if (_PyBytes_Resize(&result, (Py_ssize_t)total_output) < 0) { + return NULL; } - } + } return result; } @@ -780,19 +782,28 @@ static int init_py_zstd(PyObject *module) { #if PY_MAJOR_VERSION >= 3 static int myextension_traverse(PyObject *m, visitproc visit, void *arg) { - Py_VISIT(GETSTATE(m)->error); + struct module_state *state = GETSTATE(m); + if (state != NULL) { + Py_VISIT(state->error); + } printdi("ZSTD module->traverse\n",0); return 0; } static int myextension_clear(PyObject *self) { - Py_CLEAR(GETSTATE(self)->error); + struct module_state *state = GETSTATE(self); + if (state != NULL) { + Py_CLEAR(state->error); + } printdi("ZSTD module->clear\n",0); return 0; } static void myextension_free(void *self) { - Py_CLEAR(GETSTATE((PyObject *)self)->error); + struct module_state *state = GETSTATE((PyObject *)self); + if (state != NULL) { + Py_CLEAR(state->error); + } free_cContext(); printdi("ZSTD module->free\n",0); return; @@ -825,16 +836,12 @@ static struct PyModuleDef moduledef = { myextension_clear, // NULL myextension_free, -}; - - - -PyObject *PyInit_zstd(void) +#endif +#if PY_MAJOR_VERSION >= 3 +PyMODINIT_FUNC PyInit_zstd(void) #else - -void initzstd(void) - +PyMODINIT_FUNC initzstd(void) #endif { #if PY_MAJOR_VERSION >= 3 From 32cc418f5122ccc5a0eedbdc114817cce993f2fd Mon Sep 17 00:00:00 2001 From: LmeSzinc <37934724+LmeSzinc@users.noreply.github.com> Date: Wed, 6 May 2026 13:18:06 +0800 Subject: [PATCH 5/6] Fix: unclosed moduledef --- src/python-zstd.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/python-zstd.c b/src/python-zstd.c index 21d986197..828e546c7 100644 --- a/src/python-zstd.c +++ b/src/python-zstd.c @@ -836,6 +836,7 @@ static struct PyModuleDef moduledef = { myextension_clear, // NULL myextension_free, +}; #endif #if PY_MAJOR_VERSION >= 3 From 4d158a2622a052bb348cf019616e5570de8b536a Mon Sep 17 00:00:00 2001 From: LmeSzinc <37934724+LmeSzinc@users.noreply.github.com> Date: Wed, 6 May 2026 16:40:09 +0800 Subject: [PATCH 6/6] Fix: update version in debug_build-wheels.yml --- .github/workflows/debug_build-wheels.yml | 36 ++++++++++++------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/debug_build-wheels.yml b/.github/workflows/debug_build-wheels.yml index b64971f59..0a774da8c 100644 --- a/.github/workflows/debug_build-wheels.yml +++ b/.github/workflows/debug_build-wheels.yml @@ -21,7 +21,7 @@ jobs: tag_arch: i686 release: xenial mirror: http://azure.archive.ubuntu.com/ubuntu - version: 1.5.6.3 + version: 1.5.7.3 pyver: "2.7" getpipurl: https://bootstrap.pypa.io/pip/2.7/get-pip.py pypkg: python2.7 @@ -34,7 +34,7 @@ jobs: tag_arch: i686 release: xenial mirror: http://azure.archive.ubuntu.com/ubuntu - version: 1.5.6.3 + version: 1.5.7.3 pyver: "3.4" getpipurl: https://bootstrap.pypa.io/pip/3.4/get-pip.py pypkg: python3.4 @@ -47,7 +47,7 @@ jobs: tag_arch: i686 release: xenial mirror: http://azure.archive.ubuntu.com/ubuntu - version: 1.5.6.3 + version: 1.5.7.3 pyver: "3.5" getpipurl: https://bootstrap.pypa.io/pip/3.5/get-pip.py pypkg: python3.5 @@ -60,7 +60,7 @@ jobs: tag_arch: i686 release: xenial mirror: http://azure.archive.ubuntu.com/ubuntu - version: 1.5.6.3 + version: 1.5.7.3 pyver: "3.6" getpipurl: https://bootstrap.pypa.io/pip/3.6/get-pip.py pypkg: python3.6 @@ -74,7 +74,7 @@ jobs: tag_arch: i686 release: xenial mirror: http://azure.archive.ubuntu.com/ubuntu - version: 1.5.6.3 + version: 1.5.7.3 pyver: "3.7" getpipurl: https://bootstrap.pypa.io/pip/3.7/get-pip.py pypkg: python3.7 @@ -88,7 +88,7 @@ jobs: tag_arch: i686 release: xenial mirror: http://azure.archive.ubuntu.com/ubuntu - version: 1.5.6.3 + version: 1.5.7.3 pyver: "3.8" getpipurl: https://bootstrap.pypa.io/pip/3.8/get-pip.py pypkg: python3.8 @@ -102,7 +102,7 @@ jobs: tag_arch: i686 release: xenial mirror: http://azure.archive.ubuntu.com/ubuntu - version: 1.5.6.3 + version: 1.5.7.3 pyver: "3.9" getpipurl: https://bootstrap.pypa.io/pip/3.9/get-pip.py pypkg: python3.9 @@ -116,7 +116,7 @@ jobs: tag_arch: i686 release: focal mirror: http://azure.archive.ubuntu.com/ubuntu - version: 1.5.6.3 + version: 1.5.7.3 pyver: "3.10" getpipurl: https://bootstrap.pypa.io/pip/get-pip.py pypkg: python3.10 @@ -130,7 +130,7 @@ jobs: tag_arch: i686 release: focal mirror: http://azure.archive.ubuntu.com/ubuntu - version: 1.5.6.3 + version: 1.5.7.3 pyver: "3.11" getpipurl: https://bootstrap.pypa.io/pip/get-pip.py pypkg: python3.11 @@ -144,7 +144,7 @@ jobs: tag_arch: x86_64 release: xenial mirror: http://azure.archive.ubuntu.com/ubuntu - version: 1.5.6.3 + version: 1.5.7.3 pypkg: python2.7 pyver: "2.7" getpipurl: https://bootstrap.pypa.io/pip/2.7/get-pip.py @@ -157,7 +157,7 @@ jobs: tag_arch: x86_64 release: xenial mirror: http://azure.archive.ubuntu.com/ubuntu - version: 1.5.6.3 + version: 1.5.7.3 pypkg: python3.4 pyver: "3.4" getpipurl: https://bootstrap.pypa.io/pip/3.4/get-pip.py @@ -170,7 +170,7 @@ jobs: tag_arch: x86_64 release: xenial mirror: http://azure.archive.ubuntu.com/ubuntu - version: 1.5.6.3 + version: 1.5.7.3 pypkg: python3.5 pyver: "3.5" getpipurl: https://bootstrap.pypa.io/pip/3.5/get-pip.py @@ -183,7 +183,7 @@ jobs: tag_arch: x86_64 release: xenial mirror: http://azure.archive.ubuntu.com/ubuntu - version: 1.5.6.3 + version: 1.5.7.3 pypkg: python3.6 pypkgadd: python3.6-distutils pyver: "3.6" @@ -197,7 +197,7 @@ jobs: tag_arch: x86_64 release: xenial mirror: http://azure.archive.ubuntu.com/ubuntu - version: 1.5.6.3 + version: 1.5.7.3 pypkg: python3.7 pypkgadd: python3.7-distutils pyver: "3.7" @@ -211,7 +211,7 @@ jobs: tag_arch: x86_64 release: xenial mirror: http://azure.archive.ubuntu.com/ubuntu - version: 1.5.6.3 + version: 1.5.7.3 pypkg: python3.8 pypkgadd: python3.8-distutils pyver: "3.8" @@ -225,7 +225,7 @@ jobs: tag_arch: x86_64 release: xenial mirror: http://azure.archive.ubuntu.com/ubuntu - version: 1.5.6.3 + version: 1.5.7.3 pypkg: python3.9 pypkgadd: python3.9-distutils pyver: "3.9" @@ -239,7 +239,7 @@ jobs: tag_arch: x86_64 release: focal mirror: http://azure.archive.ubuntu.com/ubuntu - version: 1.5.6.3 + version: 1.5.7.3 pypkg: python3.10 pypkgadd: python3.10-distutils pyver: "3.10" @@ -253,7 +253,7 @@ jobs: tag_arch: x86_64 release: focal mirror: http://azure.archive.ubuntu.com/ubuntu - version: 1.5.6.3 + version: 1.5.7.3 pypkg: python3.11 pypkgadd: python3.11-distutils pyver: "3.11"