diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b99a2a..a7a01fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ All notable changes to this project will be documented in this file. - Upgrade zstd source code from v1.5.6 to [v1.5.7](https://github.com/facebook/zstd/releases/tag/v1.5.7) - Raise an exception when attempting to decompress empty data - Add `ZstdFile.name` property +- Deprecate `(de)compress_stream` functions - Build wheels for Windows ARM64 - Support for PyPy 3.11 diff --git a/docs/deprecated.rst b/docs/deprecated.rst new file mode 100644 index 0000000..9bc1cc8 --- /dev/null +++ b/docs/deprecated.rst @@ -0,0 +1,44 @@ +.. title:: pyzstd module: deprecations + +:py:func:`compress_stream` +-------------------------- + +.. sourcecode:: python + + # before + with io.open(input_file_path, 'rb') as ifh: + with io.open(output_file_path, 'wb') as ofh: + compress_stream(ifh, ofh, level_or_option=5) + + # after + with io.open(input_file_path, 'rb') as ifh: + with pyzstd.open(output_file_path, 'w', level_or_option=5) as ofh: + shutil.copyfileobj(ifh, ofh) + +.. hint:: + Instead of the ``read_size`` and ``write_size`` parameters, you can use + :py:func:`shutil.copyfileobj`'s ``length`` parameter. + +*Deprecated in version 0.17.0.* + + +:py:func:`decompress_stream` +-------------------------- + +.. sourcecode:: python + + # before + with io.open(input_file_path, 'rb') as ifh: + with io.open(output_file_path, 'wb') as ofh: + decompress_stream(ifh, ofh) + + # after + with pyzstd.open(input_file_path) as ifh: + with io.open(output_file_path, 'wb') as ofh: + shutil.copyfileobj(ifh, ofh) + +.. hint:: + Instead of the ``read_size`` and ``write_size`` parameters, you can use + :py:func:`shutil.copyfileobj`'s ``length`` parameter. + +*Deprecated in version 0.17.0.* diff --git a/docs/index.rst b/docs/index.rst index bda7a2b..40be86b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -154,79 +154,12 @@ Rich memory compression Streaming compression --------------------- - This section contains: + You can use :py:class:`ZstdFile` for compressing data as needed. Advanced users may be interested in: - * function :py:func:`compress_stream`, a fast and convenient function. * class :py:class:`ZstdCompressor`, similar to compressors in Python standard library. It would be nice to know some knowledge about zstd data, see :ref:`frame and block`. -.. py:function:: compress_stream(input_stream, output_stream, *, level_or_option=None, zstd_dict=None, pledged_input_size=None, read_size=131_072, write_size=131_591, callback=None) - - A fast and convenient function, compresses *input_stream* and writes the compressed data to *output_stream*, it doesn't close the streams. - - If input stream is ``b''``, nothing will be written to output stream. - - This function tries to zero-copy as much as possible. If the OS has read prefetching and write buffer, it may perform the tasks (read/compress/write) in parallel to some degree. - - The default values of *read_size* and *write_size* parameters are the buffer sizes recommended by zstd, increasing them may be faster, and reduces the number of callback function calls. - - .. versionadded:: 0.14.2 - - :param input_stream: Input stream that has a `.readinto(b) `_ method. - :param output_stream: Output stream that has a `.write(b) `_ method. If use *callback* function, this parameter can be ``None``. - :param level_or_option: When it's an ``int`` object, it represents :ref:`compression level`. When it's a ``dict`` object, it contains :ref:`advanced compression parameters`. The default value ``None`` means to use zstd's default compression level/parameters. - :type level_or_option: int or dict - :param zstd_dict: Pre-trained dictionary for compression. - :type zstd_dict: ZstdDict - :param pledged_input_size: If set this parameter to the size of input data, the :ref:`size` will be written into the frame header. If the actual input data doesn't match it, a :py:class:`ZstdError` exception will be raised. It may increase compression ratio slightly, and help decompression code to allocate output buffer faster. - :type pledged_input_size: int - :param read_size: Input buffer size, in bytes. - :type read_size: int - :param write_size: Output buffer size, in bytes. - :type write_size: int - :param callback: A callback function that accepts four parameters: ``(total_input, total_output, read_data, write_data)``. The first two are ``int`` objects. The last two are readonly `memoryview `_ objects, if want to reference the data (or its slice) outside the callback function, `convert `_ them to ``bytes`` objects. If input stream is ``b''``, the callback function will not be called. - :type callback: callable - :return: A 2-item tuple, ``(total_input, total_output)``, the items are ``int`` objects. - - .. sourcecode:: python - - # compress an input file, and write to an output file. - with io.open(input_file_path, 'rb') as ifh: - with io.open(output_file_path, 'wb') as ofh: - compress_stream(ifh, ofh, level_or_option=5) - - # compress a bytes object, and write to a file. - with io.BytesIO(raw_dat) as bi: - with io.open(output_file_path, 'wb') as ofh: - compress_stream(bi, ofh, pledged_input_size=len(raw_dat)) - - # Compress an input file, obtain a bytes object. - # It's faster than reading a file and compressing it in - # memory, tested on Ubuntu(Python3.8)/Windows(Python3.9). - # Maybe the OS has prefetching, it can read and compress - # data in parallel to some degree, reading file from HDD - # is the bottleneck in this case. - with io.open(input_file_path, 'rb') as ifh: - with io.BytesIO() as bo: - compress_stream(ifh, bo) - compressed_dat = bo.getvalue() - - # Print progress using callback function - def compress_print_progress(input_file_path, output_file_path): - input_file_size = os.path.getsize(input_file_path) - - def func(total_input, total_output, read_data, write_data): - # If input stream is empty, the callback function - # will not be called. So no ZeroDivisionError here. - percent = 100 * total_input / input_file_size - print(f'Progress: {percent:.1f}%', end='\r') - - with io.open(input_file_path, 'rb') as ifh: - with io.open(output_file_path, 'wb') as ofh: - compress_stream(ifh, ofh, callback=func) - - .. py:class:: ZstdCompressor A streaming compressor. It's thread-safe at method level. @@ -316,76 +249,11 @@ Streaming compression Streaming decompression ----------------------- - This section contains: + You can use :py:class:`ZstdFile` for decompressing data as needed. Advanced users may be interested in: - * function :py:func:`decompress_stream`, a fast and convenient function. * class :py:class:`ZstdDecompressor`, similar to decompressors in Python standard library. * class :py:class:`EndlessZstdDecompressor`, a decompressor accepts multiple concatenated :ref:`frames`. -.. py:function:: decompress_stream(input_stream, output_stream, *, zstd_dict=None, option=None, read_size=131_075, write_size=131_072, callback=None) - - A fast and convenient function, decompresses *input_stream* and writes the decompressed data to *output_stream*, it doesn't close the streams. - - Supports multiple concatenated :ref:`frames`. - - This function tries to zero-copy as much as possible. If the OS has read prefetching and write buffer, it may perform the tasks (read/decompress/write) in parallel to some degree. - - The default values of *read_size* and *write_size* parameters are the buffer sizes recommended by zstd, increasing them may be faster, and reduces the number of callback function calls. - - .. versionadded:: 0.14.2 - - :param input_stream: Input stream that has a `.readinto(b) `_ method. - :param output_stream: Output stream that has a `.write(b) `_ method. If use *callback* function, this parameter can be ``None``. - :param zstd_dict: Pre-trained dictionary for decompression. - :type zstd_dict: ZstdDict - :param option: A ``dict`` object, contains :ref:`advanced decompression parameters`. - :type option: dict - :param read_size: Input buffer size, in bytes. - :type read_size: int - :param write_size: Output buffer size, in bytes. - :type write_size: int - :param callback: A callback function that accepts four parameters: ``(total_input, total_output, read_data, write_data)``. The first two are ``int`` objects. The last two are readonly `memoryview `_ objects, if want to reference the data (or its slice) outside the callback function, `convert `_ them to ``bytes`` objects. If input stream is ``b''``, the callback function will not be called. - :type callback: callable - :return: A 2-item tuple, ``(total_input, total_output)``, the items are ``int`` objects. - :raises ZstdError: If decompression fails. - - .. sourcecode:: python - - # decompress an input file, and write to an output file. - with io.open(input_file_path, 'rb') as ifh: - with io.open(output_file_path, 'wb') as ofh: - decompress_stream(ifh, ofh) - - # decompress a bytes object, and write to a file. - with io.BytesIO(compressed_dat) as bi: - with io.open(output_file_path, 'wb') as ofh: - decompress_stream(bi, ofh) - - # Decompress an input file, obtain a bytes object. - # It's faster than reading a file and decompressing it in - # memory, tested on Ubuntu(Python3.8)/Windows(Python3.9). - # Maybe the OS has prefetching, it can read and decompress - # data in parallel to some degree, reading file from HDD - # is the bottleneck in this case. - with io.open(input_file_path, 'rb') as ifh: - with io.BytesIO() as bo: - decompress_stream(ifh, bo) - decompressed_dat = bo.getvalue() - - # Print progress using callback function - def decompress_print_progress(input_file_path, output_file_path): - input_file_size = os.path.getsize(input_file_path) - - def func(total_input, total_output, read_data, write_data): - # If input stream is empty, the callback function - # will not be called. So no ZeroDivisionError here. - percent = 100 * total_input / input_file_size - print(f'Progress: {percent:.1f}%', end='\r') - - with io.open(input_file_path, 'rb') as ifh: - with io.open(output_file_path, 'wb') as ofh: - decompress_stream(ifh, ofh, callback=func) - .. py:class:: ZstdDecompressor @@ -1210,7 +1078,6 @@ Advanced parameters * :py:func:`richmem_compress` function * :py:class:`ZstdCompressor` class using a single :py:attr:`~ZstdCompressor.FLUSH_FRAME` mode * :py:class:`RichMemZstdCompressor` class - * :py:func:`compress_stream` function setting *pledged_input_size* parameter The field in frame header is 1/2/4/8 bytes, depending on size value. It may help decompression code to allocate output buffer faster. @@ -1537,16 +1404,16 @@ Use with tarfile module import contextlib import io + import shutil import tarfile import tempfile - from pyzstd import decompress_stream + import pyzstd @contextlib.contextmanager - def ZstdTarReader(name, *, zstd_dict=None, option=None, **kwargs): + def ZstdTarReader(name, *, zstd_dict=None, level_or_option=None, **kwargs): with tempfile.TemporaryFile() as tmp_file: - with io.open(name, 'rb') as ifh: - decompress_stream(ifh, tmp_file, - zstd_dict=zstd_dict, option=option) + with pyzstd.open(name, level_or_option=level_or_option, zstd_dict=zstd_dict) as ifh: + shutil.copyfile(ifh, tmp_file) tmp_file.seek(0) with tarfile.TarFile(fileobj=tmp_file, **kwargs) as tar: yield tar @@ -1718,3 +1585,11 @@ Build pyzstd module with options 3️⃣ Disable mremap output buffer on CPython+Linux. On CPython(3.5~3.12)+Linux, pyzstd uses another output buffer code that can utilize the ``mremap`` mechanism, which brings some performance improvements. If this causes problems, you may use ``--no-mremap`` option to disable this code. + + +Deprecations +>>>>>>>>>>>> + +See `list of deprecations with alternatives <./deprecated.html>`_. + +Also, note that `unsupported Python versions ` are not tested against and have no wheels uploaded on PyPI. diff --git a/setup.py b/setup.py index 5e97e24..58c43e8 100644 --- a/setup.py +++ b/setup.py @@ -204,6 +204,7 @@ def do_setup(): url='https://github.com/Rogdham/pyzstd', license='BSD-3-Clause', python_requires='>=3.5', + install_requires=["typing-extensions>=4.13.2 ; python_version<'3.13'"], classifiers=[ "Development Status :: 5 - Production/Stable", @@ -229,7 +230,7 @@ def do_setup(): test_suite='tests' ) - if sys.version_info < (3, 8): + if sys.version_info < (3, 9): print() print("WARNING") print(" Python {} has reached end of life.".format(platform.python_version())) diff --git a/src/__init__.py b/src/__init__.py index 621d695..be1636a 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -17,10 +17,10 @@ _ZSTD_DStreamSizes, _finalize_dict, _train_dict, - compress_stream, + compress_stream as _compress_stream, compressionLevel_values, decompress, - decompress_stream, + decompress_stream as _decompress_stream, get_frame_info, get_frame_size, zstd_version, @@ -46,10 +46,10 @@ _ZSTD_DStreamSizes, _finalize_dict, _train_dict, - compress_stream, + compress_stream as _compress_stream, compressionLevel_values, decompress, - decompress_stream, + decompress_stream as _decompress_stream, get_frame_info, get_frame_size, zstd_version, @@ -67,6 +67,14 @@ from .zstdfile import ZstdFile, open from .seekable_zstdfile import SeekableFormatError, SeekableZstdFile +from functools import wraps + +try: + from warnings import deprecated +except ImportError: + from typing_extensions import deprecated + + __version__ = '0.16.2' __doc__ = '''\ @@ -223,3 +231,14 @@ def finalize_dict(zstd_dict, samples, dict_size, level): dict_size, level) return ZstdDict(dict_content) + + +@wraps(_compress_stream) +@deprecated("See https://pyzstd.readthedocs.io/en/stable/deprecated.html for alternatives to pyzstd.compress_stream") +def compress_stream(*args, **kwargs): + return _compress_stream(*args, **kwargs) + +@wraps(_decompress_stream) +@deprecated("See https://pyzstd.readthedocs.io/en/stable/deprecated.html for alternatives to pyzstd.decompress_stream") +def decompress_stream(*args, **kwargs): + return _decompress_stream(*args, **kwargs) diff --git a/src/__init__.pyi b/src/__init__.pyi index e96212f..edc5072 100644 --- a/src/__init__.pyi +++ b/src/__init__.pyi @@ -5,6 +5,11 @@ from typing import overload, Dict, ByteString, Optional, Union, Callable, \ Iterable, Literal, ClassVar, Tuple, NamedTuple, BinaryIO, \ TextIO +try: + from warnings import deprecated +except ImportError: + from typing_extensions import deprecated + __version__: str zstd_version: str zstd_version_info: Tuple[int, int, int] @@ -146,6 +151,7 @@ def decompress(data: ByteString, zstd_dict: Union[None, ZstdDict, ZstdDictInfo] = None, option: Optional[Dict[DParameter, int]] = None) -> bytes: ... +@deprecated("See https://pyzstd.readthedocs.io/en/stable/deprecated.html for alternatives to pyzstd.compress_stream") def compress_stream(input_stream: BinaryIO, output_stream: Union[BinaryIO, None], *, level_or_option: Union[None, int, Dict[CParameter, int]] = None, zstd_dict: Union[None, ZstdDict, ZstdDictInfo] = None, @@ -153,6 +159,7 @@ def compress_stream(input_stream: BinaryIO, output_stream: Union[BinaryIO, None] read_size: int = 131_072, write_size: int = 131_591, callback: Optional[Callable[[int, int, memoryview, memoryview], None]] = None) -> Tuple[int, int]: ... +@deprecated("See https://pyzstd.readthedocs.io/en/stable/deprecated.html for alternatives to pyzstd.decompress_stream") def decompress_stream(input_stream: BinaryIO, output_stream: Union[BinaryIO, None], *, zstd_dict: Union[None, ZstdDict, ZstdDictInfo] = None, option: Optional[Dict[DParameter, int]] = None, diff --git a/src/__main__.py b/src/__main__.py index aeb60fd..7c46f97 100644 --- a/src/__main__.py +++ b/src/__main__.py @@ -2,9 +2,10 @@ # CLI of pyzstd module: python -m pyzstd --help import argparse import os +from shutil import copyfileobj from time import time -from pyzstd import compress_stream, decompress_stream, \ +from pyzstd import \ CParameter, DParameter, \ train_dict, ZstdDict, ZstdFile, \ compressionLevel_values, zstd_version, \ @@ -14,14 +15,6 @@ C_READ_BUFFER = 131072 D_READ_BUFFER = 131075 -def progress_bar(progress, total, width=50): - # documented behavior: if input stream is empty, the callback - # function will not be called. so no ZeroDivisionError here. - percent = (progress + 1) / total - now = int(width * percent) - bar = "+" * now + "-" * (width - now) - print("|%s| %.2f%%" % (bar, 100*percent), end="\r", flush=True) - # open output file and assign to args.output def open_output(args, path): if not args.f and os.path.isfile(path): @@ -78,9 +71,6 @@ def compress_option(args): return option def compress(args): - # input file size - input_file_size = os.path.getsize(args.input.name) - # output file if args.output is None: open_output(args, args.input.name + '.zst') @@ -97,47 +87,43 @@ def compress(args): option = compress_option(args) # compress - def cb(total_input, total_output, read_data, write_data): - progress_bar(total_input, input_file_size) t1 = time() - ret = compress_stream(args.input, args.output, - level_or_option=option, - zstd_dict=args.zd, - pledged_input_size=input_file_size, - callback=cb) + with ZstdFile(args.output, 'w', level_or_option=option, zstd_dict=args.zd) as fout: + copyfileobj(args.input, fout) t2 = time() + in_size = args.input.tell() + out_size = args.output.tell() close_files(args) # post-compress message - if ret[0] != 0: - ratio = 100 * ret[1] / ret[0] + if in_size != 0: + ratio = 100 * out_size / in_size else: ratio = 100.0 msg = ('\nCompression succeeded, {:.2f} seconds.\n' 'Input {:,} bytes, output {:,} bytes, ratio {:.2f}%.\n').format( - t2-t1, *ret, ratio) + t2-t1, in_size, out_size, ratio) print(msg) def decompress(args): - # input file size - input_file_size = os.path.getsize(args.input.name) - # output file - if args.output is None and args.test is None: - from re import subn - - out_path, replaced = subn(r'(?i)^(.*)\.zst$', r'\1', args.input.name) - if not replaced: - out_path = args.input.name + '.decompressed' + if args.output is None: + if args.test is None: + from re import subn + + out_path, replaced = subn(r'(?i)^(.*)\.zst$', r'\1', args.input.name) + if not replaced: + out_path = args.input.name + '.decompressed' + else: + out_path = os.devnull open_output(args, out_path) # option option = {DParameter.windowLogMax: args.windowLogMax} # pre-decompress message - if args.output is not None: - output_name = args.output.name - else: + output_name = args.output.name + if output_name == os.devnull: output_name = 'None' print(('Decompress file:\n' ' - input file : {}\n' @@ -146,23 +132,22 @@ def decompress(args): args.input.name, output_name, args.zd)) # decompress - def cb(total_input, total_output, read_data, write_data): - progress_bar(total_input, input_file_size) t1 = time() - ret = decompress_stream(args.input, args.output, - zstd_dict=args.zd, option=option, - callback=cb) + with ZstdFile(args.input, level_or_option=option, zstd_dict=args.zd) as fin: + copyfileobj(fin, args.output) t2 = time() + in_size = args.input.tell() + out_size = args.output.tell() close_files(args) # post-decompress message - if ret[1] != 0: - ratio = 100 * ret[0] / ret[1] + if out_size != 0: + ratio = 100 * in_size / out_size else: ratio = 100.0 msg = ('\nDecompression succeeded, {:.2f} seconds.\n' 'Input {:,} bytes, output {:,} bytes, ratio {:.2f}%.\n').format( - t2-t1, *ret, ratio) + t2-t1, in_size, out_size, ratio) print(msg) def train(args): diff --git a/src/bin_ext/stream.c b/src/bin_ext/stream.c index 0959d40..7295a2d 100644 --- a/src/bin_ext/stream.c +++ b/src/bin_ext/stream.c @@ -80,6 +80,11 @@ PyDoc_STRVAR(compress_stream_doc, "----\n" "Compresses input_stream and writes the compressed data to output_stream, it\n" "doesn't close the streams.\n\n" +"----\n" +"DEPRECATION NOTICE\n" +"The (de)compress_stream are deprecated and will be removed in a future version.\n" +"See https://pyzstd.readthedocs.io/en/stable/deprecated.html for alternatives\n" +"----\n\n" "If input stream is b'', nothing will be written to output stream.\n\n" "Return a tuple, (total_input, total_output), the items are int objects.\n\n" "Parameters\n" @@ -344,6 +349,11 @@ PyDoc_STRVAR(decompress_stream_doc, "----\n" "Decompresses input_stream and writes the decompressed data to output_stream,\n" "it doesn't close the streams.\n\n" +"----\n" +"DEPRECATION NOTICE\n" +"The (de)compress_stream are deprecated and will be removed in a future version.\n" +"See https://pyzstd.readthedocs.io/en/stable/deprecated.html for alternatives\n" +"----\n\n" "Supports multiple concatenated frames.\n\n" "Return a tuple, (total_input, total_output), the items are int objects.\n\n" "Parameters\n" diff --git a/src/cffi/stream.py b/src/cffi/stream.py index b3364c5..12de8ca 100644 --- a/src/cffi/stream.py +++ b/src/cffi/stream.py @@ -33,6 +33,12 @@ def compress_stream(input_stream, output_stream, *, """Compresses input_stream and writes the compressed data to output_stream, it doesn't close the streams. + ---- + DEPRECATION NOTICE + The (de)compress_stream are deprecated and will be removed in a future version. + See https://pyzstd.readthedocs.io/en/stable/deprecated.html for alternatives + ---- + If input stream is b'', nothing will be written to output stream. Return a tuple, (total_input, total_output), the items are int objects. @@ -204,6 +210,12 @@ def decompress_stream(input_stream, output_stream, *, """Decompresses input_stream and writes the decompressed data to output_stream, it doesn't close the streams. + ---- + DEPRECATION NOTICE + The (de)compress_stream are deprecated and will be removed in a future version. + See https://pyzstd.readthedocs.io/en/stable/deprecated.html for alternatives + ---- + Supports multiple concatenated frames. Return a tuple, (total_input, total_output), the items are int objects. diff --git a/tests/test_zstd.py b/tests/test_zstd.py index fdd6a5c..ea12626 100644 --- a/tests/test_zstd.py +++ b/tests/test_zstd.py @@ -1,4 +1,5 @@ from io import BytesIO, UnsupportedOperation +from contextlib import contextmanager import builtins import gc import itertools @@ -14,6 +15,7 @@ import subprocess import tempfile import unittest +import warnings import pyzstd from pyzstd import ZstdCompressor, RichMemZstdCompressor, \ @@ -87,6 +89,21 @@ KB = 1024 MB = 1024*1024 +@contextmanager +def _check_deprecated(testcase): + with warnings.catch_warnings(record=True) as warns: + yield + testcase.assertEqual(len(warns), 1) + warn = warns[0] + testcase.assertEqual(warn.category, DeprecationWarning) + testcase.assertIn( + str(warn.message), + [ + "See https://pyzstd.readthedocs.io/en/stable/deprecated.html for alternatives to pyzstd.compress_stream", + "See https://pyzstd.readthedocs.io/en/stable/deprecated.html for alternatives to pyzstd.decompress_stream", + ] + ) + def setUpModule(): # uncompressed size 130KB, more than a zstd block. # with a frame epilogue, 4 bytes checksum. @@ -1222,7 +1239,8 @@ def test_compress_empty(self): # output b'' bi = BytesIO(b'') bo = BytesIO() - ret = compress_stream(bi, bo) + with _check_deprecated(self): + ret = compress_stream(bi, bo) self.assertEqual(ret, (0, 0)) self.assertEqual(bo.getvalue(), b'') bi.close() @@ -1242,7 +1260,8 @@ def test_decompress_empty(self): bi = BytesIO(b'') bo = BytesIO() - ret = decompress_stream(bi, bo) + with _check_deprecated(self): + ret = decompress_stream(bi, bo) self.assertEqual(ret, (0, 0)) self.assertEqual(bo.getvalue(), b'') bi.close() @@ -3710,10 +3729,11 @@ class StreamFunctionsTestCase(unittest.TestCase): def test_compress_stream(self): bi = BytesIO(THIS_FILE_BYTES) bo = BytesIO() - ret = compress_stream(bi, bo, - level_or_option=1, zstd_dict=TRAINED_DICT, - pledged_input_size=2**64-1, # backward compatible - read_size=200*KB, write_size=200*KB) + with _check_deprecated(self): + ret = compress_stream(bi, bo, + level_or_option=1, zstd_dict=TRAINED_DICT, + pledged_input_size=2**64-1, # backward compatible + read_size=200*KB, write_size=200*KB) output = bo.getvalue() self.assertEqual(ret, (len(THIS_FILE_BYTES), len(output))) self.assertEqual(decompress(output, TRAINED_DICT), THIS_FILE_BYTES) @@ -3723,7 +3743,8 @@ def test_compress_stream(self): # empty input bi = BytesIO() bo = BytesIO() - ret = compress_stream(bi, bo, pledged_input_size=None) + with _check_deprecated(self): + ret = compress_stream(bi, bo, pledged_input_size=None) self.assertEqual(ret, (0, 0)) self.assertEqual(bo.getvalue(), b'') bi.close() @@ -3733,14 +3754,16 @@ def test_compress_stream(self): bi = BytesIO(THIS_FILE_BYTES) bo = BytesIO() with self.assertRaises(ZstdError): - compress_stream(bi, bo, pledged_input_size=len(THIS_FILE_BYTES)-1) + with _check_deprecated(self): + compress_stream(bi, bo, pledged_input_size=len(THIS_FILE_BYTES)-1) bi.close() bo.close() bi = BytesIO(THIS_FILE_BYTES) bo = BytesIO() with self.assertRaises(ZstdError): - compress_stream(bi, bo, pledged_input_size=len(THIS_FILE_BYTES)+1) + with _check_deprecated(self): + compress_stream(bi, bo, pledged_input_size=len(THIS_FILE_BYTES)+1) bi.close() bo.close() @@ -3748,25 +3771,35 @@ def test_compress_stream(self): b1 = BytesIO() b2 = BytesIO() with self.assertRaisesRegex(TypeError, r'input_stream'): - compress_stream(123, b1) + with _check_deprecated(self): + compress_stream(123, b1) with self.assertRaisesRegex(TypeError, r'output_stream'): - compress_stream(b1, 123) + with _check_deprecated(self): + compress_stream(b1, 123) with self.assertRaisesRegex(TypeError, r'level_or_option'): - compress_stream(b1, b2, level_or_option='3') + with _check_deprecated(self): + compress_stream(b1, b2, level_or_option='3') with self.assertRaisesRegex(TypeError, r'zstd_dict'): - compress_stream(b1, b2, zstd_dict={}) + with _check_deprecated(self): + compress_stream(b1, b2, zstd_dict={}) with self.assertRaisesRegex(TypeError, r'zstd_dict'): - compress_stream(b1, b2, zstd_dict=b'1234567890') + with _check_deprecated(self): + compress_stream(b1, b2, zstd_dict=b'1234567890') with self.assertRaisesRegex(ValueError, r'pledged_input_size'): - compress_stream(b1, b2, pledged_input_size=-1) + with _check_deprecated(self): + compress_stream(b1, b2, pledged_input_size=-1) with self.assertRaisesRegex(ValueError, r'pledged_input_size'): - compress_stream(b1, b2, pledged_input_size=2**64+1) + with _check_deprecated(self): + compress_stream(b1, b2, pledged_input_size=2**64+1) with self.assertRaisesRegex(ValueError, r'read_size'): - compress_stream(b1, b2, read_size=-1) + with _check_deprecated(self): + compress_stream(b1, b2, read_size=-1) with self.assertRaises(OverflowError): - compress_stream(b1, b2, write_size=2**64+1) + with _check_deprecated(self): + compress_stream(b1, b2, write_size=2**64+1) with self.assertRaisesRegex(TypeError, r'callback'): - compress_stream(b1, None, callback=None) + with _check_deprecated(self): + compress_stream(b1, None, callback=None) b1.close() b2.close() @@ -3782,9 +3815,10 @@ def func(total_input, total_output, read_data, write_data): option = {CParameter.compressionLevel : 1, CParameter.checksumFlag : 1} - ret = compress_stream(bi, bo, level_or_option=option, - read_size=701, write_size=101, - callback=func) + with _check_deprecated(self): + ret = compress_stream(bi, bo, level_or_option=option, + read_size=701, write_size=101, + callback=func) bi.close() bo.close() @@ -3806,8 +3840,9 @@ def test_compress_stream_multi_thread(self): bi = BytesIO(b) bo = BytesIO() - ret = compress_stream(bi, bo, level_or_option=option, - pledged_input_size=len(b)) + with _check_deprecated(self): + ret = compress_stream(bi, bo, level_or_option=option, + pledged_input_size=len(b)) output = bo.getvalue() self.assertEqual(ret, (len(b), len(output))) self.assertEqual(decompress(output), b) @@ -3817,9 +3852,10 @@ def test_compress_stream_multi_thread(self): def test_decompress_stream(self): bi = BytesIO(COMPRESSED_THIS_FILE) bo = BytesIO() - ret = decompress_stream(bi, bo, - option={DParameter.windowLogMax:26}, - read_size=200*KB, write_size=200*KB) + with _check_deprecated(self): + ret = decompress_stream(bi, bo, + option={DParameter.windowLogMax:26}, + read_size=200*KB, write_size=200*KB) self.assertEqual(ret, (len(COMPRESSED_THIS_FILE), len(THIS_FILE_BYTES))) self.assertEqual(bo.getvalue(), THIS_FILE_BYTES) bi.close() @@ -3828,7 +3864,8 @@ def test_decompress_stream(self): # empty input bi = BytesIO() bo = BytesIO() - ret = decompress_stream(bi, bo) + with _check_deprecated(self): + ret = decompress_stream(bi, bo) self.assertEqual(ret, (0, 0)) self.assertEqual(bo.getvalue(), b'') bi.close() @@ -3838,21 +3875,29 @@ def test_decompress_stream(self): b1 = BytesIO() b2 = BytesIO() with self.assertRaisesRegex(TypeError, r'input_stream'): - decompress_stream(123, b1) + with _check_deprecated(self): + decompress_stream(123, b1) with self.assertRaisesRegex(TypeError, r'output_stream'): - decompress_stream(b1, 123) + with _check_deprecated(self): + decompress_stream(b1, 123) with self.assertRaisesRegex(TypeError, r'zstd_dict'): - decompress_stream(b1, b2, zstd_dict={}) + with _check_deprecated(self): + decompress_stream(b1, b2, zstd_dict={}) with self.assertRaisesRegex(TypeError, r'zstd_dict'): - decompress_stream(b1, b2, zstd_dict=b'1234567890') + with _check_deprecated(self): + decompress_stream(b1, b2, zstd_dict=b'1234567890') with self.assertRaisesRegex(TypeError, r'option'): - decompress_stream(b1, b2, option=3) + with _check_deprecated(self): + decompress_stream(b1, b2, option=3) with self.assertRaisesRegex(ValueError, r'read_size'): - decompress_stream(b1, b2, read_size=-1) + with _check_deprecated(self): + decompress_stream(b1, b2, read_size=-1) with self.assertRaises(OverflowError): - decompress_stream(b1, b2, write_size=2**64+1) + with _check_deprecated(self): + decompress_stream(b1, b2, write_size=2**64+1) with self.assertRaisesRegex(TypeError, r'callback'): - decompress_stream(b1, None, callback=None) + with _check_deprecated(self): + decompress_stream(b1, None, callback=None) b1.close() b2.close() @@ -3867,9 +3912,10 @@ def func(total_input, total_output, read_data, write_data): bo = BytesIO() option = {DParameter.windowLogMax : 26} - ret = decompress_stream(bi, bo, option=option, - read_size=701, write_size=401, - callback=func) + with _check_deprecated(self): + ret = decompress_stream(bi, bo, option=option, + read_size=701, write_size=401, + callback=func) bi.close() bo.close() @@ -3884,7 +3930,8 @@ def test_decompress_stream_multi_frames(self): dat = (COMPRESSED_100_PLUS_32KB + SKIPPABLE_FRAME) * 2 bi = BytesIO(dat) bo = BytesIO() - ret = decompress_stream(bi, bo, read_size=200*KB, write_size=50*KB) + with _check_deprecated(self): + ret = decompress_stream(bi, bo, read_size=200*KB, write_size=50*KB) output = bo.getvalue() self.assertEqual(ret, (len(dat), len(output))) self.assertEqual(output, DECOMPRESSED_100_PLUS_32KB + DECOMPRESSED_100_PLUS_32KB) @@ -3895,7 +3942,8 @@ def test_decompress_stream_multi_frames(self): bi = BytesIO(dat[:-1]) bo = BytesIO() with self.assertRaisesRegex(ZstdError, 'incomplete'): - decompress_stream(bi, bo) + with _check_deprecated(self): + decompress_stream(bi, bo) bi.close() bo.close() @@ -3908,13 +3956,17 @@ def write(self, b): return 'a' with self.assertRaises(TypeError): - compress_stream(M(), BytesIO()) + with _check_deprecated(self): + compress_stream(M(), BytesIO()) with self.assertRaises(TypeError): - decompress_stream(M(), BytesIO()) + with _check_deprecated(self): + decompress_stream(M(), BytesIO()) with self.assertRaises(TypeError): - compress_stream(BytesIO(THIS_FILE_BYTES), M()) + with _check_deprecated(self): + compress_stream(BytesIO(THIS_FILE_BYTES), M()) with self.assertRaises(TypeError): - decompress_stream(BytesIO(COMPRESSED_100_PLUS_32KB), M()) + with _check_deprecated(self): + decompress_stream(BytesIO(COMPRESSED_100_PLUS_32KB), M()) # wrong value class N: @@ -3927,55 +3979,69 @@ def write(self, b): # < 0 with self.assertRaisesRegex(ValueError, r'input_stream.readinto.*?<= \d+'): - compress_stream(N(-1), BytesIO()) + with _check_deprecated(self): + compress_stream(N(-1), BytesIO()) with self.assertRaisesRegex(ValueError, r'input_stream.readinto.*?<= \d+'): - decompress_stream(N(-2), BytesIO()) + with _check_deprecated(self): + decompress_stream(N(-2), BytesIO()) with self.assertRaisesRegex(ValueError, r'output_stream.write.*?<= \d+'): - compress_stream(BytesIO(THIS_FILE_BYTES), N(-2)) + with _check_deprecated(self): + compress_stream(BytesIO(THIS_FILE_BYTES), N(-2)) with self.assertRaisesRegex(ValueError, r'output_stream.write.*?<= \d+'): - decompress_stream(BytesIO(COMPRESSED_100_PLUS_32KB), N(-1)) + with _check_deprecated(self): + decompress_stream(BytesIO(COMPRESSED_100_PLUS_32KB), N(-1)) # should > upper bound (~128 KiB) with self.assertRaisesRegex(ValueError, r'input_stream.readinto.*?<= \d+'): - compress_stream(N(10000000), BytesIO()) + with _check_deprecated(self): + compress_stream(N(10000000), BytesIO()) with self.assertRaisesRegex(ValueError, r'input_stream.readinto.*?<= \d+'): - decompress_stream(N(10000000), BytesIO()) + with _check_deprecated(self): + decompress_stream(N(10000000), BytesIO()) with self.assertRaisesRegex(ValueError, r'output_stream.write.*?<= \d+'): - compress_stream(BytesIO(THIS_FILE_BYTES), N(10000000)) + with _check_deprecated(self): + compress_stream(BytesIO(THIS_FILE_BYTES), N(10000000)) with self.assertRaisesRegex(ValueError, r'output_stream.write.*?<= \d+'): - decompress_stream(BytesIO(COMPRESSED_100_PLUS_32KB), N(10000000)) + with _check_deprecated(self): + decompress_stream(BytesIO(COMPRESSED_100_PLUS_32KB), N(10000000)) def test_empty_input_no_callback(self): def cb(a,b,c,d): self.fail('callback function should not be called') # callback function will not be called for empty input, # it's a promised behavior. - compress_stream(io.BytesIO(b''), io.BytesIO(), callback=cb) - decompress_stream(io.BytesIO(b''), io.BytesIO(), callback=cb) + with _check_deprecated(self): + compress_stream(io.BytesIO(b''), io.BytesIO(), callback=cb) + with _check_deprecated(self): + decompress_stream(io.BytesIO(b''), io.BytesIO(), callback=cb) def test_stream_dict(self): zd = ZstdDict(THIS_FILE_BYTES, True) # default with BytesIO(THIS_FILE_BYTES) as bi, BytesIO() as bo: - ret = compress_stream(bi, bo, zstd_dict=zd) + with _check_deprecated(self): + ret = compress_stream(bi, bo, zstd_dict=zd) compressed = bo.getvalue() self.assertEqual(ret, (len(THIS_FILE_BYTES), len(compressed))) with BytesIO(compressed) as bi, BytesIO() as bo: - ret = decompress_stream(bi, bo, zstd_dict=zd) + with _check_deprecated(self): + ret = decompress_stream(bi, bo, zstd_dict=zd) decompressed = bo.getvalue() self.assertEqual(ret, (len(compressed), len(decompressed))) self.assertEqual(decompressed, THIS_FILE_BYTES) # .as_(un)digested_dict with BytesIO(THIS_FILE_BYTES) as bi, BytesIO() as bo: - ret = compress_stream(bi, bo, zstd_dict=zd.as_undigested_dict) + with _check_deprecated(self): + ret = compress_stream(bi, bo, zstd_dict=zd.as_undigested_dict) compressed = bo.getvalue() self.assertEqual(ret, (len(THIS_FILE_BYTES), len(compressed))) with BytesIO(compressed) as bi, BytesIO() as bo: - ret = decompress_stream(bi, bo, zstd_dict=zd.as_digested_dict) + with _check_deprecated(self): + ret = decompress_stream(bi, bo, zstd_dict=zd.as_digested_dict) decompressed = bo.getvalue() self.assertEqual(ret, (len(compressed), len(decompressed))) self.assertEqual(decompressed, THIS_FILE_BYTES) @@ -3984,12 +4050,14 @@ def test_stream_prefix(self): zd = ZstdDict(THIS_FILE_BYTES, True) with BytesIO(THIS_FILE_BYTES) as bi, BytesIO() as bo: - ret = compress_stream(bi, bo, zstd_dict=zd.as_prefix) + with _check_deprecated(self): + ret = compress_stream(bi, bo, zstd_dict=zd.as_prefix) compressed = bo.getvalue() self.assertEqual(ret, (len(THIS_FILE_BYTES), len(compressed))) with BytesIO(compressed) as bi, BytesIO() as bo: - ret = decompress_stream(bi, bo, zstd_dict=zd.as_prefix) + with _check_deprecated(self): + ret = decompress_stream(bi, bo, zstd_dict=zd.as_prefix) decompressed = bo.getvalue() self.assertEqual(ret, (len(compressed), len(decompressed))) self.assertEqual(decompressed, THIS_FILE_BYTES)