Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions Lib/test/test_bz2.py
Original file line number Diff line number Diff line change
Expand Up @@ -967,6 +967,29 @@ def testDecompressorChunksMaxsize(self):
self.assertEqual(out, self.BIG_TEXT)
self.assertEqual(bzd.unused_data, b"")

def test_decompressor_max_length_zero(self):
# max_length=0 buffers the input and never produces output,
# however many times it is repeated.
bzd = BZ2Decompressor()
for _ in range(20):
self.assertEqual(bzd.decompress(b'', max_length=0), b'')
self.assertEqual(bzd.decompress(self.DATA, max_length=0), b'')
for _ in range(20):
self.assertEqual(bzd.decompress(b'', max_length=0), b'')
self.assertFalse(bzd.needs_input)
self.assertFalse(bzd.eof)
self.assertEqual(bzd.decompress(b''), self.TEXT)
self.assertTrue(bzd.eof)

# Output pending from an earlier call is kept for later.
bzd = BZ2Decompressor()
out = bzd.decompress(self.DATA, max_length=1)
for _ in range(20):
self.assertEqual(bzd.decompress(b'', max_length=0), b'')
self.assertFalse(bzd.needs_input)
self.assertEqual(out + bzd.decompress(b''), self.TEXT)
self.assertTrue(bzd.eof)

def test_decompressor_inputbuf_1(self):
# Test reusing input buffer after moving existing
# contents to beginning
Expand Down
23 changes: 23 additions & 0 deletions Lib/test/test_zlib.py
Original file line number Diff line number Diff line change
Expand Up @@ -1231,6 +1231,29 @@ def testDecompressorChunksMaxsize(self):
self.assertEqual(out, self.BIG_TEXT)
self.assertEqual(zlibd.unused_data, b"")

def test_decompressor_max_length_zero(self):
# max_length=0 buffers the input and never produces output,
# however many times it is repeated.
zlibd = zlib._ZlibDecompressor()
for _ in range(20):
self.assertEqual(zlibd.decompress(b'', max_length=0), b'')
self.assertEqual(zlibd.decompress(self.DATA, max_length=0), b'')
for _ in range(20):
self.assertEqual(zlibd.decompress(b'', max_length=0), b'')
self.assertFalse(zlibd.needs_input)
self.assertFalse(zlibd.eof)
self.assertEqual(zlibd.decompress(b''), self.TEXT)
self.assertTrue(zlibd.eof)

# Output pending from an earlier call is kept for later.
zlibd = zlib._ZlibDecompressor()
out = zlibd.decompress(self.DATA, max_length=1)
for _ in range(20):
self.assertEqual(zlibd.decompress(b'', max_length=0), b'')
self.assertFalse(zlibd.needs_input)
self.assertEqual(out + zlibd.decompress(b''), self.TEXT)
self.assertTrue(zlibd.eof)

def test_decompressor_inputbuf_1(self):
# Test reusing input buffer after moving existing
# contents to beginning
Expand Down
35 changes: 35 additions & 0 deletions Lib/test/test_zstd.py
Original file line number Diff line number Diff line change
Expand Up @@ -853,6 +853,29 @@ def test_decompress_empty_content_frame(self):
self.assertEqual(d.unused_data, b'')
self.assertEqual(d.unused_data, b'') # twice

def test_decompressor_max_length_zero(self):
# max_length=0 buffers the input and never produces output,
# however many times it is repeated.
d = ZstdDecompressor()
for _ in range(20):
self.assertEqual(d.decompress(b'', max_length=0), b'')
self.assertEqual(d.decompress(COMPRESSED_100_PLUS_32KB, max_length=0), b'')
for _ in range(20):
self.assertEqual(d.decompress(b'', max_length=0), b'')
self.assertFalse(d.needs_input)
self.assertFalse(d.eof)
self.assertEqual(d.decompress(b''), DECOMPRESSED_100_PLUS_32KB)
self.assertTrue(d.eof)

# Output pending from an earlier call is kept for later.
d = ZstdDecompressor()
out = d.decompress(COMPRESSED_100_PLUS_32KB, max_length=1)
for _ in range(20):
self.assertEqual(d.decompress(b'', max_length=0), b'')
self.assertFalse(d.needs_input)
self.assertEqual(out + d.decompress(b''), DECOMPRESSED_100_PLUS_32KB)
self.assertTrue(d.eof)

class DecompressorFlagsTestCase(unittest.TestCase):

@classmethod
Expand Down Expand Up @@ -1167,6 +1190,18 @@ def test_decompressor_skippable(self):
d = ZstdDecompressor()
dat = d.decompress(SKIPPABLE_FRAME, 0)

self.assertEqual(dat, b'')
self.assertFalse(d.eof)
self.assertFalse(d.needs_input)

dat = d.decompress(b'', 0)

self.assertEqual(dat, b'')
self.assertFalse(d.eof)
self.assertFalse(d.needs_input)

dat = d.decompress(b'')

self.assertEqual(dat, b'')
self.assertTrue(d.eof)
self.assertFalse(d.needs_input)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
Fix :meth:`compression.zstd.ZstdDecompressor.decompress` raising an error when
called repeatedly with ``max_length=0``. Calling ``decompress()`` with
``max_length=0`` on the Zstandard decompressor, :class:`bz2.BZ2Decompressor`
and :class:`!zlib._ZlibDecompressor` now always buffers the input and returns
``b''`` without decompressing anything.
7 changes: 7 additions & 0 deletions Modules/_bz2module.c
Original file line number Diff line number Diff line change
Expand Up @@ -421,6 +421,12 @@ decompress_buf(BZ2Decompressor *d, Py_ssize_t max_length)
goto error;
}

/* With max_length == 0 there is no room for output. Don't call into
libbz2: any input is kept in the input buffer for the next call. */
if (max_length == 0) {
goto finish;
}

for (;;) {
int bzret;
/* On a 64-bit system, buffer length might not fit in avail_out, so we
Expand Down Expand Up @@ -456,6 +462,7 @@ decompress_buf(BZ2Decompressor *d, Py_ssize_t max_length)
}
}

finish:
result = OutputBuffer_Finish(&buffer, bzs->avail_out);
if (result != NULL) {
return result;
Expand Down
7 changes: 7 additions & 0 deletions Modules/_zstd/decompressor.c
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,12 @@ decompress_lock_held(ZstdDecompressor *self, ZSTD_inBuffer *in,
}
assert(out.pos == 0);

/* With max_length == 0 there is no room for output. Don't call into
libzstd: any input is kept in the input buffer for the next call. */
if (max_length == 0) {
goto finish;
}

while (1) {
/* Decompress */
Py_BEGIN_ALLOW_THREADS
Expand Down Expand Up @@ -269,6 +275,7 @@ decompress_lock_held(ZstdDecompressor *self, ZSTD_inBuffer *in,
}
}

finish:
/* Return a bytes object */
ret = _OutputBuffer_Finish(&buffer, &out);
if (ret != NULL) {
Expand Down
6 changes: 6 additions & 0 deletions Modules/zlibmodule.c
Original file line number Diff line number Diff line change
Expand Up @@ -1488,6 +1488,12 @@ decompress_buf(ZlibDecompressor *self, Py_ssize_t max_length)

int err = Z_OK;

/* With max_length == 0 there is no room for output. Don't call into
zlib: any input is kept in the input buffer for the next call. */
if (max_length == 0) {
return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES);
}

/* When sys.maxsize is passed as default use DEF_BUF_SIZE as start buffer.
In this particular case the data may not necessarily be very big, so
it is better to grow dynamically.*/
Expand Down
Loading