Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions Doc/library/bz2.rst
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,10 @@ The :mod:`!bz2` module contains:
readers or writers, just like its equivalent classes in :mod:`gzip` and
:mod:`lzma` have always been.

.. versionchanged:: next
A damaged bzip2 stream that follows another stream now raises
:exc:`OSError` instead of being ignored as trailing data.


Incremental (de)compression
---------------------------
Expand Down Expand Up @@ -307,6 +311,10 @@ One-shot (de)compression
.. versionchanged:: 3.3
Support for multi-stream inputs was added.

.. versionchanged:: next
A damaged bzip2 stream that follows another stream now raises
:exc:`OSError` instead of being ignored as trailing data.


Miscellaneous
-------------
Expand Down
8 changes: 8 additions & 0 deletions Doc/library/lzma.rst
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,10 @@ Reading and writing compressed files
.. versionchanged:: 3.6
Accepts a :term:`path-like object`.

.. versionchanged:: next
A damaged ``.xz`` stream that follows another stream now raises
:exc:`LZMAError` instead of being ignored as trailing data.


Compressing and decompressing data in memory
--------------------------------------------
Expand Down Expand Up @@ -300,6 +304,10 @@ Compressing and decompressing data in memory
See :class:`LZMADecompressor` above for a description of the *format*,
*memlimit* and *filters* arguments.

.. versionchanged:: next
A damaged ``.xz`` stream that follows another stream now raises
:exc:`LZMAError` instead of being ignored as trailing data.


Miscellaneous
-------------
Expand Down
13 changes: 9 additions & 4 deletions Lib/bz2.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@
# Value 2 no longer used
_MODE_WRITE = 3

# Every bzip2 stream starts with one of these, "BZh" and the block size.
_STREAM_MAGIC = tuple(b"BZh%d" % level for level in range(1, 10))


class BZ2File(_streams.BaseStream):

Expand Down Expand Up @@ -91,7 +94,8 @@ def __init__(self, filename, mode="r", *, compresslevel=9):

if self._mode == _MODE_READ:
raw = _streams.DecompressReader(self._fp,
BZ2Decompressor, trailing_error=OSError)
BZ2Decompressor, trailing_error=OSError,
stream_magic=_STREAM_MAGIC)
self._buffer = io.BufferedReader(raw)
else:
self._pos = 0
Expand Down Expand Up @@ -342,10 +346,11 @@ def decompress(data):
try:
res = decomp.decompress(data)
except OSError:
if results:
break # Leftover data is not a valid bzip2 stream; ignore it.
else:
if not results:
raise # Error on the first iteration; bail out.
if data.startswith(_STREAM_MAGIC):
raise # Leftover data is a damaged bzip2 stream; bail out.
break # Leftover data is not a valid bzip2 stream; ignore it.
results.append(res)
if not decomp.eof:
raise ValueError("Compressed data ended before the "
Expand Down
11 changes: 10 additions & 1 deletion Lib/compression/_common/_streams.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,8 @@ class DecompressReader(io.RawIOBase):
def readable(self):
return True

def __init__(self, fp, decomp_factory, trailing_error=(), **decomp_args):
def __init__(self, fp, decomp_factory, trailing_error=(), *,
stream_magic=(), **decomp_args):
self._fp = fp
self._eof = False
self._pos = 0 # Current offset in decompressed stream
Expand All @@ -56,6 +57,11 @@ def __init__(self, fp, decomp_factory, trailing_error=(), **decomp_args):
# trailing data to ignore
self._trailing_error = trailing_error

# Prefix (or tuple of prefixes) that every compressed stream starts
# with, if the format has one. Data starting with it is a damaged
# stream rather than trailing data, so its error is not ignored.
self._stream_magic = stream_magic

def close(self):
self._decompressor = None
return super().close()
Expand Down Expand Up @@ -90,6 +96,9 @@ def read(self, size=-1):
try:
data = self._decompressor.decompress(rawblock, size)
except self._trailing_error:
if (self._stream_magic
and rawblock.startswith(self._stream_magic)):
raise # Damaged stream, not trailing data.
# Trailing data isn't a valid compressed stream; ignore it.
break
else:
Expand Down
18 changes: 14 additions & 4 deletions Lib/lzma.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,13 @@
_MODE_WRITE = 3


def _stream_magic(format):
# Every stream of an .xz file starts with these bytes. The other
# formats, including an .lzma stream read with FORMAT_AUTO, have
# nothing that tells a damaged stream from trailing data.
return b"\xfd7zXZ\x00" if format in (FORMAT_AUTO, FORMAT_XZ) else b""


class LZMAFile(_streams.BaseStream):

"""A file object providing transparent LZMA (de)compression.
Expand Down Expand Up @@ -130,7 +137,8 @@ def __init__(self, filename=None, mode="r", *,

if self._mode == _MODE_READ:
raw = _streams.DecompressReader(self._fp, LZMADecompressor,
trailing_error=LZMAError, format=format, filters=filters)
trailing_error=LZMAError, stream_magic=_stream_magic(format),
format=format, filters=filters)
self._buffer = io.BufferedReader(raw)

def close(self):
Expand Down Expand Up @@ -347,15 +355,17 @@ def decompress(data, format=FORMAT_AUTO, memlimit=None, filters=None):
For incremental decompression, use an LZMADecompressor instead.
"""
results = []
magic = _stream_magic(format)
while True:
decomp = LZMADecompressor(format, memlimit, filters)
try:
res = decomp.decompress(data)
except LZMAError:
if results:
break # Leftover data is not a valid LZMA/XZ stream; ignore it.
else:
if not results:
raise # Error on the first iteration; bail out.
if magic and data.startswith(magic):
raise # Leftover data is a damaged XZ stream; bail out.
break # Leftover data is not a valid LZMA/XZ stream; ignore it.
results.append(res)
if not decomp.eof:
raise LZMAError("Compressed data ended before the "
Expand Down
28 changes: 28 additions & 0 deletions Lib/test/test_bz2.py
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,22 @@ def testReadMultiStreamTrailingJunk(self):
with BZ2File(self.filename) as bz2f:
self.assertEqual(bz2f.read(), self.TEXT * 5)

def testReadMultiStreamDamagedStream(self):
# A damaged bzip2 stream following a valid one is not trailing junk,
# whatever the size of the reads.
for pos in 4, 100, -3:
with self.subTest(pos=pos):
damaged = bytearray(self.DATA)
damaged[pos] ^= 1
self.createTempFile(suffix=damaged)
with BZ2File(self.filename) as bz2f:
self.assertRaises(OSError, bz2f.read)
with BZ2File(self.filename) as bz2f:
self.assertEqual(bz2f.read(len(self.TEXT)), self.TEXT)
with self.assertRaises(OSError):
while bz2f.read(10):
pass

def testRead0(self):
self.createTempFile()
with BZ2File(self.filename) as bz2f:
Expand Down Expand Up @@ -1100,6 +1116,18 @@ def testDecompressMultiStreamTrailingJunk(self):
text = bz2.decompress(self.DATA * 5 + self.BAD_DATA)
self.assertEqual(text, self.TEXT * 5)

def testDecompressMultiStreamDamagedStream(self):
# A damaged bzip2 stream following a valid one is not trailing junk.
for pos in 4, 100, -3:
with self.subTest(pos=pos):
damaged = bytearray(self.DATA)
damaged[pos] ^= 1
self.assertRaises(OSError, bz2.decompress,
self.DATA + damaged)
# Not a bzip2 header: the block size digit must be 1 to 9.
text = bz2.decompress(self.DATA + b"BZh0" + self.BAD_DATA)
self.assertEqual(text, self.TEXT)


class OpenTest(BaseTest):
"Test the open function."
Expand Down
29 changes: 29 additions & 0 deletions Lib/test/test_lzma.py
Original file line number Diff line number Diff line change
Expand Up @@ -524,6 +524,16 @@ def test_decompress_multistream_trailing_junk(self):
ddata = lzma.decompress(COMPRESSED_XZ * 3 + COMPRESSED_BOGUS)
self.assertEqual(ddata, INPUT * 3)

def test_decompress_multistream_damaged_stream(self):
# A damaged .xz stream following a valid one is not trailing junk.
for pos in 6, 100, -20, -1:
damaged = bytearray(COMPRESSED_XZ)
damaged[pos] ^= 1
for format in lzma.FORMAT_AUTO, lzma.FORMAT_XZ:
with self.subTest(pos=pos, format=format):
self.assertRaises(LZMAError, lzma.decompress,
COMPRESSED_XZ + damaged, format)


class TempFile:
"""Context manager - creates a file, and deletes it on __exit__."""
Expand Down Expand Up @@ -885,6 +895,25 @@ def test_read_multistream_trailing_junk(self):
with LZMAFile(BytesIO(COMPRESSED_XZ * 5 + COMPRESSED_BOGUS)) as f:
self.assertEqual(f.read(), INPUT * 5)

def test_read_multistream_damaged_stream(self):
# A damaged .xz stream following a valid one is not trailing junk,
# whatever the size of the reads.
for pos in 6, 100, -20, -1:
damaged = bytearray(COMPRESSED_XZ)
damaged[pos] ^= 1
cdata = COMPRESSED_XZ + damaged
for format in lzma.FORMAT_AUTO, lzma.FORMAT_XZ:
with self.subTest(pos=pos, format=format):
with LZMAFile(BytesIO(cdata), format=format) as f:
self.assertRaises(LZMAError, f.read)
with LZMAFile(BytesIO(cdata), format=format) as f:
self.assertEqual(f.read(len(INPUT)), INPUT)
with self.assertRaises(LZMAError):
while f.read(10):
pass
with LZMAFile(BytesIO(cdata), format=format) as f:
self.assertRaises(LZMAError, f.seek, 0, 2)

def test_read_from_file(self):
with TempFile(TESTFN, COMPRESSED_XZ):
with LZMAFile(TESTFN) as f:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
Raise an error in :class:`lzma.LZMAFile`, :func:`lzma.decompress`,
:class:`bz2.BZ2File` and :func:`bz2.decompress` for a damaged ``.xz`` or bzip2
stream that follows another stream, instead of silently ignoring it and
everything after it as trailing data.
Loading