From 14a2690f8aa34582844f9c7edde3d65a65337d3c Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Thu, 22 Oct 2026 08:29:27 +0000 Subject: [PATCH] gh-160314: Raise for a damaged later stream in lzma and bz2 A damaged .xz or bzip2 stream that followed another stream was taken for trailing data and silently ignored. Leftover data that starts with the stream magic is now treated as a stream, as gzip does. --- Doc/library/bz2.rst | 8 +++++ Doc/library/lzma.rst | 8 +++++ Lib/bz2.py | 13 ++++++--- Lib/compression/_common/_streams.py | 11 ++++++- Lib/lzma.py | 18 +++++++++--- Lib/test/test_bz2.py | 28 ++++++++++++++++++ Lib/test/test_lzma.py | 29 +++++++++++++++++++ ...-10-22-18-30-00.gh-issue-160314.q7RkZd.rst | 4 +++ 8 files changed, 110 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-10-22-18-30-00.gh-issue-160314.q7RkZd.rst diff --git a/Doc/library/bz2.rst b/Doc/library/bz2.rst index e52565dd4bb0a7..bb7028526acf97 100644 --- a/Doc/library/bz2.rst +++ b/Doc/library/bz2.rst @@ -191,6 +191,10 @@ The :mod:`!bz2` module contains: readers or writers, just like its equivalent classes in :mod:`gzip` and :mod:`lzma` have always been. + .. versionchanged:: next + A damaged bzip2 stream that follows another stream now raises + :exc:`OSError` instead of being ignored as trailing data. + Incremental (de)compression --------------------------- @@ -307,6 +311,10 @@ One-shot (de)compression .. versionchanged:: 3.3 Support for multi-stream inputs was added. + .. versionchanged:: next + A damaged bzip2 stream that follows another stream now raises + :exc:`OSError` instead of being ignored as trailing data. + Miscellaneous ------------- diff --git a/Doc/library/lzma.rst b/Doc/library/lzma.rst index ae3dcd811fbec0..ef6e931fb11f4f 100644 --- a/Doc/library/lzma.rst +++ b/Doc/library/lzma.rst @@ -140,6 +140,10 @@ Reading and writing compressed files .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. versionchanged:: next + A damaged ``.xz`` stream that follows another stream now raises + :exc:`LZMAError` instead of being ignored as trailing data. + Compressing and decompressing data in memory -------------------------------------------- @@ -300,6 +304,10 @@ Compressing and decompressing data in memory See :class:`LZMADecompressor` above for a description of the *format*, *memlimit* and *filters* arguments. + .. versionchanged:: next + A damaged ``.xz`` stream that follows another stream now raises + :exc:`LZMAError` instead of being ignored as trailing data. + Miscellaneous ------------- diff --git a/Lib/bz2.py b/Lib/bz2.py index 79305669efc38c..7f03597e5e81ce 100644 --- a/Lib/bz2.py +++ b/Lib/bz2.py @@ -24,6 +24,9 @@ # Value 2 no longer used _MODE_WRITE = 3 +# Every bzip2 stream starts with one of these, "BZh" and the block size. +_STREAM_MAGIC = tuple(b"BZh%d" % level for level in range(1, 10)) + class BZ2File(_streams.BaseStream): @@ -91,7 +94,8 @@ def __init__(self, filename, mode="r", *, compresslevel=9): if self._mode == _MODE_READ: raw = _streams.DecompressReader(self._fp, - BZ2Decompressor, trailing_error=OSError) + BZ2Decompressor, trailing_error=OSError, + stream_magic=_STREAM_MAGIC) self._buffer = io.BufferedReader(raw) else: self._pos = 0 @@ -342,10 +346,11 @@ def decompress(data): try: res = decomp.decompress(data) except OSError: - if results: - break # Leftover data is not a valid bzip2 stream; ignore it. - else: + if not results: raise # Error on the first iteration; bail out. + if data.startswith(_STREAM_MAGIC): + raise # Leftover data is a damaged bzip2 stream; bail out. + break # Leftover data is not a valid bzip2 stream; ignore it. results.append(res) if not decomp.eof: raise ValueError("Compressed data ended before the " diff --git a/Lib/compression/_common/_streams.py b/Lib/compression/_common/_streams.py index 9f367d4e30440f..09c65507297886 100644 --- a/Lib/compression/_common/_streams.py +++ b/Lib/compression/_common/_streams.py @@ -36,7 +36,8 @@ class DecompressReader(io.RawIOBase): def readable(self): return True - def __init__(self, fp, decomp_factory, trailing_error=(), **decomp_args): + def __init__(self, fp, decomp_factory, trailing_error=(), *, + stream_magic=(), **decomp_args): self._fp = fp self._eof = False self._pos = 0 # Current offset in decompressed stream @@ -56,6 +57,11 @@ def __init__(self, fp, decomp_factory, trailing_error=(), **decomp_args): # trailing data to ignore self._trailing_error = trailing_error + # Prefix (or tuple of prefixes) that every compressed stream starts + # with, if the format has one. Data starting with it is a damaged + # stream rather than trailing data, so its error is not ignored. + self._stream_magic = stream_magic + def close(self): self._decompressor = None return super().close() @@ -90,6 +96,9 @@ def read(self, size=-1): try: data = self._decompressor.decompress(rawblock, size) except self._trailing_error: + if (self._stream_magic + and rawblock.startswith(self._stream_magic)): + raise # Damaged stream, not trailing data. # Trailing data isn't a valid compressed stream; ignore it. break else: diff --git a/Lib/lzma.py b/Lib/lzma.py index 73a7c702e03711..ac2c2f7a76a8c4 100644 --- a/Lib/lzma.py +++ b/Lib/lzma.py @@ -37,6 +37,13 @@ _MODE_WRITE = 3 +def _stream_magic(format): + # Every stream of an .xz file starts with these bytes. The other + # formats, including an .lzma stream read with FORMAT_AUTO, have + # nothing that tells a damaged stream from trailing data. + return b"\xfd7zXZ\x00" if format in (FORMAT_AUTO, FORMAT_XZ) else b"" + + class LZMAFile(_streams.BaseStream): """A file object providing transparent LZMA (de)compression. @@ -130,7 +137,8 @@ def __init__(self, filename=None, mode="r", *, if self._mode == _MODE_READ: raw = _streams.DecompressReader(self._fp, LZMADecompressor, - trailing_error=LZMAError, format=format, filters=filters) + trailing_error=LZMAError, stream_magic=_stream_magic(format), + format=format, filters=filters) self._buffer = io.BufferedReader(raw) def close(self): @@ -347,15 +355,17 @@ def decompress(data, format=FORMAT_AUTO, memlimit=None, filters=None): For incremental decompression, use an LZMADecompressor instead. """ results = [] + magic = _stream_magic(format) while True: decomp = LZMADecompressor(format, memlimit, filters) try: res = decomp.decompress(data) except LZMAError: - if results: - break # Leftover data is not a valid LZMA/XZ stream; ignore it. - else: + if not results: raise # Error on the first iteration; bail out. + if magic and data.startswith(magic): + raise # Leftover data is a damaged XZ stream; bail out. + break # Leftover data is not a valid LZMA/XZ stream; ignore it. results.append(res) if not decomp.eof: raise LZMAError("Compressed data ended before the " diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index 2c95a8eec510b0..a21ed230fe8a50 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -156,6 +156,22 @@ def testReadMultiStreamTrailingJunk(self): with BZ2File(self.filename) as bz2f: self.assertEqual(bz2f.read(), self.TEXT * 5) + def testReadMultiStreamDamagedStream(self): + # A damaged bzip2 stream following a valid one is not trailing junk, + # whatever the size of the reads. + for pos in 4, 100, -3: + with self.subTest(pos=pos): + damaged = bytearray(self.DATA) + damaged[pos] ^= 1 + self.createTempFile(suffix=damaged) + with BZ2File(self.filename) as bz2f: + self.assertRaises(OSError, bz2f.read) + with BZ2File(self.filename) as bz2f: + self.assertEqual(bz2f.read(len(self.TEXT)), self.TEXT) + with self.assertRaises(OSError): + while bz2f.read(10): + pass + def testRead0(self): self.createTempFile() with BZ2File(self.filename) as bz2f: @@ -1100,6 +1116,18 @@ def testDecompressMultiStreamTrailingJunk(self): text = bz2.decompress(self.DATA * 5 + self.BAD_DATA) self.assertEqual(text, self.TEXT * 5) + def testDecompressMultiStreamDamagedStream(self): + # A damaged bzip2 stream following a valid one is not trailing junk. + for pos in 4, 100, -3: + with self.subTest(pos=pos): + damaged = bytearray(self.DATA) + damaged[pos] ^= 1 + self.assertRaises(OSError, bz2.decompress, + self.DATA + damaged) + # Not a bzip2 header: the block size digit must be 1 to 9. + text = bz2.decompress(self.DATA + b"BZh0" + self.BAD_DATA) + self.assertEqual(text, self.TEXT) + class OpenTest(BaseTest): "Test the open function." diff --git a/Lib/test/test_lzma.py b/Lib/test/test_lzma.py index df0b66039d77a1..d5431ed6df49d5 100644 --- a/Lib/test/test_lzma.py +++ b/Lib/test/test_lzma.py @@ -524,6 +524,16 @@ def test_decompress_multistream_trailing_junk(self): ddata = lzma.decompress(COMPRESSED_XZ * 3 + COMPRESSED_BOGUS) self.assertEqual(ddata, INPUT * 3) + def test_decompress_multistream_damaged_stream(self): + # A damaged .xz stream following a valid one is not trailing junk. + for pos in 6, 100, -20, -1: + damaged = bytearray(COMPRESSED_XZ) + damaged[pos] ^= 1 + for format in lzma.FORMAT_AUTO, lzma.FORMAT_XZ: + with self.subTest(pos=pos, format=format): + self.assertRaises(LZMAError, lzma.decompress, + COMPRESSED_XZ + damaged, format) + class TempFile: """Context manager - creates a file, and deletes it on __exit__.""" @@ -885,6 +895,25 @@ def test_read_multistream_trailing_junk(self): with LZMAFile(BytesIO(COMPRESSED_XZ * 5 + COMPRESSED_BOGUS)) as f: self.assertEqual(f.read(), INPUT * 5) + def test_read_multistream_damaged_stream(self): + # A damaged .xz stream following a valid one is not trailing junk, + # whatever the size of the reads. + for pos in 6, 100, -20, -1: + damaged = bytearray(COMPRESSED_XZ) + damaged[pos] ^= 1 + cdata = COMPRESSED_XZ + damaged + for format in lzma.FORMAT_AUTO, lzma.FORMAT_XZ: + with self.subTest(pos=pos, format=format): + with LZMAFile(BytesIO(cdata), format=format) as f: + self.assertRaises(LZMAError, f.read) + with LZMAFile(BytesIO(cdata), format=format) as f: + self.assertEqual(f.read(len(INPUT)), INPUT) + with self.assertRaises(LZMAError): + while f.read(10): + pass + with LZMAFile(BytesIO(cdata), format=format) as f: + self.assertRaises(LZMAError, f.seek, 0, 2) + def test_read_from_file(self): with TempFile(TESTFN, COMPRESSED_XZ): with LZMAFile(TESTFN) as f: diff --git a/Misc/NEWS.d/next/Library/2026-10-22-18-30-00.gh-issue-160314.q7RkZd.rst b/Misc/NEWS.d/next/Library/2026-10-22-18-30-00.gh-issue-160314.q7RkZd.rst new file mode 100644 index 00000000000000..3694e255546f68 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-22-18-30-00.gh-issue-160314.q7RkZd.rst @@ -0,0 +1,4 @@ +Raise an error in :class:`lzma.LZMAFile`, :func:`lzma.decompress`, +:class:`bz2.BZ2File` and :func:`bz2.decompress` for a damaged ``.xz`` or bzip2 +stream that follows another stream, instead of silently ignoring it and +everything after it as trailing data.