From c4e4413ac295e1c70bdaa4bbcc23cb7fcada0b09 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Sun, 1 Nov 2026 03:34:44 +0000 Subject: [PATCH] gh-88300: Support .xz stream padding in lzma.decompress() and LZMAFile Null bytes in multiples of four after an .xz stream are now skipped, so that a stream following them is decoded instead of ignored. --- Lib/compression/_common/_streams.py | 64 ++++++++++++++++++- Lib/lzma.py | 3 +- Lib/test/test_lzma.py | 55 ++++++++++++++++ ...6-11-01-03-34-23.gh-issue-88300.GsD4W1.rst | 3 + 4 files changed, 121 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-11-01-03-34-23.gh-issue-88300.GsD4W1.rst diff --git a/Lib/compression/_common/_streams.py b/Lib/compression/_common/_streams.py index 0e3308c0c6e5fdc..9bf988300d4ae35 100644 --- a/Lib/compression/_common/_streams.py +++ b/Lib/compression/_common/_streams.py @@ -37,7 +37,7 @@ def readable(self): return True def __init__(self, fp, decomp_factory, trailing_error=(), *, - stream_magic=(), **decomp_args): + stream_magic=(), stream_padding=False, **decomp_args): self._fp = fp self._eof = False self._pos = 0 # Current offset in decompressed stream @@ -62,6 +62,12 @@ def __init__(self, fp, decomp_factory, trailing_error=(), *, # stream rather than trailing data, so its error is not ignored. self._stream_magic = stream_magic + # Whether a stream that starts with stream_magic (bytes) may be + # followed by Stream Padding, as in the .xz format. _head holds the + # first bytes of the current stream. + self._stream_padding = stream_padding and bool(stream_magic) + self._head = b"" + def close(self): self._decompressor = None return super().close() @@ -88,11 +94,15 @@ def read(self, size=-1): if self._decompressor.eof: rawblock = (self._decompressor.unused_data or self._fp.read(BUFFER_SIZE)) + if self._stream_padding and self._head == self._stream_magic: + rawblock = self._skip_padding(rawblock) if not rawblock: break # Continue to next stream. self._decompressor = self._decomp_factory( **self._decomp_args) + if self._stream_padding: + self._head = rawblock[:len(self._stream_magic)] try: data = self._decompressor.decompress(rawblock, size) except self._trailing_error: @@ -109,6 +119,10 @@ def read(self, size=-1): "end-of-stream marker was reached") else: rawblock = b"" + if (self._stream_padding + and len(self._head) < len(self._stream_magic)): + self._head += rawblock[:len(self._stream_magic) + - len(self._head)] data = self._decompressor.decompress(rawblock, size) if data: break @@ -129,12 +143,30 @@ def readall(self): return b"".join(chunks) + def _skip_padding(self, rawblock): + # Skip the null bytes at the start of the data following a stream + # if they are Stream Padding, whose size is a multiple of four. + size = 0 + while True: + rest = rawblock.lstrip(b"\0") + size += len(rawblock) - len(rest) + if rest: + break + rawblock = self._fp.read(BUFFER_SIZE) + if not rawblock: + break + if size % 4: + # Not Stream Padding: leave the null bytes to the decompressor. + return bytes(size) + rest + return rest + # Rewind the file to the beginning of the data stream. def _rewind(self): self._fp.seek(0) self._eof = False self._pos = 0 self._decompressor = self._decomp_factory(**self._decomp_args) + self._head = b"" def seek(self, offset, whence=io.SEEK_SET): # Recalculate offset as an absolute file position. @@ -171,15 +203,37 @@ def tell(self): return self._pos +def _padding_size(data, pos): + """Return the size of the Stream Padding at *data[pos:]*, or 0. + + Stream Padding is a run of null bytes whose size is a multiple of four. + """ + end = pos + chunk_size = 4 + while end < len(data): + with data[end:end + chunk_size] as chunk: + block = chunk.tobytes() + rest = len(block.lstrip(b"\0")) + end += len(block) - rest + if rest: + break + chunk_size *= 2 + size = end - pos + return 0 if size % 4 else size + + def _decompress_all(data, decomp_factory, trailing_error=(), *, - stream_magic=(), truncated_error, **decomp_args): + stream_magic=(), stream_padding=False, truncated_error, + **decomp_args): """Decompress all of the concatenated streams in *data*. Each stream is decompressed by a new decompressor, made by calling *decomp_factory* with *decomp_args*. As in DecompressReader, an error of class *trailing_error* in a stream after the first one ends the data, unless that stream starts with *stream_magic*. A stream without - an end-of-stream marker raises *truncated_error*. + an end-of-stream marker raises *truncated_error*. If *stream_padding* + is true, a stream that starts with *stream_magic* (bytes) may be + followed by Stream Padding, as in the .xz format. """ results = [] # The views are released on exit, so that the caller can resize a @@ -220,6 +274,10 @@ def _decompress_all(data, decomp_factory, trailing_error=(), *, raise truncated_error("Compressed data ended before the " "end-of-stream marker was reached") pos -= len(decomp.unused_data) + if stream_padding and stream_magic: + with data[start:start + len(stream_magic)] as head: + if head == stream_magic: + pos += _padding_size(data, pos) if pos == len(data): break # Later streams start small: a larger first chunk reads further diff --git a/Lib/lzma.py b/Lib/lzma.py index df749fc119a5843..6421b3f82602ce1 100644 --- a/Lib/lzma.py +++ b/Lib/lzma.py @@ -138,7 +138,7 @@ def __init__(self, filename=None, mode="r", *, if self._mode == _MODE_READ: raw = _streams.DecompressReader(self._fp, LZMADecompressor, trailing_error=LZMAError, stream_magic=_stream_magic(format), - format=format, filters=filters) + stream_padding=True, format=format, filters=filters) self._buffer = io.BufferedReader(raw) def close(self): @@ -356,6 +356,7 @@ def decompress(data, format=FORMAT_AUTO, memlimit=None, filters=None): """ return _streams._decompress_all(data, LZMADecompressor, LZMAError, stream_magic=_stream_magic(format), + stream_padding=True, truncated_error=LZMAError, format=format, memlimit=memlimit, filters=filters) diff --git a/Lib/test/test_lzma.py b/Lib/test/test_lzma.py index c14b3ebf586c96f..110b14c4165db15 100644 --- a/Lib/test/test_lzma.py +++ b/Lib/test/test_lzma.py @@ -585,6 +585,28 @@ def test_decompress_multistream_damaged_stream(self): self.assertRaises(LZMAError, lzma.decompress, COMPRESSED_XZ + damaged, format) + def test_decompress_stream_padding(self): + # Null bytes in multiples of four can follow each .xz stream. + cdata = (COMPRESSED_XZ + bytes(4) + COMPRESSED_XZ + bytes(20) + + COMPRESSED_XZ + bytes(100_000) + COMPRESSED_XZ + bytes(8)) + for format in lzma.FORMAT_AUTO, lzma.FORMAT_XZ: + with self.subTest(format=format): + self.assertEqual(lzma.decompress(COMPRESSED_XZ + bytes(4), + format), INPUT) + self.assertEqual(lzma.decompress(cdata, format), INPUT * 4) + # A damaged stream after padding is not trailing junk. + damaged = bytearray(COMPRESSED_XZ) + damaged[-20] ^= 1 + self.assertRaises(LZMAError, lzma.decompress, + COMPRESSED_XZ + bytes(4) + damaged, format) + self.assertRaises(LZMAError, lzma.decompress, + bytes(4) + COMPRESSED_XZ, format) + # Not after an .lzma stream. + for format in lzma.FORMAT_AUTO, lzma.FORMAT_ALONE: + with self.subTest(format=format): + self.assertRaises(LZMAError, lzma.decompress, + COMPRESSED_ALONE + bytes(4), format) + class TempFile: """Context manager - creates a file, and deletes it on __exit__.""" @@ -965,6 +987,39 @@ def test_read_multistream_damaged_stream(self): with LZMAFile(BytesIO(cdata), format=format) as f: self.assertRaises(LZMAError, f.seek, 0, 2) + def test_read_stream_padding(self): + # Null bytes in multiples of four can follow each .xz stream, + # whatever the size of the reads. + cdata = (COMPRESSED_XZ + bytes(4) + COMPRESSED_XZ + bytes(20) + + COMPRESSED_XZ + bytes(10_000) + COMPRESSED_XZ + bytes(8)) + damaged = bytearray(COMPRESSED_XZ) + damaged[-20] ^= 1 + damaged = COMPRESSED_XZ + bytes(4) + damaged + for buffer_size in _streams.BUFFER_SIZE, len(COMPRESSED_XZ) + 2, 3: + for format in lzma.FORMAT_AUTO, lzma.FORMAT_XZ: + with (self.subTest(buffer_size=buffer_size, format=format), + support.swap_attr(_streams, "BUFFER_SIZE", + buffer_size)): + with LZMAFile(BytesIO(COMPRESSED_XZ + bytes(4)), + format=format) as f: + self.assertEqual(f.read(), INPUT) + with LZMAFile(BytesIO(cdata), format=format) as f: + self.assertEqual(f.read(), INPUT * 4) + self.assertEqual(f.seek(0), 0) + chunks = [] + while result := f.read(10): + chunks.append(result) + self.assertEqual(b"".join(chunks), INPUT * 4) + # A damaged stream after padding is not trailing junk. + with LZMAFile(BytesIO(damaged), format=format) as f: + self.assertRaises(LZMAError, f.read) + # Not after an .lzma stream. + for format in lzma.FORMAT_AUTO, lzma.FORMAT_ALONE: + with self.subTest(format=format): + with LZMAFile(BytesIO(COMPRESSED_ALONE + bytes(4)), + format=format) as f: + self.assertRaises(EOFError, f.read) + def test_read_from_file(self): with TempFile(TESTFN, COMPRESSED_XZ): with LZMAFile(TESTFN) as f: diff --git a/Misc/NEWS.d/next/Library/2026-11-01-03-34-23.gh-issue-88300.GsD4W1.rst b/Misc/NEWS.d/next/Library/2026-11-01-03-34-23.gh-issue-88300.GsD4W1.rst new file mode 100644 index 000000000000000..dd44eee8fe7b44d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-01-03-34-23.gh-issue-88300.GsD4W1.rst @@ -0,0 +1,3 @@ +Fix :func:`lzma.decompress` and :class:`lzma.LZMAFile` raising an error for +``.xz`` stream padding (null bytes in multiples of four) at the end of the +data, and ignoring any ``.xz`` streams that follow such padding.