Merge branch '3.14' into backport-1259f16-3.14

Author: gpshead

Doc/library/numbers.rst

@@ -93,7 +93,7 @@ them to the same values. This may be subtle if there are two different
 extensions of the real numbers. See also :ref:`numeric-hash`.
 
 
-Adding More Numeric ABCs
+Adding more numeric ABCs
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
 There are, of course, more possible ABCs for numbers, and this would
@@ -143,11 +143,12 @@ defined as::
 
 
 There are 5 different cases for a mixed-type operation on subclasses
-of :class:`Complex`. I'll refer to all of the above code that doesn't
-refer to ``MyIntegral`` and ``OtherTypeIKnowAbout`` as
-"boilerplate". ``a`` will be an instance of ``A``, which is a subtype
-of :class:`Complex` (``a : A <: Complex``), and ``b : B <:
-Complex``. I'll consider ``a + b``:
+of :class:`Complex`. All of the above code that doesn't refer to
+``MyIntegral`` and ``OtherTypeIKnowAbout`` is referred to as "boilerplate".
+
+In this description, ``a`` is an instance of ``A`` and ``b`` is an
+instance of ``B``, where ``A`` and ``B`` are both subtypes of :class:`Complex`.
+Consider ``a + b``:
 
 1. If ``A`` defines an :meth:`~object.__add__` which accepts ``b``, all is
    well.
@@ -162,13 +163,14 @@ Complex``. I'll consider ``a + b``:
 4. If it falls back to the boilerplate, there are no more possible
    methods to try, so this is where the default implementation
    should live.
-5. If ``B <: A``, Python tries ``B.__radd__`` before
+5. If ``B`` is a subtype of ``A``, Python tries ``B.__radd__`` before
    ``A.__add__``. This is ok, because it was implemented with
    knowledge of ``A``, so it can handle those instances before
    delegating to :class:`Complex`.
 
-If ``A <: Complex`` and ``B <: Real`` without sharing any other knowledge,
-then the appropriate shared operation is the one involving the built
+If ``A`` is a subtype of :class:`Complex` and ``B`` is a subtype of
+:class:`Real` without sharing any other knowledge, then the appropriate
+shared operation is the one involving the built
 in :class:`complex`, and both :meth:`~object.__radd__` s land there, so ``a+b
 == b+a``.
 

Lib/bz2.py

@@ -22,6 +22,9 @@
 # Value 2 no longer used
 _MODE_WRITE    = 3
 
+# Every bzip2 stream starts with one of these, "BZh" and the block size.
+_STREAM_MAGIC = (b'BZh1', b'BZh2', b'BZh3', b'BZh4', b'BZh5', b'BZh6', b'BZh7', b'BZh8', b'BZh9')
+
 
 class BZ2File(_streams.BaseStream):
 
@@ -89,7 +92,8 @@ def __init__(self, filename, mode="r", *, compresslevel=9):
 
         if self._mode == _MODE_READ:
             raw = _streams.DecompressReader(self._fp,
-                BZ2Decompressor, trailing_error=OSError)
+                BZ2Decompressor, trailing_error=OSError,
+                stream_magic=_STREAM_MAGIC)
             self._buffer = io.BufferedReader(raw)
         else:
             self._pos = 0
@@ -334,19 +338,8 @@ def decompress(data):
 
     For incremental decompression, use a BZ2Decompressor object instead.
     """
-    results = []
-    while data:
-        decomp = BZ2Decompressor()
-        try:
-            res = decomp.decompress(data)
-        except OSError:
-            if results:
-                break  # Leftover data is not a valid bzip2 stream; ignore it.
-            else:
-                raise  # Error on the first iteration; bail out.
-        results.append(res)
-        if not decomp.eof:
-            raise ValueError("Compressed data ended before the "
-                             "end-of-stream marker was reached")
-        data = decomp.unused_data
-    return b"".join(results)
+    if not data:
+        return b""
+    return _streams._decompress_all(data, BZ2Decompressor, OSError,
+                                    stream_magic=_STREAM_MAGIC,
+                                    truncated_error=ValueError)

Lib/compression/_common/_streams.py

@@ -36,7 +36,8 @@ class DecompressReader(io.RawIOBase):
     def readable(self):
         return True
 
-    def __init__(self, fp, decomp_factory, trailing_error=(), **decomp_args):
+    def __init__(self, fp, decomp_factory, trailing_error=(), *,
+                 stream_magic=(), **decomp_args):
         self._fp = fp
         self._eof = False
         self._pos = 0  # Current offset in decompressed stream
@@ -56,6 +57,11 @@ def __init__(self, fp, decomp_factory, trailing_error=(), **decomp_args):
         # trailing data to ignore
         self._trailing_error = trailing_error
 
+        # Prefix (or tuple of prefixes) that every compressed stream starts
+        # with, if the format has one.  Data starting with it is a damaged
+        # stream rather than trailing data, so its error is not ignored.
+        self._stream_magic = stream_magic
+
     def close(self):
         self._decompressor = None
         return super().close()
@@ -90,6 +96,9 @@ def read(self, size=-1):
                 try:
                     data = self._decompressor.decompress(rawblock, size)
                 except self._trailing_error:
+                    if (self._stream_magic
+                            and rawblock.startswith(self._stream_magic)):
+                        raise  # Damaged stream, not trailing data.
                     # Trailing data isn't a valid compressed stream; ignore it.
                     break
             else:
@@ -160,3 +169,60 @@ def seek(self, offset, whence=io.SEEK_SET):
     def tell(self):
         """Return the current file position."""
         return self._pos
+
+
+def _decompress_all(data, decomp_factory, trailing_error=(), *,
+                    stream_magic=(), truncated_error, **decomp_args):
+    """Decompress all of the concatenated streams in *data*.
+
+    Each stream is decompressed by a new decompressor, made by calling
+    *decomp_factory* with *decomp_args*.  As in DecompressReader, an error
+    of class *trailing_error* in a stream after the first one ends the
+    data, unless that stream starts with *stream_magic*.  A stream without
+    an end-of-stream marker raises *truncated_error*.
+    """
+    results = []
+    # The views are released on exit, so that the caller can resize a
+    # bytearray argument again, also while handling an exception.
+    with memoryview(data) as view, view.cast("B") as data:
+        pos = 0
+        # There is usually a single stream: the first chunk is all of the
+        # data, to decompress it in one go.
+        chunk_size = len(data)
+        while True:
+            decomp = decomp_factory(**decomp_args)
+            start = pos
+            res = []
+            try:
+                # Feed the stream in chunks of increasing size, so that what
+                # is read past its end (and copied into unused_data) is
+                # bounded by the size of the stream, not by the size of all
+                # the streams that follow it.
+                while not decomp.eof and pos < len(data):
+                    with data[pos:pos + chunk_size] as chunk:
+                        res.append(decomp.decompress(chunk))
+                        pos += len(chunk)
+                    chunk_size *= 2
+            except trailing_error:
+                if not results:
+                    raise  # Error in the first stream; bail out.
+                if stream_magic:
+                    if isinstance(stream_magic, bytes):
+                        stream_magic = (stream_magic,)
+                    size = max(map(len, stream_magic))
+                    with data[start:start + size] as head:
+                        if head.tobytes().startswith(stream_magic):
+                            raise  # Damaged stream, not trailing data.
+                # Trailing data isn't a valid compressed stream; ignore it.
+                break
+            results.extend(res)
+            if not decomp.eof:
+                raise truncated_error("Compressed data ended before the "
+                                      "end-of-stream marker was reached")
+            pos -= len(decomp.unused_data)
+            if pos == len(data):
+                break
+            # Later streams start small: a larger first chunk reads further
+            # past the end of a short stream, a smaller one needs more calls.
+            chunk_size = 1024
+    return b"".join(results)

Lib/compression/zstd/__init__.py

@@ -30,6 +30,7 @@
 import enum
 from _zstd import (ZstdCompressor, ZstdDecompressor, ZstdDict, ZstdError,
                    get_frame_size, zstd_version)
+from compression._common import _streams
 from compression.zstd._zstdfile import ZstdFile, open, _nbytes
 
 # zstd_version_number is (MAJOR * 100 * 100 + MINOR * 100 + RELEASE)
@@ -161,17 +162,9 @@ def decompress(data, zstd_dict=None, options=None):
 
     For incremental decompression, use a ZstdDecompressor instead.
     """
-    results = []
-    while True:
-        decomp = ZstdDecompressor(options=options, zstd_dict=zstd_dict)
-        results.append(decomp.decompress(data))
-        if not decomp.eof:
-            raise ZstdError('Compressed data ended before the '
-                            'end-of-stream marker was reached')
-        data = decomp.unused_data
-        if not data:
-            break
-    return b''.join(results)
+    return _streams._decompress_all(data, ZstdDecompressor,
+                                    truncated_error=ZstdError,
+                                    options=options, zstd_dict=zstd_dict)
 
 
 class CompressionParameter(enum.IntEnum):

Lib/gzip.py

@@ -650,24 +650,47 @@ def decompress(data):
     Return the decompressed string.
     """
     decompressed_members = []
+    fp = io.BytesIO(data)
+    # There is usually a single member: the first chunk is all of the data,
+    # to decompress it in one go.
+    chunk_size = len(data)
     while True:
-        fp = io.BytesIO(data)
         if _read_gzip_header(fp) is None:
             return b"".join(decompressed_members)
         # Use a zlib raw deflate compressor
         do = zlib.decompressobj(wbits=-zlib.MAX_WBITS)
-        # Read all the data except the header
-        decompressed = do.decompress(data[fp.tell():])
-        if not do.eof or len(do.unused_data) < 8:
+        crc = zlib.crc32(b"")
+        length = 0
+        # Feed the member in chunks of increasing size, so that what is read
+        # past its end (and copied into unused_data) is bounded by the size
+        # of the member, not by the size of all the members that follow it.
+        while not do.eof and (chunk := fp.read(chunk_size)):
+            decompressed = do.decompress(chunk)
+            crc = zlib.crc32(decompressed, crc)
+            length += len(decompressed)
+            decompressed_members.append(decompressed)
+            chunk_size *= 2
+        # Later members start small: a larger first chunk reads further past
+        # the end of a short member, a smaller one needs more calls.
+        chunk_size = 1024
+        # Rewind to the end of the deflate stream and read the trailer
+        fp.seek(-len(do.unused_data), io.SEEK_CUR)
+        trailer = fp.read(8)
+        if not do.eof or len(trailer) < 8:
             raise EOFError("Compressed file ended before the end-of-stream "
                            "marker was reached")
-        crc, length = struct.unpack("<II", do.unused_data[:8])
-        if crc != zlib.crc32(decompressed):
+        crc32, isize = struct.unpack("<II", trailer)
+        if crc32 != crc:
             raise BadGzipFile("CRC check failed")
-        if length != (len(decompressed) & 0xffffffff):
+        if isize != (length & 0xffffffff):
             raise BadGzipFile("Incorrect length of data produced")
-        decompressed_members.append(decompressed)
-        data = do.unused_data[8:].lstrip(b"\x00")
+        # Gzip files can be padded with zeroes and still have archives.
+        # Consume all zero bytes and set the file position to the first
+        # non-zero byte.
+        while chunk := fp.read(chunk_size):
+            if chunk := chunk.lstrip(b"\x00"):
+                fp.seek(-len(chunk), io.SEEK_CUR)
+                break
 
 
 def main():

Lib/lzma.py

@@ -35,6 +35,13 @@
 _MODE_WRITE    = 3
 
 
+def _stream_magic(format):
+    # Every stream of an .xz file starts with these bytes.  The other
+    # formats, including an .lzma stream read with FORMAT_AUTO, have
+    # nothing that tells a damaged stream from trailing data.
+    return b"\xfd7zXZ\x00" if format in (FORMAT_AUTO, FORMAT_XZ) else b""
+
+
 class LZMAFile(_streams.BaseStream):
 
     """A file object providing transparent LZMA (de)compression.
@@ -128,7 +135,8 @@ def __init__(self, filename=None, mode="r", *,
 
         if self._mode == _MODE_READ:
             raw = _streams.DecompressReader(self._fp, LZMADecompressor,
-                trailing_error=LZMAError, format=format, filters=filters)
+                trailing_error=LZMAError, stream_magic=_stream_magic(format),
+                format=format, filters=filters)
             self._buffer = io.BufferedReader(raw)
 
     def close(self):
@@ -344,21 +352,8 @@ def decompress(data, format=FORMAT_AUTO, memlimit=None, filters=None):
 
     For incremental decompression, use an LZMADecompressor instead.
     """
-    results = []
-    while True:
-        decomp = LZMADecompressor(format, memlimit, filters)
-        try:
-            res = decomp.decompress(data)
-        except LZMAError:
-            if results:
-                break  # Leftover data is not a valid LZMA/XZ stream; ignore it.
-            else:
-                raise  # Error on the first iteration; bail out.
-        results.append(res)
-        if not decomp.eof:
-            raise LZMAError("Compressed data ended before the "
-                            "end-of-stream marker was reached")
-        data = decomp.unused_data
-        if not data:
-            break
-    return b"".join(results)
+    return _streams._decompress_all(data, LZMADecompressor, LZMAError,
+                                    stream_magic=_stream_magic(format),
+                                    truncated_error=LZMAError,
+                                    format=format, memlimit=memlimit,
+                                    filters=filters)

Lib/tarfile.py

@@ -2707,6 +2707,13 @@ def _extract_member(self, tarinfo, targetpath, set_attrs=True,
         else:
             self.makefile(tarinfo, targetpath)
 
+        if tarinfo.islnk() and os.path.islink(targetpath):
+            # makelink_with_filter() fell back to extracting a copy of a
+            # symlink member, which already has its own attributes set.
+            # Setting the hard link's attributes here would follow the
+            # symlink, possibly to a file outside the destination directory.
+            set_attrs = False
+
         if set_attrs:
             self.chown(tarinfo, targetpath, numeric_owner)
             if not tarinfo.issym():