Merge branch '3.14' into backport-47c249a-3.14

Author: eendebakpt

Lib/codecs.py

@@ -400,9 +400,10 @@ def reset(self):
         pass
 
     def seek(self, offset, whence=0):
-        self.stream.seek(offset, whence)
+        # reset() may write to the stream: call it at the old position.
         if whence == 0 and offset == 0:
             self.reset()
+        self.stream.seek(offset, whence)
 
     def __getattr__(self, name,
                     getattr=getattr):
@@ -739,10 +740,11 @@ def reset(self):
         self.writer.reset()
 
     def seek(self, offset, whence=0):
-        self.stream.seek(offset, whence)
-        self.reader.reset()
+        # writer.reset() may write to the stream: call it at the old position.
         if whence == 0 and offset == 0:
             self.writer.reset()
+        self.stream.seek(offset, whence)
+        self.reader.reset()
 
     def __getattr__(self, name,
                     getattr=getattr):
@@ -862,8 +864,8 @@ def reset(self):
     def seek(self, offset, whence=0):
         # Seeks must be propagated to both the readers and writers
         # as they might need to reset their internal buffers.
-        self.reader.seek(offset, whence)
         self.writer.seek(offset, whence)
+        self.reader.seek(offset, whence)
 
     def __getattr__(self, name,
                     getattr=getattr):

Lib/gzip.py

@@ -732,7 +732,7 @@ def main():
                              compresslevel=compresslevel)
             else:
                 f = builtins.open(arg, "rb")
-                g = open(arg + ".gz", "wb")
+                g = open(arg + ".gz", "wb", compresslevel=compresslevel)
         while True:
             chunk = f.read(READ_BUFFER_SIZE)
             if not chunk:

Lib/multiprocessing/shared_memory.py

@@ -305,14 +305,21 @@ def _extract_recreation_code(value):
         else:
             return 3  # NoneType
 
+    @staticmethod
+    def _encode_value(value):
+        if not isinstance(value, str):
+            return value
+        else:
+            return value.encode(_encoding)
+
     def __init__(self, sequence=None, *, name=None):
         if name is None or sequence is not None:
             sequence = sequence or ()
             _formats = [
                 self._types_mapping[type(item)]
                     if not isinstance(item, (str, bytes))
                     else self._types_mapping[type(item)] % (
-                        self._alignment * (len(item) // self._alignment + 1),
+                        self._alignment * ((len(self._encode_value(item)) - 1) // self._alignment + 1),
                     )
                 for item in sequence
             ]
@@ -353,7 +360,7 @@ def __init__(self, sequence=None, *, name=None):
                 "".join(_formats),
                 self.shm.buf,
                 self._offset_data_start,
-                *(v.encode(_enc) if isinstance(v, str) else v for v in sequence)
+                *(self._encode_value(v) for v in sequence)
             )
             struct.pack_into(
                 self._format_packing_metainfo,
@@ -463,8 +470,7 @@ def __setitem__(self, position, value):
         else:
             allocated_length = self._allocated_offsets[position + 1] - item_offset
 
-            encoded_value = (value.encode(_encoding)
-                             if isinstance(value, str) else value)
+            encoded_value = self._encode_value(value)
             if len(encoded_value) > allocated_length:
                 raise ValueError("bytes/str item exceeds available storage")
             if current_format[-1] == "s":

Lib/tarfile.py

@@ -2707,6 +2707,13 @@ def _extract_member(self, tarinfo, targetpath, set_attrs=True,
         else:
             self.makefile(tarinfo, targetpath)
 
+        if tarinfo.islnk() and os.path.islink(targetpath):
+            # makelink_with_filter() fell back to extracting a copy of a
+            # symlink member, which already has its own attributes set.
+            # Setting the hard link's attributes here would follow the
+            # symlink, possibly to a file outside the destination directory.
+            set_attrs = False
+
         if set_attrs:
             self.chown(tarinfo, targetpath, numeric_owner)
             if not tarinfo.issym():

Lib/test/_test_multiprocessing.py

@@ -4688,7 +4688,7 @@ def test_shared_memory_SharedMemoryManager_basics(self):
 
     def test_shared_memory_ShareableList_basics(self):
         sl = shared_memory.ShareableList(
-            ['howdy', b'HoWdY', -273.154, 100, None, True, 42]
+            ['howdy', b'HoWdY', -273.154, 100, None, True, 42, '💥 💥']
         )
         self.addCleanup(sl.shm.unlink)
 
@@ -4698,22 +4698,22 @@ def test_shared_memory_ShareableList_basics(self):
 
         # Index Out of Range (get)
         with self.assertRaises(IndexError):
-            sl[7]
+            sl[8]
 
         # Index Out of Range (set)
         with self.assertRaises(IndexError):
-            sl[7] = 2
+            sl[8] = 2
 
         # Assign value without format change (str -> str)
         current_format = sl._get_packing_format(0)
         sl[0] = 'howdy'
         self.assertEqual(current_format, sl._get_packing_format(0))
 
         # Verify attributes are readable.
-        self.assertEqual(sl.format, '8s8sdqxxxxxx?xxxxxxxx?q')
+        self.assertEqual(sl.format, '8s8sdqxxxxxx?xxxxxxxx?q16s')
 
         # Exercise len().
-        self.assertEqual(len(sl), 7)
+        self.assertEqual(len(sl), 8)
 
         # Exercise index().
         with warnings.catch_warnings():
@@ -4725,30 +4725,31 @@ def test_shared_memory_ShareableList_basics(self):
 
         # Exercise retrieving individual values.
         self.assertEqual(sl[0], 'howdy')
-        self.assertEqual(sl[-2], True)
+        self.assertEqual(sl[-3], True)
+        self.assertEqual(sl[-1], '💥 💥')
 
         # Exercise iterability.
         self.assertEqual(
             tuple(sl),
-            ('howdy', b'HoWdY', -273.154, 100, None, True, 42)
+            ('howdy', b'HoWdY', -273.154, 100, None, True, 42, '💥 💥')
         )
 
         # Exercise modifying individual values.
         sl[3] = 42
         self.assertEqual(sl[3], 42)
         sl[4] = 'some'  # Change type at a given position.
         self.assertEqual(sl[4], 'some')
-        self.assertEqual(sl.format, '8s8sdq8sxxxxxxx?q')
+        self.assertEqual(sl.format, '8s8sdq8sxxxxxxx?q16s')
         with self.assertRaisesRegex(ValueError,
                                     "exceeds available storage"):
             sl[4] = 'far too many'
         self.assertEqual(sl[4], 'some')
-        sl[0] = 'encodés'  # Exactly 8 bytes of UTF-8 data
-        self.assertEqual(sl[0], 'encodés')
+        sl[0] = 'éncodé'  # Exactly 8 bytes of UTF-8 data
+        self.assertEqual(sl[0], 'éncodé')
         self.assertEqual(sl[1], b'HoWdY')  # no spillage
         with self.assertRaisesRegex(ValueError,
                                     "exceeds available storage"):
-            sl[0] = 'encodées'  # Exactly 9 bytes of UTF-8 data
+            sl[0] = 'éncodés'  # Exactly 9 bytes of UTF-8 data
         self.assertEqual(sl[1], b'HoWdY')
         with self.assertRaisesRegex(ValueError,
                                     "exceeds available storage"):

Lib/test/test_codecencodings_cn.py

@@ -92,5 +92,15 @@ class Test_HZ(multibytecodec_support.TestBase, unittest.TestCase):
         (b'~{Dc~\n:C~}', 'strict', None), # ~\n only in ASCII mode
     )
 
+    def test_escape_at_buffer_boundary(self):
+        # '~~' must not be cut in two when the output buffer is enlarged.
+        for pairs in range(40):
+            for letters in range(3):
+                text = 'a\u4e2d' * pairs + 'x' * letters + '~{'
+                with self.subTest(pairs=pairs, letters=letters):
+                    encoded = text.encode('hz')
+                    self.assertEndsWith(encoded, b'~~{')
+                    self.assertEqual(encoded.decode('hz'), text)
+
 if __name__ == "__main__":
     unittest.main()

Lib/test/test_gzip.py

@@ -1189,7 +1189,8 @@ def test_compress_infile_outfile_default(self):
 
     @create_and_remove_directory(TEMPDIR)
     def test_compress_infile_outfile(self):
-        for compress_level in ('--fast', '--best'):
+        # The XFL header byte records the compression level used.
+        for compress_level, xfl in (('--fast', 4), ('--best', 2)):
             with self.subTest(compress_level=compress_level):
                 local_testgzip = os.path.join(TEMPDIR, 'testgzip')
                 gzipname = local_testgzip + '.gz'
@@ -1203,6 +1204,8 @@ def test_compress_infile_outfile(self):
                 self.assertTrue(os.path.exists(gzipname))
                 self.assertEqual(out, b'')
                 self.assertEqual(err, b'')
+                with open(gzipname, 'rb') as fp:
+                    self.assertEqual(fp.read()[8], xfl)
                 os.remove(gzipname)
                 self.assertFalse(os.path.exists(gzipname))
 

Lib/test/test_multibytecodec.py

@@ -39,6 +39,10 @@ def test_nullcoding(self):
             self.assertEqual(str(b'', enc), '')
             self.assertEqual(''.encode(enc), b'')
 
+    def test_hz_tilde_at_buffer_boundary(self):
+        text = 'a' + '中a' * 6 + '~'
+        self.assertEqual(text.encode('hz').decode('hz'), text)
+
     def test_str_decode(self):
         for enc in ALL_CJKENCODINGS:
             self.assertEqual('abcd'.encode(enc), b'abcd')
@@ -91,6 +95,12 @@ def test_decode_unicode(self):
 
 class Test_IncrementalEncoder(unittest.TestCase):
 
+    def test_iso2022_kr_reset(self):
+        encoder = codecs.getincrementalencoder('iso2022_kr')()
+        encoder.encode('가')
+        encoder.reset()
+        self.assertEqual(encoder.encode('가', final=True), '가'.encode('iso2022_kr'))
+
     def test_stateless(self):
         # cp949 encoder isn't stateful at all.
         encoder = codecs.getincrementalencoder('cp949')()
@@ -242,6 +252,19 @@ def test_subinterp(self):
 
 class Test_IncrementalDecoder(unittest.TestCase):
 
+    def test_iso2022_split_escape(self):
+        raw = b'\x1b$0000000B'
+        decoder = codecs.getincrementaldecoder('iso2022_jp')('replace')
+        decoded = decoder.decode(raw[:-1]) + decoder.decode(raw[-1:], final=True)
+        self.assertEqual(decoded, raw.decode('iso2022_jp', 'replace'))
+
+    def test_iso2022_reset(self):
+        decoder = codecs.getincrementaldecoder('iso2022_jp')()
+        decoder.decode(b'\x1b%')
+        decoder.reset()
+        with self.assertRaises(UnicodeDecodeError):
+            decoder.decode(b'\xff', final=True)
+
     def test_dbcs(self):
         # cp949 decoder is simple with only 1 or 2 bytes sequences.
         decoder = codecs.getincrementaldecoder('cp949')()
@@ -325,7 +348,7 @@ def test_setstate_validates_input(self):
         self.assertRaises(TypeError, decoder.setstate, 123)
         self.assertRaises(TypeError, decoder.setstate, ("invalid", 0))
         self.assertRaises(TypeError, decoder.setstate, (b"1234", "invalid"))
-        self.assertRaises(UnicodeDecodeError, decoder.setstate, (b"123456789", 0))
+        self.assertRaises(UnicodeDecodeError, decoder.setstate, (b"0" * 17, 0))
 
     def test_setstate_invalid_designation(self):
         # gh-153603: an unknown charset designation in the state must not crash
@@ -438,6 +461,42 @@ def test_incrementalencoder_final(self):
         self.assertEqual(output, self.expected_reset)
         self.assertEqual(encoder.encode('', final=True), b'')
 
+    def test_streamwriter_reset(self):
+        stream = io.BytesIO()
+        writer = codecs.getwriter(self.encoding)(stream)
+        writer.reset()
+        self.assertEqual(stream.getvalue(), b'')
+        writer.write(self.text)
+        self.assertEqual(stream.getvalue(), self.expected)
+        writer.reset()
+        self.assertEqual(stream.getvalue(), self.expected_reset)
+        writer.reset()
+        self.assertEqual(stream.getvalue(), self.expected_reset)
+        writer.write(self.text)
+        writer.reset()
+        self.assertEqual(stream.getvalue(), self.expected_reset * 2)
+
+    def test_streamwriter_seek0(self):
+        # The reset sequence is written before the stream is rewound.
+        info = codecs.lookup(self.encoding)
+        for wrap in (
+            info.streamwriter,
+            lambda stream: codecs.StreamReaderWriter(
+                stream, info.streamreader, info.streamwriter),
+            lambda stream: codecs.EncodedFile(stream, 'utf-8', self.encoding),
+        ):
+            stream = io.BytesIO()
+            writer = wrap(stream)
+            text = self.text
+            if isinstance(writer, codecs.StreamRecoder):
+                text = text.encode()
+            writer.write(text)
+            writer.seek(0)
+            self.assertEqual(stream.tell(), 0)
+            self.assertEqual(stream.getvalue(), self.expected_reset)
+            writer.write(text)
+            self.assertEqual(stream.getvalue(), self.expected_reset)
+
     def test_incrementalencoder_error(self):
         # The output of a failed encode() is lost, so the shift state
         # must not be changed.

Lib/test/test_tarfile.py

@@ -4526,6 +4526,33 @@ def test_sneaky_hardlink_fallback_deep(self):
                     self.expect_file("a/b/s", symlink_to=os.path.join('..', 'escape'))
                     self.expect_file("s", symlink_to=os.path.join('..', 'escape'))
 
+    @symlink_test
+    def test_sneaky_hardlink_fallback_attrs(self):
+        # Test that when a hardlink is extracted as a copy of a symlink
+        # member, the hardlink's attributes are not applied to the file
+        # the symlink points to.
+        with ArchiveMaker() as arc:
+            arc.add("a/s", symlink_to=os.path.join("..", "flag"))
+            arc.add("s", hardlink_to=os.path.join("a", "s"),
+                    mode='-rwxr-xr-x', mtime=86400, uid=1337, gid=1337)
+
+        for filter in 'tar', 'fully_trusted':
+            with (
+                self.subTest(filter),
+                unittest.mock.patch("os.chown", create=True) as mock_chown,
+                self.check_context(arc.open(), filter) as cc,
+            ):
+                if not os_helper.can_symlink():
+                    self.expect_file("a/")
+                else:
+                    self.expect_file("a/s", symlink_to=os.path.join('..', 'flag'))
+                    self.expect_file("s", symlink_to=os.path.join('..', 'flag'))
+                st = (cc.outerdir / 'flag').stat()
+                self.assertNotEqual(st.st_mtime, 86400)
+                if sys.platform != "win32":
+                    self.assertNotEqual(st.st_mode & 0o777, 0o755)
+                mock_chown.assert_not_called()
+
     @symlink_test
     @os_helper.skip_unless_hardlink
     def test_sneaky_hardlink_relocation(self):

Lib/test/test_zipfile/test_core.py

@@ -2132,6 +2132,20 @@ def test_is_zip_valid_file(self):
         self.assertTrue(zipfile.is_zipfile(fp))
         self.assertEqual(fp.tell(), mid)
 
+    def test_is_zip_bad_central_directory_offset(self):
+        """Check that is_zipfile() rejects a central directory before the file."""
+        fp = io.BytesIO()
+        with zipfile.ZipFile(fp, mode="w") as zipf:
+            zipf.writestr("foo.txt", b"O, for a Muse of Fire!")
+        data = bytearray(fp.getvalue())
+        endrec = len(data) - zipfile.sizeEndCentDir
+        struct.pack_into("<L", data, endrec + 12, len(data))
+        fp = io.BytesIO(data)
+        pos = fp.seek(5)
+        self.assertFalse(zipfile.is_zipfile(fp))
+        self.assertEqual(fp.tell(), pos)
+        self.assertRaises(zipfile.BadZipFile, zipfile.ZipFile, io.BytesIO(data))
+
     def test_non_existent_file_raises_OSError(self):
         # make sure we don't raise an AttributeError when a partially-constructed
         # ZipFile instance is finalized; this tests for regression on SF tracker