Merge branch 'main' into is_immortal

Author: vstinner

Doc/c-api/bytes.rst

@@ -231,6 +231,7 @@ called with a non-bytes parameter.
    Resize a bytes object. *newsize* will be the new length of the bytes object.
    You can think of it as creating a new bytes object and destroying the old
    one, only more efficiently.
+
    Pass the address of an
    existing bytes object as an lvalue (it may be written into), and the new size
    desired.  On success, *\*bytes* holds the resized bytes object and ``0`` is
@@ -239,6 +240,11 @@ called with a non-bytes parameter.
    *\*bytes* is set to ``NULL``, :exc:`MemoryError` is set, and ``-1`` is
    returned.
 
+   While bytes objects are usually immutable in Python, this special C API
+   allows mutating a bytes object in-place. The returned bytes object can still
+   be mutated using :c:func:`PyBytesWriter_GetData`; except if *newsize* is
+   zero in which case it returns the immutable empty bytes string.
+
    .. soft-deprecated:: 3.15
       Use the :c:type:`PyBytesWriter` API instead.
 
@@ -290,10 +296,10 @@ object.
 
 .. c:type:: PyBytesWriter
 
-   A bytes writer instance.
+   A bytes writer object.
 
-   The API is **not thread safe**: a writer should only be used by a single
-   thread at the same time.
+   The API is **not thread safe**. A :c:type:`PyBytesWriter` object must only
+   be used by a single thread, it must not be shared between threads.
 
    The instance must be destroyed by :c:func:`PyBytesWriter_Finish` on
    success, or :c:func:`PyBytesWriter_Discard` on error.
@@ -429,7 +435,7 @@ Low-level API
    On success, return ``0``.
    On error, set an exception and return ``-1``.
 
-   *size* can be negative to shrink the writer.
+   *grow* can be negative to shrink the writer.
 
 .. c:function:: void* PyBytesWriter_GrowAndUpdatePointer(PyBytesWriter *writer, Py_ssize_t size, void *buf)
 

Doc/library/json.rst

@@ -261,6 +261,8 @@ Basic Usage
       into JSON and then back into a dictionary, the dictionary may not equal
       the original one. That is, ``loads(dumps(x)) != x`` if x has non-string
       keys.
+      *sort_keys* sorts the keys before they are coerced to strings,
+      so numeric keys are sorted by value, not by their string representation.
 
 .. function:: load(fp, *, cls=None, object_hook=None, parse_float=None, \
                    parse_int=None, parse_constant=None, \

Doc/library/tk.rst

@@ -34,7 +34,7 @@ alternative `GUI frameworks and tools <https://wiki.python.org/moin/GuiProgrammi
    tkinter.colorchooser.rst
    tkinter.font.rst
    tkinter.fontchooser.rst
-   dialog.rst
+   tkinter.dialogs.rst
    tkinter.messagebox.rst
    tkinter.scrolledtext.rst
    tkinter.systray.rst

Doc/library/dialog.rst Doc/library/tkinter.dialogs.rstDoc/library/dialog.rst renamed to Doc/library/tkinter.dialogs.rst

@@ -82,3 +82,6 @@ reference/expressions.html: generator.__next__
 reference/expressions.html: generator.close
 reference/expressions.html: generator.send
 reference/expressions.html: generator.throw
+
+# Renamed to library/tkinter.dialogs.html
+library/dialog.html: (page missing)

Doc/tools/removed-ids.txt

@@ -77,6 +77,10 @@ PyAPI_FUNC(PyObject *) _PyBytes_Repeat(PyObject *self, Py_ssize_t n);
 
 extern int _PyBytes_ResizeKeepOnError(PyObject **pv, Py_ssize_t newsize);
 
+#ifndef NDEBUG
+extern int _PyBytes_IsMutable(PyObject *obj);
+#endif
+
 /* --- PyBytesWriter ------------------------------------------------------ */
 
 struct PyBytesWriter {

Include/internal/pycore_bytesobject.h

@@ -325,7 +325,9 @@ def set_width(self):
         # http://www.tcl.tk/man/tcl8.6/TkCmd/text.htm#M21
         zero_char_width = \
             Font(text, font=text.cget('font')).measure('0')
-        self.width = pixel_width // zero_char_width
+        # Some fonts report a zero width for '0' (gh-90304).
+        self.width = (pixel_width // zero_char_width if zero_char_width
+                      else text.tk.getint(text.cget('width')))
 
     def new_callback(self, event):
         dirname, basename = self.io.defaultfilename()

Lib/idlelib/editor.py

@@ -3,6 +3,7 @@
 from idlelib import editor
 import unittest
 from collections import namedtuple
+from unittest import mock
 from test.support import requires
 from tkinter import Tk, Text
 
@@ -30,6 +31,18 @@ def test_init(self):
         self.assertEqual(e.root, self.root)
         e._close()
 
+    def test_set_width_zero_char_width(self):
+        # A zero-width '0' must not raise ZeroDivisionError (gh-90304).
+        e = Editor(root=self.root)
+        try:
+            with mock.patch.object(editor, 'Font') as MockFont:
+                MockFont.return_value.measure.return_value = 0
+                e.set_width()
+            self.assertEqual(e.width,
+                             e.text.tk.getint(e.text.cget('width')))
+        finally:
+            e._close()
+
 
 class GetLineIndentTest(unittest.TestCase):
     def test_empty_lines(self):

Lib/idlelib/idle_test/test_editor.py

@@ -1869,7 +1869,7 @@ def nested():
         self.assertEqual(type_repr(t'''{ 0
             & 1
             | 2
-        }'''), 't"""{ 0\n            & 1\n            | 2}"""')
+        }'''), 't"""{ 0\n            & 1\n            | 2\n        }"""')
         self.assertEqual(
             type_repr(Template("hi", Interpolation(42, "42"))), "t'hi{42}'"
         )

Lib/test/test_annotationlib.py

@@ -1,7 +1,9 @@
 import sys
+import textwrap
 import unittest
 from test import support
 from test.support import import_helper
+from test.support.script_helper import assert_python_failure
 
 _testlimitedcapi = import_helper.import_module('_testlimitedcapi')
 _testcapi = import_helper.import_module('_testcapi')
@@ -316,12 +318,18 @@ def test_join(self):
             bytes_join(b'', NULL)
 
 
+def get_data_canary(writer):
+    size = writer.get_size() + 1
+    return writer.get_data(size)
+
+
 class BaseWriterTest:
     RESULT_TYPE = NotImplementedError
     SMALL_BUFFER = 11  # bytes
     assert SMALL_BUFFER < _testcapi.PyBytesWriter_small_buffer
     LARGE_BUFFER = _testcapi.PyBytesWriter_small_buffer + 17  # bytes
     NEW_BYTE = b'\xff'
+    CANARY_BYTE = b'\xdd'
 
     def create_writer(self, alloc=0, string=b''):
         raise NotImplementedError
@@ -344,6 +352,7 @@ def test_get_data(self):
         # Test PyBytesWriter_GetData()
         writer = self.create_writer(6)
         NEW_BYTE = self.NEW_BYTE
+        CANARY_BYTE = self.CANARY_BYTE
         self.assertEqual(writer.get_data(), NEW_BYTE * 6)
         writer.write(0, b'abc')
         self.assertEqual(writer.get_data(), b'abc' + NEW_BYTE * 3)
@@ -357,7 +366,7 @@ def test_get_data(self):
         writer.write(0, b's' * small)
         self.assertEqual(writer.get_data(), b's' * small)
         writer.resize(large)
-        self.assertEqual(writer.get_data(), b's' * small + NEW_BYTE * (large - small))
+        self.assertEqual(writer.get_data(), b's' * small + CANARY_BYTE + NEW_BYTE * (large - small - 1))
         writer.write(small, b'L' * (large - small))
         self.assertEqual(writer.get_data(), b's' * small + b'L' * (large - small))
 
@@ -443,6 +452,47 @@ def test_resize(self):
                     writer.resize(_testcapi.PY_SSIZE_T_MAX)
                 self.assertEqual(writer.finish(), b'x' * size)
 
+    @unittest.skipUnless(support.Py_DEBUG, 'need debug build')
+    def test_resize_canary(self):
+        CANARY_BYTE = self.CANARY_BYTE
+        for size in (self.SMALL_BUFFER, self.LARGE_BUFFER):
+            with self.subTest(size=size):
+                # Truncate the last byte
+                data = b'x' * size
+                writer = self.create_writer(size)
+                writer.write(0, data)
+                self.assertEqual(get_data_canary(writer), data + CANARY_BYTE)
+                writer.resize(size - 1)
+                self.assertEqual(get_data_canary(writer), data[:-1] + CANARY_BYTE)
+                self.assertEqual(writer.finish(),  data[:-1])
+
+                # Make the buffer empty
+                writer = self.create_writer(size)
+                writer.write(0, data)
+                writer.resize(0)
+                self.assertEqual(writer.get_data(), b'')
+                self.assertEqual(writer.finish(),  b'')
+
+    @support.nomemtest
+    def test_resize_error(self):
+        # Test PyBytesWriter_Resize() error
+        init = b'x' * self.LARGE_BUFFER
+        writer = self.create_writer(len(init))
+        writer.write(0, init)
+        size = len(init) + 100
+        try:
+            with self.assertRaises(MemoryError):
+                _testcapi.set_nomemory(0)
+                writer.resize(size)
+        finally:
+            _testcapi.remove_mem_hooks()
+        suffix = b'still working'
+        writer.write_bytes(suffix, -1)
+        self.assertEqual(writer.finish(), init + suffix)
+
+        # Note: PyBytesWriter_Resize() leaves the buffer unchanged (no resize)
+        # if the new size is smaller than the allocated size
+
     def test_grow(self):
         # Test PyBytesWriter_Grow()
         writer = self.create_writer(0)
@@ -476,31 +526,51 @@ def test_grow(self):
             with self.subTest(size=size):
                 writer = self.create_writer()
                 writer.write_bytes(b'x' * size, -1)
-                with self.assertRaisesRegex(ValueError, 'size must be >= 0'):
-                    writer.grow(-1)
+                with self.assertRaisesRegex(ValueError, 'invalid size'):
+                    writer.grow(-size - 1)
                 with self.assertRaises(MemoryError):
                     writer.grow(_testcapi.PY_SSIZE_T_MAX)
                 self.assertEqual(writer.finish(), b'x' * size)
 
+    @unittest.skipUnless(support.Py_DEBUG, 'need debug build')
+    def test_grow_canary(self):
+        CANARY_BYTE = self.CANARY_BYTE
+        for size in (self.SMALL_BUFFER, self.LARGE_BUFFER):
+            with self.subTest(size=size):
+                # Truncate the last byte
+                data = b'x' * size
+                writer = self.create_writer(size)
+                writer.write(0, data)
+                self.assertEqual(get_data_canary(writer), data + CANARY_BYTE)
+                writer.grow(-1)
+                self.assertEqual(get_data_canary(writer), data[:-1] + CANARY_BYTE)
+                self.assertEqual(writer.finish(),  data[:-1])
+
+                # Make the buffer empty
+                writer = self.create_writer(size)
+                writer.write(0, data)
+                writer.grow(-size)
+                self.assertEqual(writer.get_data(), b'')
+                self.assertEqual(writer.finish(),  b'')
+
     @support.nomemtest
-    def test_resize_error(self):
-        # Test PyBytesWriter_Resize() error
+    def test_grow_error(self):
+        # Test PyBytesWriter_Grow() error
         init = b'x' * self.LARGE_BUFFER
         writer = self.create_writer(len(init))
         writer.write(0, init)
-        size = len(init) + 100
         try:
             with self.assertRaises(MemoryError):
                 _testcapi.set_nomemory(0)
-                writer.resize(size)
+                writer.grow(100)
         finally:
             _testcapi.remove_mem_hooks()
         suffix = b'still working'
         writer.write_bytes(suffix, -1)
         self.assertEqual(writer.finish(), init + suffix)
 
-        # Note: PyBytesWriter_Resize() leaves the buffer unchanged (no resize)
-        # if the new size is smaller than the allocated size
+        # Note: PyBytesWriter_Grow() leaves the buffer unchanged (no resize)
+        # if grow is negative.
 
     def test_format_i(self):
         # Test PyBytesWriter_Format()
@@ -513,6 +583,67 @@ def test_format_i(self):
         writer.format_i(b'y=%i', 456)
         self.assertEqual(writer.finish(), b'x=123, y=456')
 
+    @unittest.skipUnless(support.Py_DEBUG, 'need a Python debug build')
+    def test_canary_byte(self):
+        small_buffer = _testcapi.PyBytesWriter_small_buffer
+        large_size = small_buffer * 10
+        use_bytearray = (self.RESULT_TYPE == bytearray)
+
+        # Test small buffer and large buffer
+        for size in (0, self.SMALL_BUFFER, self.LARGE_BUFFER):
+            for operation in (
+                'writer.get_data()',
+                'writer.get_size()',
+                f'writer.resize({size} * 2)',
+                f'writer.grow({size})',
+                'writer.discard()',
+                'writer.finish()',
+            ):
+                with self.subTest(size=size, operation=operation):
+                    code = textwrap.dedent(f"""
+                        from test.support import SuppressCrashReport
+                        import os
+                        import _testcapi
+                        size = {size}
+                        # Add an extra '#' byte to trigger a buffer overflow
+                        data = b'x' * size + b'#'
+                        use_bytearray = {use_bytearray}
+                        writer = _testcapi.PyBytesWriter(size, use_bytearray)
+                        with SuppressCrashReport():
+                            writer.write(0, data, check=False)
+                            try:
+                                {operation}
+                            except:
+                                # Ignore all exceptions
+                                pass
+                            # If we reached this line, the operation didn't
+                            # detect the overflow. Exit immediatetly without
+                            # calling the writer destructor since it can detect
+                            # the overflow.
+                            os._exit(0)
+                    """)
+                    proc = assert_python_failure('-c', code)
+                    self.assertIn(b'Buffer overflow detected in PyBytesWriter',
+                                  proc.err)
+                    self.assertIn(f'at position {size}'.encode(),
+                                  proc.err)
+
+    @unittest.skipUnless(support.Py_DEBUG, 'need debug build')
+    def test_get_data_canary(self):
+        # Test PyBytesWriter_GetData()
+        NEW_BYTE = self.NEW_BYTE
+        CANARY_BYTE = self.CANARY_BYTE
+
+        writer = self.create_writer(6)
+        self.assertEqual(get_data_canary(writer),
+                         NEW_BYTE * 6 + CANARY_BYTE)
+        writer.write(0, b'abc')
+        self.assertEqual(get_data_canary(writer),
+                         b'abc' + NEW_BYTE * 3 + CANARY_BYTE)
+        writer.write(3, b'123')
+        self.assertEqual(get_data_canary(writer),
+                         b'abc123' + CANARY_BYTE)
+
 
 class BytesWriterTest(BaseWriterTest, unittest.TestCase):
     RESULT_TYPE = bytes