From 988e830dde9992871e7995b4b8e3a71522668653 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 27 Oct 2026 16:15:30 -0700 Subject: [PATCH 1/3] gh-71448: Split BytesIO resizing into realloc and prepare-write helpers realloc_buffer_lock_held() reallocates to an exact size, unsharing if needed; prepare_write_lock_held() ensures a private buffer of at least the given size with the existing overallocation policy. Callers no longer open-code the shared/unshared choice. No behavior change. Co-Authored-By: Claude Opus 5.5 --- Modules/_io/bytesio.c | 77 +++++++++++++++++++------------------------ 1 file changed, 33 insertions(+), 44 deletions(-) diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index 2571da4538363f1..4701c12f26c6243 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -185,11 +185,19 @@ unshare_buffer_lock_held(bytesio *self, size_t size) return 0; } -/* Internal routine for changing the size of the buffer of BytesIO objects. - The caller should ensure that the 'size' argument is non-negative. Returns - 0 on success, -1 otherwise. */ +/* Reallocate the buffer of a BytesIO object to exactly 'alloc' bytes, keeping + the data and unsharing it if needed. Returns 0 on success, -1 otherwise. */ static int -resize_buffer_lock_held(bytesio *self, size_t size) +realloc_buffer_lock_held(bytesio *self, size_t alloc) +{ + return SHARED_BUF(self) ? unshare_buffer_lock_held(self, alloc) + : resize_unshared_buffer_lock_held(self, alloc); +} + +/* Make the buffer of a BytesIO object private and at least 'size' bytes long, + keeping the first string_size bytes. Returns 0 on success, -1 otherwise. */ +static int +prepare_write_lock_held(bytesio *self, size_t size) { _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); @@ -205,13 +213,12 @@ resize_buffer_lock_held(bytesio *self, size_t size) if (size > PY_SSIZE_T_MAX) goto overflow; - if (size < alloc / 2) { - /* Major downsize; resize down to exact size. */ - alloc = size + 1; - } - else if (size < alloc) { - /* Within allocated size; quick exit */ - return 0; + if (size <= alloc) { + /* Within allocated size; quick exit unless shared */ + if (!SHARED_BUF(self)) { + return 0; + } + alloc = Py_MAX(size, (size_t)self->string_size); } else if (size <= alloc * 1.125) { /* Moderate upsize; overallocate similar to list_resize() */ @@ -225,16 +232,7 @@ resize_buffer_lock_held(bytesio *self, size_t size) goto overflow; } - if (SHARED_BUF(self)) { - if (unshare_buffer_lock_held(self, alloc) < 0) - return -1; - } - else { - if (resize_unshared_buffer_lock_held(self, alloc) < 0) - return -1; - } - - return 0; + return realloc_buffer_lock_held(self, alloc); overflow: PyErr_SetString(PyExc_OverflowError, @@ -269,17 +267,9 @@ write_bytes_lock_held(bytesio *self, PyObject *b) assert(self->pos >= 0); size_t endpos = (size_t)self->pos + len; - if (endpos > (size_t)PyBytes_GET_SIZE(self->buf)) { - if (resize_buffer_lock_held(self, endpos) < 0) { - len = -1; - goto done; - } - } - else if (SHARED_BUF(self)) { - if (unshare_buffer_lock_held(self, Py_MAX(endpos, (size_t)self->string_size)) < 0) { - len = -1; - goto done; - } + if (prepare_write_lock_held(self, endpos) < 0) { + len = -1; + goto done; } assert(_PyBytes_IsMutable(self->buf)); @@ -433,13 +423,8 @@ _io_BytesIO_getvalue_impl(bytesio *self) self->string_size); if (self->string_size != PyBytes_GET_SIZE(self->buf)) { - if (SHARED_BUF(self)) { - if (unshare_buffer_lock_held(self, self->string_size) < 0) - return NULL; - } - else { - if (resize_unshared_buffer_lock_held(self, self->string_size) < 0) - return NULL; + if (realloc_buffer_lock_held(self, self->string_size) < 0) { + return NULL; } } set_shared_buf(self); @@ -774,9 +759,12 @@ _io_BytesIO_truncate_impl(bytesio *self, PyObject *size) if (new_size < self->string_size) { Py_ssize_t old_string_size = self->string_size; self->string_size = new_size; - if (resize_buffer_lock_held(self, new_size) < 0) { - self->string_size = old_string_size; - return NULL; + if (new_size < PyBytes_GET_SIZE(self->buf) / 2) { + /* Major downsize; resize down to exact size. */ + if (realloc_buffer_lock_held(self, new_size + 1) < 0) { + self->string_size = old_string_size; + return NULL; + } } } @@ -1284,9 +1272,10 @@ bytesiobuf_getbuffer_lock_held(PyObject *op, Py_buffer *view, int flags) if (check_closed(b)) { return -1; } - if (FT_ATOMIC_LOAD_SSIZE_RELAXED(b->exports) == 0 && SHARED_BUF(b)) { - if (unshare_buffer_lock_held(b, b->string_size) < 0) + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(b->exports) == 0) { + if (prepare_write_lock_held(b, b->string_size) < 0) { return -1; + } } /* cannot fail if view != NULL and readonly == 0 */ From b0b473f0cc27630dcb94a3cd811a15e1e52a164d Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 27 Oct 2026 16:23:20 -0700 Subject: [PATCH 2/3] gh-71448: Support extension in BytesIO.truncate truncate() past the end now zero-fills, as IOBase.truncate() documents. A buffer shared with the caller is unshared first, even when the new size fits the current allocation. _pyio.StringIO inherits this through _pyio.BytesIO; the C StringIO does not extend. The two already disagree on truncate units (code points vs UTF-8 bytes). Co-Authored-By: Claude Opus 5.5 --- Doc/library/io.rst | 7 ++++ Lib/_pyio.py | 2 +- Lib/test/test_io/test_memoryio.py | 41 ++++++++++++++++--- ...6-06-25-22-43-34.gh-issue-71448.ajvMxg.rst | 2 + Modules/_io/bytesio.c | 19 +++++---- 5 files changed, 57 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-06-25-22-43-34.gh-issue-71448.ajvMxg.rst diff --git a/Doc/library/io.rst b/Doc/library/io.rst index 09e9e42565a99ef..ab7a68f77717902 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -786,6 +786,13 @@ than raw I/O does. .. versionadded:: 3.5 + .. method:: truncate(size=None, /) + + In :class:`BytesIO`, this is the same as :meth:`IOBase.truncate`. + + .. versionchanged:: next + Now extends the underlying buffer as :meth:`IOBase.truncate` documents. + .. class:: BufferedReader(raw, buffer_size=DEFAULT_BUFFER_SIZE) A buffered binary stream providing higher-level access to a readable, non diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 370b518f1e5e5a6..145a7cc564ce68e 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -1033,7 +1033,7 @@ def truncate(self, pos=None): pos = pos_index() if pos < 0: raise ValueError("negative truncate position %r" % (pos,)) - del self._buffer[pos:] + self._buffer.resize(pos) return pos def readable(self): diff --git a/Lib/test/test_io/test_memoryio.py b/Lib/test/test_io/test_memoryio.py index e8b47951b403c11..bcaf04663b8e5b8 100644 --- a/Lib/test/test_io/test_memoryio.py +++ b/Lib/test/test_io/test_memoryio.py @@ -584,6 +584,41 @@ def test_relative_seek(self): memio.seek(1, 1) self.assertEqual(memio.read(), buf[1:]) + def test_truncate_extend(self): + # gh-71448: Extending zero-fills, within the allocation or past it, + # and never writes into a buffer shared with the caller. + buf = self.buftype("abcdef") + private = self.ioclass() + private.write(buf) + for memio in self.ioclass(buf), private: + memio.seek(7) + self.assertEqual(memio.truncate(3), 3) + self.assertEqual(memio.truncate(5), 5) + self.assertEqual(memio.getvalue(), b"abc\0\0") + self.assertEqual(memio.truncate(), 7) + self.assertEqual(memio.getvalue(), b"abc\0\0\0\0") + self.assertEqual(memio.tell(), 7) + self.assertEqual(buf, b"abcdef") + + def test_truncate_extend_too_large(self): + for memio in self.ioclass(b"abc"), self.ioclass(): + memio.write(b"abc") + for size in sys.maxsize, sys.maxsize - 1: + with self.subTest(size=size): + self.assertRaises((MemoryError, OverflowError), + memio.truncate, size) + self.assertEqual(memio.getvalue(), b"abc") + self.assertEqual(memio.tell(), 3) + + @support.bigmemtest(size=support._2G, memuse=1, dry_run=False) + def test_truncate_extend_bigmem(self, size): + # Sizes above 2**31 - 1 must work where C long is 32-bit. + memio = self.ioclass(b"abc") + self.assertEqual(memio.truncate(size), size) + with memio.getbuffer() as buf: + self.assertEqual(len(buf), size) + self.assertEqual(buf[:4], b"abc\0") + def test_issue141311(self): memio = self.ioclass() # Seek allows PY_SSIZE_T_MAX, read should handle that. @@ -984,12 +1019,6 @@ def __index__(self): finally: size.result.release() - # Sizes above 2**31 - 1 must work where C long is 32-bit. - if sys.maxsize > 2**31: - stream = self.ioclass(b'abc') - self.assertEqual(stream.truncate(2**31), 2**31) - self.assertEqual(stream.getvalue(), b'abc') - def test_write_overflow(self): # gh-160781: overallocation at a position near sys.maxsize # must raise OverflowError, not SystemError. diff --git a/Misc/NEWS.d/next/Library/2026-06-25-22-43-34.gh-issue-71448.ajvMxg.rst b/Misc/NEWS.d/next/Library/2026-06-25-22-43-34.gh-issue-71448.ajvMxg.rst new file mode 100644 index 000000000000000..ae78aa2d0cbeed0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-25-22-43-34.gh-issue-71448.ajvMxg.rst @@ -0,0 +1,2 @@ +Update :class:`io.BytesIO` implementation of :meth:`io.IOBase.truncate` to +support extension on truncate to larger size than allocated. diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index 4701c12f26c6243..10ea56316aca9e2 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -164,8 +164,7 @@ scan_eol_lock_held(bytesio *self, Py_ssize_t len) } /* Internal routine for detaching the shared buffer of BytesIO objects. - The caller should ensure that the 'size' argument is non-negative and - not lesser than self->string_size. Returns 0 on success, -1 otherwise. */ + Data past 'size' is dropped. Returns 0 on success, -1 otherwise. */ static int unshare_buffer_lock_held(bytesio *self, size_t size) { @@ -174,12 +173,12 @@ unshare_buffer_lock_held(bytesio *self, size_t size) PyObject *new_buf; assert(SHARED_BUF(self)); assert(FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) == 0); - assert(size >= (size_t)self->string_size); + assert(size <= PY_SSIZE_T_MAX); new_buf = PyBytes_FromStringAndSize(NULL, size); if (new_buf == NULL) return -1; memcpy(PyBytes_AS_STRING(new_buf), PyBytes_AS_STRING(self->buf), - self->string_size); + Py_MIN(size, (size_t)self->string_size)); Py_SETREF(self->buf, new_buf); clear_shared_buf(self); return 0; @@ -757,16 +756,22 @@ _io_BytesIO_truncate_impl(bytesio *self, PyObject *size) } if (new_size < self->string_size) { - Py_ssize_t old_string_size = self->string_size; - self->string_size = new_size; if (new_size < PyBytes_GET_SIZE(self->buf) / 2) { /* Major downsize; resize down to exact size. */ if (realloc_buffer_lock_held(self, new_size + 1) < 0) { - self->string_size = old_string_size; return NULL; } } } + else if (new_size > self->string_size) { + if (prepare_write_lock_held(self, new_size) < 0) { + return NULL; + } + /* Fill new space with zeros */ + memset(PyBytes_AS_STRING(self->buf) + self->string_size, '\0', + new_size - self->string_size); + } + self->string_size = new_size; return PyLong_FromSsize_t(new_size); } From 9e0377c3a486fb5b8da04b8c4ac149a3c3f042ce Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Thu, 29 Oct 2026 14:59:52 -0700 Subject: [PATCH 3/3] gh-71448: Test BytesIO.truncate() failure on shared and private buffers The loop wrote to both streams first, so neither started shared, and which path the second size took depended on the build: getvalue() leaves the buffer shared on free-threading (sticky buf_shared) but not on the default build once its result is dropped. Use a fresh shared and private stream for each size. Co-Authored-By: Claude Opus 5.5 --- Lib/test/test_io/test_memoryio.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_io/test_memoryio.py b/Lib/test/test_io/test_memoryio.py index bcaf04663b8e5b8..5ee05c2ab5ce9b0 100644 --- a/Lib/test/test_io/test_memoryio.py +++ b/Lib/test/test_io/test_memoryio.py @@ -601,14 +601,14 @@ def test_truncate_extend(self): self.assertEqual(buf, b"abcdef") def test_truncate_extend_too_large(self): - for memio in self.ioclass(b"abc"), self.ioclass(): - memio.write(b"abc") - for size in sys.maxsize, sys.maxsize - 1: - with self.subTest(size=size): + for size in sys.maxsize, sys.maxsize - 1: + private = self.ioclass() + private.write(b"abc") + for memio in self.ioclass(b"abc"), private: + with self.subTest(size=size, shared=memio is not private): self.assertRaises((MemoryError, OverflowError), memio.truncate, size) self.assertEqual(memio.getvalue(), b"abc") - self.assertEqual(memio.tell(), 3) @support.bigmemtest(size=support._2G, memuse=1, dry_run=False) def test_truncate_extend_bigmem(self, size):