gh-156939: Detect buffer overflow in bytes and bytearray

When Python is built in debug mode, bytes an bytearray destructors
now check if the trailing null byte has been overridden to detect
overflow. Add bytes_dealloc() to implement the check.

Author: vstinner

Lib/test/test_capi/test_bytearray.py

@@ -1,6 +1,9 @@
 import sys
+import textwrap
 import unittest
+from test import support
 from test.support import import_helper
+from test.support.script_helper import assert_python_failure
 
 _testlimitedcapi = import_helper.import_module('_testlimitedcapi')
 from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX
@@ -172,6 +175,24 @@ def test_resize(self):
         # CRASHES resize(object(), 0)
         # CRASHES resize(NULL, 0)
 
+    @unittest.skipUnless(support.Py_DEBUG, 'need debug build (Py_DEBUG)')
+    def test_detect_overflow(self):
+        # Test detection of buffer overflow
+        size = 123
+        code = textwrap.dedent(f'''
+            from test.support import SuppressCrashReport
+            import _testcapi
+
+            size = {size}
+            with SuppressCrashReport():
+                # Trigger a buffer overflow in a new bytearray
+                ba = _testcapi.bytearray_overflow(size)
+                ba = None
+        ''')
+        proc = assert_python_failure('-c', code)
+        self.assertIn(b'Buffer overflow detected in bytearray object', proc.err)
+        self.assertIn(f'at position {size}'.encode(), proc.err)
+
 
 if __name__ == "__main__":
     unittest.main()

Lib/test/test_capi/test_bytes.py

@@ -317,6 +317,24 @@ def test_join(self):
         with self.assertRaises(SystemError):
             bytes_join(b'', NULL)
 
+    @unittest.skipUnless(support.Py_DEBUG, 'need debug build (Py_DEBUG)')
+    def test_detect_overflow(self):
+        # Test detection of buffer overflow
+        size = 123
+        code = textwrap.dedent(f'''
+            from test.support import SuppressCrashReport
+            import _testcapi
+
+            size = {size}
+            with SuppressCrashReport():
+                # Trigger a buffer overflow in a new bytes
+                ba = _testcapi.bytes_overflow(size)
+                ba = None
+        ''')
+        proc = assert_python_failure('-c', code)
+        self.assertIn(b'Buffer overflow detected in bytes object', proc.err)
+        self.assertIn(f'at position {size}'.encode(), proc.err)
+
 
 def get_data_canary(writer):
     size = writer.get_size() + 1

Misc/NEWS.d/next/Core_and_Builtins/2026-09-14-23-16-26.gh-issue-156939.oZqlSt.rst

@@ -0,0 +1,3 @@
+When Python is built in debug mode, :class:`bytes` and :class:`bytearray`
+destructors now check if the trailing null byte has been overridden to detect
+buffer overflow. Patch by Victor Stinner.

Modules/_testcapi/bytes.c

@@ -528,13 +528,46 @@ test_byteswriter_ptr(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args))
 }
 
 
+static PyObject *
+bytes_overflow(PyObject *Py_UNUSED(module), PyObject *arg)
+{
+    PyObject *bytes = PyObject_CallOneArg((PyObject*)&PyBytes_Type, arg);
+    if (bytes == NULL) {
+        return NULL;
+    }
+
+    char *data = PyBytes_AS_STRING(bytes);
+    Py_ssize_t size = PyBytes_GET_SIZE(bytes);
+    memset(data, 'x', size);
+    data[size] = '#';  // Buffer overflow!
+    return bytes;
+}
+
+
+static PyObject *
+bytearray_overflow(PyObject *Py_UNUSED(module), PyObject *arg)
+{
+    PyObject *bytearray = PyObject_CallOneArg((PyObject*)&PyByteArray_Type, arg);
+    if (bytearray == NULL) {
+        return NULL;
+    }
+
+    char *data = PyByteArray_AS_STRING(bytearray);
+    Py_ssize_t size = PyByteArray_GET_SIZE(bytearray);
+    data[size] = '#';  // Buffer overflow!
+    return bytearray;
+}
+
+
 static PyMethodDef test_methods[] = {
     {"bytes_resize", bytes_resize, METH_VARARGS},
     {"bytes_join", bytes_join, METH_VARARGS},
     {"byteswriter_abc", byteswriter_abc, METH_NOARGS},
     {"byteswriter_resize", byteswriter_resize, METH_NOARGS},
     {"byteswriter_highlevel", byteswriter_highlevel, METH_NOARGS},
     {"test_byteswriter_ptr", test_byteswriter_ptr, METH_NOARGS},
+    {"bytes_overflow", bytes_overflow, METH_O},
+    {"bytearray_overflow", bytearray_overflow, METH_O},
     {NULL},
 };
 

Modules/_testcapi/mem.c

@@ -448,13 +448,15 @@ test_pyobject_new(PyObject *self, PyObject *Py_UNUSED(ignored))
     if (obj == NULL) {
         goto alloc_failed;
     }
+    memset(PyBytes_AS_STRING(obj), 0, 3 + 1);  // +1 for the null byte
     Py_DECREF(obj);
 
     // PyObject_NEW_VAR()
     obj = PyObject_NEW_VAR(PyObject, var_type, 3);
     if (obj == NULL) {
         goto alloc_failed;
     }
+    memset(PyBytes_AS_STRING(obj), 0, 3 + 1);  // +1 for the null byte
     Py_DECREF(obj);
 
     Py_RETURN_NONE;

Objects/bytearrayobject.c

@@ -1272,6 +1272,21 @@ static void
 bytearray_dealloc(PyObject *op)
 {
     PyByteArrayObject *self = _PyByteArray_CAST(op);
+
+#ifdef Py_DEBUG
+    // Make sure that the trailing null byte was not modified
+    if (self->ob_bytes_object != NULL) {
+        char *data = PyByteArray_AS_STRING(self);
+        Py_ssize_t size = PyByteArray_GET_SIZE(self);
+        if (data[size] != '\0') {
+            _Py_FatalErrorFormat(__func__,
+                                 "Buffer overflow detected in bytearray "
+                                 "object %p at position %zd",
+                                 self, size);
+        }
+    }
+#endif
+
     if (self->ob_exports > 0) {
         PyErr_SetString(PyExc_SystemError,
                         "deallocated bytearray object has exported buffers");

Objects/bytesobject.c

@@ -3191,12 +3191,36 @@ bytes_iteritem(PyObject *obj, Py_ssize_t index)
     return (_PyObjectIndexPair) { .object = l, .index = index + 1 };
 }
 
+#ifdef Py_DEBUG
+static void
+bytes_dealloc(PyObject *op)
+{
+    // Make sure that the trailing null byte was not modified
+    PyBytesObject *self = _PyBytes_CAST(op);
+    char *data = PyBytes_AS_STRING(self);
+    Py_ssize_t size = PyBytes_GET_SIZE(self);
+    if (data[size] != '\0') {
+        _Py_FatalErrorFormat(__func__,
+                             "Buffer overflow detected in bytes object %p "
+                             "at position %zd",
+                             self, size);
+    }
+
+    Py_TYPE(self)->tp_free((PyObject *)self);
+}
+#endif
+
+
 PyTypeObject PyBytes_Type = {
     PyVarObject_HEAD_INIT(&PyType_Type, 0)
     "bytes",
     PyBytesObject_SIZE,
     sizeof(char),
+#ifdef Py_DEBUG
+    bytes_dealloc,                              /* tp_dealloc */
+#else
     0,                                          /* tp_dealloc */
+#endif
     0,                                          /* tp_vectorcall_offset */
     0,                                          /* tp_getattr */
     0,                                          /* tp_setattr */
@@ -3665,6 +3689,20 @@ byteswriter_write_canary_byte(PyBytesWriter *writer)
     unsigned char *data = (unsigned char*)byteswriter_data(writer);
     data[writer->size] = PyBytesWriter_CANARY_BYTE;
 }
+
+
+static void
+byteswriter_reset_trailing_byte(PyBytesWriter *writer)
+{
+    if (writer->obj != NULL) {
+        // PyBytesArray writes non-zero canary byte as the last byte.
+        // bytes/bytearray expects the last byte to be a null byte.
+        // Reset the last byte to null for bytes/bytearray.
+        Py_ssize_t allocated = byteswriter_allocated(writer);
+        char *data = byteswriter_data(writer);
+        data[allocated] = '\0';
+    }
+}
 #endif
 
 
@@ -3814,6 +3852,7 @@ PyBytesWriter_Discard(PyBytesWriter *writer)
 
 #ifdef Py_DEBUG
     byteswriter_check_canary_byte(writer);
+    byteswriter_reset_trailing_byte(writer);
 #endif
 
     Py_XDECREF(writer->obj);
@@ -3838,16 +3877,8 @@ PyBytesWriter_FinishWithSize(PyBytesWriter *writer, Py_ssize_t size)
     }
 
 #ifdef Py_DEBUG
-    // Check for buffer overflow
     byteswriter_check_canary_byte(writer);
-
-    if (writer->obj != NULL) {
-        // byteswriter_write_canary_byte() can override the trailing NUL byte.
-        // So reset the trailing NUL byte to NUL.
-        Py_ssize_t allocated = byteswriter_allocated(writer);
-        char *data = byteswriter_data(writer);
-        data[allocated] = '\0';
-    }
+    byteswriter_reset_trailing_byte(writer);
 #endif
 
     PyObject *result;