Skip to content

gh-155742: Use PyBytesWriter in CJK codecs #157535

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vstinner merged 2 commits into from
Sep 15, 2026
+17 −23
Merged

gh-155742: Use PyBytesWriter in CJK codecs #157535

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
48b4368
gh-155742: Use PyBytesWriter in CJK codecs
vstinner Sep 15, 2026
18333b6
Remove redundant cast
vstinner Sep 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 17 additions & 23 deletions Modules/cjkcodecs/multibytecodec.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,8 @@ typedef struct {
PyObject *inobj;
Py_ssize_t inpos, inlen;
unsigned char *outbuf, *outbuf_end;
PyObject *excobj, *outobj;
PyObject *excobj;
PyBytesWriter *writer;
} MultibyteEncodeBuffer;

typedef struct {
Expand Down Expand Up @@ -209,21 +210,21 @@ expand_encodebuffer(MultibyteEncodeBuffer *buf, Py_ssize_t esize)
Py_ssize_t orgpos, orgsize, incsize;

orgpos = (Py_ssize_t)((char *)buf->outbuf -
PyBytes_AS_STRING(buf->outobj));
orgsize = PyBytes_GET_SIZE(buf->outobj);
(char *)PyBytesWriter_GetData(buf->writer));
orgsize = PyBytesWriter_GetSize(buf->writer);
incsize = (esize < (orgsize >> 1) ? (orgsize >> 1) | 1 : esize);

if (orgsize > PY_SSIZE_T_MAX - incsize) {
PyErr_NoMemory();
return -1;
}

if (_PyBytes_Resize(&buf->outobj, orgsize + incsize) == -1)
if (PyBytesWriter_Resize(buf->writer, orgsize + incsize) == -1)
return -1;

buf->outbuf = (unsigned char *)PyBytes_AS_STRING(buf->outobj) +orgpos;
buf->outbuf_end = (unsigned char *)PyBytes_AS_STRING(buf->outobj)
+ PyBytes_GET_SIZE(buf->outobj);
unsigned char *data = PyBytesWriter_GetData(buf->writer);
buf->outbuf = data + orgpos;
buf->outbuf_end = data + PyBytesWriter_GetSize(buf->writer);

return 0;
}
Expand Down Expand Up @@ -503,18 +504,18 @@ multibytecodec_encode(const MultibyteCodec *codec,
PyObject *errors, int flags)
{
MultibyteEncodeBuffer buf;
Py_ssize_t finalsize, r = 0;
Py_ssize_t r = 0;
Py_ssize_t datalen;
int kind;
const void *data;

datalen = PyUnicode_GET_LENGTH(text);

if (datalen == 0 && !(flags & MBENC_RESET))
return PyBytes_FromStringAndSize(NULL, 0);
return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES);

buf.excobj = NULL;
buf.outobj = NULL;
buf.writer = NULL;
buf.inobj = text; /* borrowed reference */
buf.inpos = 0;
buf.inlen = datalen;
Expand All @@ -526,11 +527,11 @@ multibytecodec_encode(const MultibyteCodec *codec,
goto errorexit;
}

buf.outobj = PyBytes_FromStringAndSize(NULL, datalen * 2 + 16);
if (buf.outobj == NULL)
buf.writer = PyBytesWriter_Create(datalen * 2 + 16);
if (buf.writer == NULL)
goto errorexit;
buf.outbuf = (unsigned char *)PyBytes_AS_STRING(buf.outobj);
buf.outbuf_end = buf.outbuf + PyBytes_GET_SIZE(buf.outobj);
buf.outbuf = (unsigned char *)PyBytesWriter_GetData(buf.writer);
buf.outbuf_end = buf.outbuf + PyBytesWriter_GetSize(buf.writer);

while (buf.inpos < buf.inlen) {
/* we don't reuse inleft and outleft here.
Expand Down Expand Up @@ -563,21 +564,14 @@ multibytecodec_encode(const MultibyteCodec *codec,
goto errorexit;
}

finalsize = (Py_ssize_t)((char *)buf.outbuf -
PyBytes_AS_STRING(buf.outobj));

if (finalsize != PyBytes_GET_SIZE(buf.outobj))
if (_PyBytes_Resize(&buf.outobj, finalsize) == -1)
goto errorexit;

if (inpos_t)
*inpos_t = buf.inpos;
Py_XDECREF(buf.excobj);
return buf.outobj;
return PyBytesWriter_FinishWithPointer(buf.writer, buf.outbuf);

errorexit:
Py_XDECREF(buf.excobj);
Py_XDECREF(buf.outobj);
PyBytesWriter_Discard(buf.writer);
return NULL;
}

Expand Down
Loading