From 935200be3c421a559c833ac6e8cc26c6412e6037 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" <68491+gpshead@users.noreply.github.com> Date: Sat, 31 Oct 2026 23:44:02 -0700 Subject: [PATCH] gh-160185: Fix CJK StreamWriter.reset() not returning to ASCII (GH-160186) reset() returned early when nothing was pending, so the shift-back sequence of hz and iso2022_* was never written (regression from gh-67436). Also fix a reference leak of the stream.write() result. (cherry picked from commit aa4f541a9742fa1de1008d13cbf23b2eb3799e3a) Co-authored-by: Gregory P. Smith <68491+gpshead@users.noreply.github.com> --- Lib/codecs.py | 10 +++--- Lib/test/test_multibytecodec.py | 36 +++++++++++++++++++ ...-10-22-00-29-55.gh-issue-160185.BPz40E.rst | 5 +++ Modules/cjkcodecs/multibytecodec.c | 13 ++++--- 4 files changed, 56 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-10-22-00-29-55.gh-issue-160185.BPz40E.rst diff --git a/Lib/codecs.py b/Lib/codecs.py index e4a8010aba90a50..a43421e1ea05089 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -400,9 +400,10 @@ def reset(self): pass def seek(self, offset, whence=0): - self.stream.seek(offset, whence) + # reset() may write to the stream: call it at the old position. if whence == 0 and offset == 0: self.reset() + self.stream.seek(offset, whence) def __getattr__(self, name, getattr=getattr): @@ -739,10 +740,11 @@ def reset(self): self.writer.reset() def seek(self, offset, whence=0): - self.stream.seek(offset, whence) - self.reader.reset() + # writer.reset() may write to the stream: call it at the old position. if whence == 0 and offset == 0: self.writer.reset() + self.stream.seek(offset, whence) + self.reader.reset() def __getattr__(self, name, getattr=getattr): @@ -862,8 +864,8 @@ def reset(self): def seek(self, offset, whence=0): # Seeks must be propagated to both the readers and writers # as they might need to reset their internal buffers. - self.reader.seek(offset, whence) self.writer.seek(offset, whence) + self.reader.seek(offset, whence) def __getattr__(self, name, getattr=getattr): diff --git a/Lib/test/test_multibytecodec.py b/Lib/test/test_multibytecodec.py index 8d5eb78608afc71..ab7c63539c4fe48 100644 --- a/Lib/test/test_multibytecodec.py +++ b/Lib/test/test_multibytecodec.py @@ -438,6 +438,42 @@ def test_incrementalencoder_final(self): self.assertEqual(output, self.expected_reset) self.assertEqual(encoder.encode('', final=True), b'') + def test_streamwriter_reset(self): + stream = io.BytesIO() + writer = codecs.getwriter(self.encoding)(stream) + writer.reset() + self.assertEqual(stream.getvalue(), b'') + writer.write(self.text) + self.assertEqual(stream.getvalue(), self.expected) + writer.reset() + self.assertEqual(stream.getvalue(), self.expected_reset) + writer.reset() + self.assertEqual(stream.getvalue(), self.expected_reset) + writer.write(self.text) + writer.reset() + self.assertEqual(stream.getvalue(), self.expected_reset * 2) + + def test_streamwriter_seek0(self): + # The reset sequence is written before the stream is rewound. + info = codecs.lookup(self.encoding) + for wrap in ( + info.streamwriter, + lambda stream: codecs.StreamReaderWriter( + stream, info.streamreader, info.streamwriter), + lambda stream: codecs.EncodedFile(stream, 'utf-8', self.encoding), + ): + stream = io.BytesIO() + writer = wrap(stream) + text = self.text + if isinstance(writer, codecs.StreamRecoder): + text = text.encode() + writer.write(text) + writer.seek(0) + self.assertEqual(stream.tell(), 0) + self.assertEqual(stream.getvalue(), self.expected_reset) + writer.write(text) + self.assertEqual(stream.getvalue(), self.expected_reset) + def test_incrementalencoder_error(self): # The output of a failed encode() is lost, so the shift state # must not be changed. diff --git a/Misc/NEWS.d/next/Library/2026-10-22-00-29-55.gh-issue-160185.BPz40E.rst b/Misc/NEWS.d/next/Library/2026-10-22-00-29-55.gh-issue-160185.BPz40E.rst new file mode 100644 index 000000000000000..a9d7aef03214a34 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-22-00-29-55.gh-issue-160185.BPz40E.rst @@ -0,0 +1,5 @@ +Fix :meth:`codecs.StreamWriter.reset` of the stateful CJK codecs (``hz`` and +the ``iso2022_*`` family) not writing the sequence that returns the output to +ASCII. ``seek(0)`` of :class:`codecs.StreamWriter`, +:class:`codecs.StreamReaderWriter` and :class:`codecs.StreamRecoder` now +resets the writer before moving the stream position. diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index 4f0f201daf74916..1d709b841f29152 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -1866,13 +1866,17 @@ _multibytecodec_MultibyteStreamWriter_reset_impl(MultibyteStreamWriterObject *se PyTypeObject *cls) /*[clinic end generated code: output=32ef224c2a38aa3d input=28af6a9cd38d1979]*/ { - PyObject *pwrt; + PyObject *pwrt, *pending; - if (!self->pending) - Py_RETURN_NONE; + /* even without pending characters, a stateful codec may have to + * write a sequence that returns the output to its initial state */ + pending = self->pending; + if (pending == NULL) { + pending = Py_GetConstantBorrowed(Py_CONSTANT_EMPTY_STR); + } pwrt = multibytecodec_encode(self->codec, &self->state, - self->pending, NULL, self->errors, + pending, NULL, self->errors, MBENC_FLUSH | MBENC_RESET); /* some pending buffer can be truncated when UnicodeEncodeError is * raised on 'strict' mode. but, 'reset' method is designed to @@ -1895,6 +1899,7 @@ _multibytecodec_MultibyteStreamWriter_reset_impl(MultibyteStreamWriterObject *se Py_DECREF(pwrt); return NULL; } + Py_DECREF(wr); } Py_DECREF(pwrt);