[3.15] gh-159739: Preserve CJK codec state across chunk boundaries and callbacks (GH-160400) (#161069)

The HZ encoder can consume a tilde after writing only its first escape byte. A split ISO-2022 escape can exceed the incremental decoder’s pending buffer, and a reset can leave the previous character set selected. Reserve space for both HZ bytes before consuming the tilde, allow the full pending escape, and restore the initial ISO-2022 state on reset. Keep the native codec table and the active encoder error policy alive so replacing their Python owners cannot leave a codec using freed memory.
(cherry picked from commit a96fbdc)

Co-authored-by: Jelle Zijlstra <jelle@openai.com>

Author: gpshead

Lib/test/test_multibytecodec.py

@@ -39,6 +39,10 @@ def test_nullcoding(self):
             self.assertEqual(str(b'', enc), '')
             self.assertEqual(''.encode(enc), b'')
 
+    def test_hz_tilde_at_buffer_boundary(self):
+        text = 'a' + '中a' * 6 + '~'
+        self.assertEqual(text.encode('hz').decode('hz'), text)
+
     def test_str_decode(self):
         for enc in ALL_CJKENCODINGS:
             self.assertEqual('abcd'.encode(enc), b'abcd')
@@ -91,6 +95,12 @@ def test_decode_unicode(self):
 
 class Test_IncrementalEncoder(unittest.TestCase):
 
+    def test_iso2022_kr_reset(self):
+        encoder = codecs.getincrementalencoder('iso2022_kr')()
+        encoder.encode('가')
+        encoder.reset()
+        self.assertEqual(encoder.encode('가', final=True), '가'.encode('iso2022_kr'))
+
     def test_stateless(self):
         # cp949 encoder isn't stateful at all.
         encoder = codecs.getincrementalencoder('cp949')()
@@ -242,6 +252,19 @@ def test_subinterp(self):
 
 class Test_IncrementalDecoder(unittest.TestCase):
 
+    def test_iso2022_split_escape(self):
+        raw = b'\x1b$0000000B'
+        decoder = codecs.getincrementaldecoder('iso2022_jp')('replace')
+        decoded = decoder.decode(raw[:-1]) + decoder.decode(raw[-1:], final=True)
+        self.assertEqual(decoded, raw.decode('iso2022_jp', 'replace'))
+
+    def test_iso2022_reset(self):
+        decoder = codecs.getincrementaldecoder('iso2022_jp')()
+        decoder.decode(b'\x1b%')
+        decoder.reset()
+        with self.assertRaises(UnicodeDecodeError):
+            decoder.decode(b'\xff', final=True)
+
     def test_dbcs(self):
         # cp949 decoder is simple with only 1 or 2 bytes sequences.
         decoder = codecs.getincrementaldecoder('cp949')()
@@ -325,7 +348,7 @@ def test_setstate_validates_input(self):
         self.assertRaises(TypeError, decoder.setstate, 123)
         self.assertRaises(TypeError, decoder.setstate, ("invalid", 0))
         self.assertRaises(TypeError, decoder.setstate, (b"1234", "invalid"))
-        self.assertRaises(UnicodeDecodeError, decoder.setstate, (b"123456789", 0))
+        self.assertRaises(UnicodeDecodeError, decoder.setstate, (b"0" * 17, 0))
 
     def test_setstate_invalid_designation(self):
         # gh-153603: an unknown charset designation in the state must not crash

Misc/NEWS.d/next/Library/2026-10-22-16-23-01.gh-issue-159739.CjkR5a.rst

@@ -0,0 +1,4 @@
+Fix HZ encoding of a tilde at an output buffer boundary and ISO-2022 decoding
+of split escape sequences. Reset ISO-2022 incremental codecs to their initial
+state. Keep multibyte codec tables and custom encoder error policies alive
+when Python code replaces them during use.

Modules/cjkcodecs/_codecs_cn.c

@@ -361,11 +361,13 @@ ENCODER(hz)
                 NEXT_OUT(2);
                 state->c[CN_STATE_OFFSET] = 0;
             }
-            WRITEBYTE1((unsigned char)c);
-            NEXT(1, 1);
             if (c == '~') {
-                WRITEBYTE1('~');
-                NEXT_OUT(1);
+                WRITEBYTE2('~', '~');
+                NEXT(1, 2);
+            }
+            else {
+                WRITEBYTE1((unsigned char)c);
+                NEXT(1, 1);
             }
             continue;
         }

Modules/cjkcodecs/_codecs_iso2022.c

@@ -174,9 +174,8 @@ ENCODER_RESET(iso2022)
     if (STATE_G0 != CHARSET_ASCII) {
         WRITEBYTE3(ESC, '(', 'B');
         NEXT_OUT(3);
-        STATE_SETG0(CHARSET_ASCII);
     }
-    return 0;
+    return iso2022_encode_init(state, codec);
 }
 
 ENCODER(iso2022)
@@ -311,9 +310,7 @@ DECODER_INIT(iso2022)
 
 DECODER_RESET(iso2022)
 {
-    STATE_SETG0(CHARSET_ASCII);
-    STATE_CLEARFLAG(F_SHIFTED);
-    return 0;
+    return iso2022_decode_init(state, codec);
 }
 
 static Py_ssize_t

Modules/cjkcodecs/multibytecodec.c

@@ -513,6 +513,11 @@ multibytecodec_encode(const MultibyteCodec *codec,
     if (datalen == 0 && !(flags & MBENC_RESET))
         return PyBytes_FromStringAndSize(NULL, 0);
 
+    /* A callback may change a stateful encoder's errors attribute. */
+    if (ERROR_ISCUSTOM(errors)) {
+        Py_INCREF(errors);
+    }
+
     buf.excobj = NULL;
     buf.outobj = NULL;
     buf.inobj = text;   /* borrowed reference */
@@ -573,10 +578,12 @@ multibytecodec_encode(const MultibyteCodec *codec,
     if (inpos_t)
         *inpos_t = buf.inpos;
     Py_XDECREF(buf.excobj);
+    ERROR_DECREF(errors);
     return buf.outobj;
 
 errorexit:
     Py_XDECREF(buf.excobj);
+    ERROR_DECREF(errors);
     Py_XDECREF(buf.outobj);
     return NULL;
 }
@@ -1112,7 +1119,7 @@ mbiencoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
         self->codec->encinit(&self->state, self->codec) != 0)
         goto errorexit;
 
-    Py_DECREF(codec);
+    self->codec_owner = codec;
     return (PyObject *)self;
 
 errorexit:
@@ -1133,6 +1140,7 @@ mbiencoder_traverse(PyObject *op, visitproc visit, void *arg)
     MultibyteIncrementalEncoderObject *self = _MultibyteIncrementalEncoderObject_CAST(op);
     if (ERROR_ISCUSTOM(self->errors))
         Py_VISIT(self->errors);
+    Py_VISIT(self->codec_owner);
     return 0;
 }
 
@@ -1144,6 +1152,7 @@ mbiencoder_dealloc(PyObject *op)
     PyObject_GC_UnTrack(self);
     ERROR_DECREF(self->errors);
     Py_CLEAR(self->pending);
+    Py_XDECREF(self->codec_owner);
     tp->tp_free(self);
     Py_DECREF(tp);
 }
@@ -1395,7 +1404,7 @@ mbidecoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
         self->codec->decinit(&self->state, self->codec) != 0)
         goto errorexit;
 
-    Py_DECREF(codec);
+    self->codec_owner = codec;
     return (PyObject *)self;
 
 errorexit:
@@ -1416,6 +1425,7 @@ mbidecoder_traverse(PyObject *op, visitproc visit, void *arg)
     MultibyteIncrementalDecoderObject *self = _MultibyteIncrementalDecoderObject_CAST(op);
     if (ERROR_ISCUSTOM(self->errors))
         Py_VISIT(self->errors);
+    Py_VISIT(self->codec_owner);
     return 0;
 }
 
@@ -1426,6 +1436,7 @@ mbidecoder_dealloc(PyObject *op)
     PyTypeObject *tp = Py_TYPE(self);
     PyObject_GC_UnTrack(self);
     ERROR_DECREF(self->errors);
+    Py_XDECREF(self->codec_owner);
     tp->tp_free(self);
     Py_DECREF(tp);
 }
@@ -1712,7 +1723,7 @@ mbstreamreader_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
         self->codec->decinit(&self->state, self->codec) != 0)
         goto errorexit;
 
-    Py_DECREF(codec);
+    self->codec_owner = codec;
     return (PyObject *)self;
 
 errorexit:
@@ -1734,6 +1745,7 @@ mbstreamreader_traverse(PyObject *op, visitproc visit, void *arg)
     if (ERROR_ISCUSTOM(self->errors))
         Py_VISIT(self->errors);
     Py_VISIT(self->stream);
+    Py_VISIT(self->codec_owner);
     return 0;
 }
 
@@ -1745,6 +1757,7 @@ mbstreamreader_dealloc(PyObject *op)
     PyObject_GC_UnTrack(self);
     ERROR_DECREF(self->errors);
     Py_XDECREF(self->stream);
+    Py_XDECREF(self->codec_owner);
     tp->tp_free(self);
     Py_DECREF(tp);
 }
@@ -1936,7 +1949,7 @@ mbstreamwriter_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
         self->codec->encinit(&self->state, self->codec) != 0)
         goto errorexit;
 
-    Py_DECREF(codec);
+    self->codec_owner = codec;
     return (PyObject *)self;
 
 errorexit:
@@ -1958,6 +1971,7 @@ mbstreamwriter_traverse(PyObject *op, visitproc visit, void *arg)
     if (ERROR_ISCUSTOM(self->errors))
         Py_VISIT(self->errors);
     Py_VISIT(self->stream);
+    Py_VISIT(self->codec_owner);
     return 0;
 }
 
@@ -1969,6 +1983,7 @@ mbstreamwriter_dealloc(PyObject *op)
     PyObject_GC_UnTrack(self);
     ERROR_DECREF(self->errors);
     Py_XDECREF(self->stream);
+    Py_XDECREF(self->codec_owner);
     tp->tp_free(self);
     Py_DECREF(tp);
 }

Modules/cjkcodecs/multibytecodec.h

@@ -77,6 +77,7 @@ typedef struct {
 #define _MultibyteStatefulCodec_HEAD            \
     PyObject_HEAD                               \
     const MultibyteCodec *codec;                \
+    PyObject *codec_owner;                      \
     MultibyteCodec_State state;                 \
     PyObject *errors;
 typedef struct {
@@ -91,7 +92,8 @@ typedef struct {
     _MultibyteStatefulEncoder_HEAD
 } MultibyteStatefulEncoderContext;
 
-#define MAXDECPENDING   8
+/* ISO-2022 may retain an incomplete escape sequence up to 15 bytes. */
+#define MAXDECPENDING   16
 #define _MultibyteStatefulDecoder_HEAD          \
     _MultibyteStatefulCodec_HEAD                \
     unsigned char pending[MAXDECPENDING];       \