Skip to content

Commit 231b079

Browse files
gpsheadgpshead
authored
Merge branch '3.15' into backport-aa4f541-3.15
2 parents 935200b + 06367c6 commit 231b079

6 files changed

Lines changed: 58 additions & 15 deletions

File tree

‎Lib/test/test_multibytecodec.py‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,10 @@ def test_nullcoding(self):
3939
self.assertEqual(str(b'', enc), '')
4040
self.assertEqual(''.encode(enc), b'')
4141

42+
def test_hz_tilde_at_buffer_boundary(self):
43+
text = 'a' + '中a' * 6 + '~'
44+
self.assertEqual(text.encode('hz').decode('hz'), text)
45+
4246
def test_str_decode(self):
4347
for enc in ALL_CJKENCODINGS:
4448
self.assertEqual('abcd'.encode(enc), b'abcd')
@@ -91,6 +95,12 @@ def test_decode_unicode(self):
9195

9296
class Test_IncrementalEncoder(unittest.TestCase):
9397

98+
def test_iso2022_kr_reset(self):
99+
encoder = codecs.getincrementalencoder('iso2022_kr')()
100+
encoder.encode('가')
101+
encoder.reset()
102+
self.assertEqual(encoder.encode('가', final=True), '가'.encode('iso2022_kr'))
103+
94104
def test_stateless(self):
95105
# cp949 encoder isn't stateful at all.
96106
encoder = codecs.getincrementalencoder('cp949')()
@@ -242,6 +252,19 @@ def test_subinterp(self):
242252

243253
class Test_IncrementalDecoder(unittest.TestCase):
244254

255+
def test_iso2022_split_escape(self):
256+
raw = b'\x1b$0000000B'
257+
decoder = codecs.getincrementaldecoder('iso2022_jp')('replace')
258+
decoded = decoder.decode(raw[:-1]) + decoder.decode(raw[-1:], final=True)
259+
self.assertEqual(decoded, raw.decode('iso2022_jp', 'replace'))
260+
261+
def test_iso2022_reset(self):
262+
decoder = codecs.getincrementaldecoder('iso2022_jp')()
263+
decoder.decode(b'\x1b%')
264+
decoder.reset()
265+
with self.assertRaises(UnicodeDecodeError):
266+
decoder.decode(b'\xff', final=True)
267+
245268
def test_dbcs(self):
246269
# cp949 decoder is simple with only 1 or 2 bytes sequences.
247270
decoder = codecs.getincrementaldecoder('cp949')()
@@ -325,7 +348,7 @@ def test_setstate_validates_input(self):
325348
self.assertRaises(TypeError, decoder.setstate, 123)
326349
self.assertRaises(TypeError, decoder.setstate, ("invalid", 0))
327350
self.assertRaises(TypeError, decoder.setstate, (b"1234", "invalid"))
328-
self.assertRaises(UnicodeDecodeError, decoder.setstate, (b"123456789", 0))
351+
self.assertRaises(UnicodeDecodeError, decoder.setstate, (b"0" * 17, 0))
329352

330353
def test_setstate_invalid_designation(self):
331354
# gh-153603: an unknown charset designation in the state must not crash

‎Misc/NEWS.d/next/Library/2026-10-22-16-23-01.gh-issue-159739.CjkR5a.rst‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Fix HZ encoding of a tilde at an output buffer boundary and ISO-2022 decoding
2+
of split escape sequences. Reset ISO-2022 incremental codecs to their initial
3+
state. Keep multibyte codec tables and custom encoder error policies alive
4+
when Python code replaces them during use.

‎Modules/cjkcodecs/_codecs_cn.c‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -361,11 +361,13 @@ ENCODER(hz)
361361
NEXT_OUT(2);
362362
state->c[CN_STATE_OFFSET] = 0;
363363
}
364-
WRITEBYTE1((unsigned char)c);
365-
NEXT(1, 1);
366364
if (c == '~') {
367-
WRITEBYTE1('~');
368-
NEXT_OUT(1);
365+
WRITEBYTE2('~', '~');
366+
NEXT(1, 2);
367+
}
368+
else {
369+
WRITEBYTE1((unsigned char)c);
370+
NEXT(1, 1);
369371
}
370372
continue;
371373
}

‎Modules/cjkcodecs/_codecs_iso2022.c‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -174,9 +174,8 @@ ENCODER_RESET(iso2022)
174174
if (STATE_G0 != CHARSET_ASCII) {
175175
WRITEBYTE3(ESC, '(', 'B');
176176
NEXT_OUT(3);
177-
STATE_SETG0(CHARSET_ASCII);
178177
}
179-
return 0;
178+
return iso2022_encode_init(state, codec);
180179
}
181180

182181
ENCODER(iso2022)
@@ -311,9 +310,7 @@ DECODER_INIT(iso2022)
311310

312311
DECODER_RESET(iso2022)
313312
{
314-
STATE_SETG0(CHARSET_ASCII);
315-
STATE_CLEARFLAG(F_SHIFTED);
316-
return 0;
313+
return iso2022_decode_init(state, codec);
317314
}
318315

319316
static Py_ssize_t

‎Modules/cjkcodecs/multibytecodec.c‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -513,6 +513,11 @@ multibytecodec_encode(const MultibyteCodec *codec,
513513
if (datalen == 0 && !(flags & MBENC_RESET))
514514
return PyBytes_FromStringAndSize(NULL, 0);
515515

516+
/* A callback may change a stateful encoder's errors attribute. */
517+
if (ERROR_ISCUSTOM(errors)) {
518+
Py_INCREF(errors);
519+
}
520+
516521
buf.excobj = NULL;
517522
buf.outobj = NULL;
518523
buf.inobj = text; /* borrowed reference */
@@ -573,10 +578,12 @@ multibytecodec_encode(const MultibyteCodec *codec,
573578
if (inpos_t)
574579
*inpos_t = buf.inpos;
575580
Py_XDECREF(buf.excobj);
581+
ERROR_DECREF(errors);
576582
return buf.outobj;
577583

578584
errorexit:
579585
Py_XDECREF(buf.excobj);
586+
ERROR_DECREF(errors);
580587
Py_XDECREF(buf.outobj);
581588
return NULL;
582589
}
@@ -1112,7 +1119,7 @@ mbiencoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
11121119
self->codec->encinit(&self->state, self->codec) != 0)
11131120
goto errorexit;
11141121

1115-
Py_DECREF(codec);
1122+
self->codec_owner = codec;
11161123
return (PyObject *)self;
11171124

11181125
errorexit:
@@ -1133,6 +1140,7 @@ mbiencoder_traverse(PyObject *op, visitproc visit, void *arg)
11331140
MultibyteIncrementalEncoderObject *self = _MultibyteIncrementalEncoderObject_CAST(op);
11341141
if (ERROR_ISCUSTOM(self->errors))
11351142
Py_VISIT(self->errors);
1143+
Py_VISIT(self->codec_owner);
11361144
return 0;
11371145
}
11381146

@@ -1144,6 +1152,7 @@ mbiencoder_dealloc(PyObject *op)
11441152
PyObject_GC_UnTrack(self);
11451153
ERROR_DECREF(self->errors);
11461154
Py_CLEAR(self->pending);
1155+
Py_XDECREF(self->codec_owner);
11471156
tp->tp_free(self);
11481157
Py_DECREF(tp);
11491158
}
@@ -1395,7 +1404,7 @@ mbidecoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
13951404
self->codec->decinit(&self->state, self->codec) != 0)
13961405
goto errorexit;
13971406

1398-
Py_DECREF(codec);
1407+
self->codec_owner = codec;
13991408
return (PyObject *)self;
14001409

14011410
errorexit:
@@ -1416,6 +1425,7 @@ mbidecoder_traverse(PyObject *op, visitproc visit, void *arg)
14161425
MultibyteIncrementalDecoderObject *self = _MultibyteIncrementalDecoderObject_CAST(op);
14171426
if (ERROR_ISCUSTOM(self->errors))
14181427
Py_VISIT(self->errors);
1428+
Py_VISIT(self->codec_owner);
14191429
return 0;
14201430
}
14211431

@@ -1426,6 +1436,7 @@ mbidecoder_dealloc(PyObject *op)
14261436
PyTypeObject *tp = Py_TYPE(self);
14271437
PyObject_GC_UnTrack(self);
14281438
ERROR_DECREF(self->errors);
1439+
Py_XDECREF(self->codec_owner);
14291440
tp->tp_free(self);
14301441
Py_DECREF(tp);
14311442
}
@@ -1712,7 +1723,7 @@ mbstreamreader_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
17121723
self->codec->decinit(&self->state, self->codec) != 0)
17131724
goto errorexit;
17141725

1715-
Py_DECREF(codec);
1726+
self->codec_owner = codec;
17161727
return (PyObject *)self;
17171728

17181729
errorexit:
@@ -1734,6 +1745,7 @@ mbstreamreader_traverse(PyObject *op, visitproc visit, void *arg)
17341745
if (ERROR_ISCUSTOM(self->errors))
17351746
Py_VISIT(self->errors);
17361747
Py_VISIT(self->stream);
1748+
Py_VISIT(self->codec_owner);
17371749
return 0;
17381750
}
17391751

@@ -1745,6 +1757,7 @@ mbstreamreader_dealloc(PyObject *op)
17451757
PyObject_GC_UnTrack(self);
17461758
ERROR_DECREF(self->errors);
17471759
Py_XDECREF(self->stream);
1760+
Py_XDECREF(self->codec_owner);
17481761
tp->tp_free(self);
17491762
Py_DECREF(tp);
17501763
}
@@ -1941,7 +1954,7 @@ mbstreamwriter_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
19411954
self->codec->encinit(&self->state, self->codec) != 0)
19421955
goto errorexit;
19431956

1944-
Py_DECREF(codec);
1957+
self->codec_owner = codec;
19451958
return (PyObject *)self;
19461959

19471960
errorexit:
@@ -1963,6 +1976,7 @@ mbstreamwriter_traverse(PyObject *op, visitproc visit, void *arg)
19631976
if (ERROR_ISCUSTOM(self->errors))
19641977
Py_VISIT(self->errors);
19651978
Py_VISIT(self->stream);
1979+
Py_VISIT(self->codec_owner);
19661980
return 0;
19671981
}
19681982

@@ -1974,6 +1988,7 @@ mbstreamwriter_dealloc(PyObject *op)
19741988
PyObject_GC_UnTrack(self);
19751989
ERROR_DECREF(self->errors);
19761990
Py_XDECREF(self->stream);
1991+
Py_XDECREF(self->codec_owner);
19771992
tp->tp_free(self);
19781993
Py_DECREF(tp);
19791994
}

‎Modules/cjkcodecs/multibytecodec.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@ typedef struct {
7777
#define _MultibyteStatefulCodec_HEAD \
7878
PyObject_HEAD \
7979
const MultibyteCodec *codec; \
80+
PyObject *codec_owner; \
8081
MultibyteCodec_State state; \
8182
PyObject *errors;
8283
typedef struct {
@@ -91,7 +92,8 @@ typedef struct {
9192
_MultibyteStatefulEncoder_HEAD
9293
} MultibyteStatefulEncoderContext;
9394

94-
#define MAXDECPENDING 8
95+
/* ISO-2022 may retain an incomplete escape sequence up to 15 bytes. */
96+
#define MAXDECPENDING 16
9597
#define _MultibyteStatefulDecoder_HEAD \
9698
_MultibyteStatefulCodec_HEAD \
9799
unsigned char pending[MAXDECPENDING]; \

0 commit comments

Comments
 (0)