Skip to content

Commit a96fbdc

Browse files
jelle-openaijelle-openai
authored
gh-159739: Preserve CJK codec state across chunk boundaries and callbacks (#160400)
The HZ encoder can consume a tilde after writing only its first escape byte. A split ISO-2022 escape can exceed the incremental decoder’s pending buffer, and a reset can leave the previous character set selected. Reserve space for both HZ bytes before consuming the tilde, allow the full pending escape, and restore the initial ISO-2022 state on reset. Keep the native codec table and the active encoder error policy alive so replacing their Python owners cannot leave a codec using freed memory.
1 parent 612ce88 commit a96fbdc

6 files changed

Lines changed: 58 additions & 15 deletions

File tree

‎Lib/test/test_multibytecodec.py‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,10 @@ def test_nullcoding(self):
3939
self.assertEqual(str(b'', enc), '')
4040
self.assertEqual(''.encode(enc), b'')
4141

42+
def test_hz_tilde_at_buffer_boundary(self):
43+
text = 'a' + '中a' * 6 + '~'
44+
self.assertEqual(text.encode('hz').decode('hz'), text)
45+
4246
def test_str_decode(self):
4347
for enc in ALL_CJKENCODINGS:
4448
self.assertEqual('abcd'.encode(enc), b'abcd')
@@ -91,6 +95,12 @@ def test_decode_unicode(self):
9195

9296
class Test_IncrementalEncoder(unittest.TestCase):
9397

98+
def test_iso2022_kr_reset(self):
99+
encoder = codecs.getincrementalencoder('iso2022_kr')()
100+
encoder.encode('가')
101+
encoder.reset()
102+
self.assertEqual(encoder.encode('가', final=True), '가'.encode('iso2022_kr'))
103+
94104
def test_stateless(self):
95105
# cp949 encoder isn't stateful at all.
96106
encoder = codecs.getincrementalencoder('cp949')()
@@ -242,6 +252,19 @@ def test_subinterp(self):
242252

243253
class Test_IncrementalDecoder(unittest.TestCase):
244254

255+
def test_iso2022_split_escape(self):
256+
raw = b'\x1b$0000000B'
257+
decoder = codecs.getincrementaldecoder('iso2022_jp')('replace')
258+
decoded = decoder.decode(raw[:-1]) + decoder.decode(raw[-1:], final=True)
259+
self.assertEqual(decoded, raw.decode('iso2022_jp', 'replace'))
260+
261+
def test_iso2022_reset(self):
262+
decoder = codecs.getincrementaldecoder('iso2022_jp')()
263+
decoder.decode(b'\x1b%')
264+
decoder.reset()
265+
with self.assertRaises(UnicodeDecodeError):
266+
decoder.decode(b'\xff', final=True)
267+
245268
def test_dbcs(self):
246269
# cp949 decoder is simple with only 1 or 2 bytes sequences.
247270
decoder = codecs.getincrementaldecoder('cp949')()
@@ -325,7 +348,7 @@ def test_setstate_validates_input(self):
325348
self.assertRaises(TypeError, decoder.setstate, 123)
326349
self.assertRaises(TypeError, decoder.setstate, ("invalid", 0))
327350
self.assertRaises(TypeError, decoder.setstate, (b"1234", "invalid"))
328-
self.assertRaises(UnicodeDecodeError, decoder.setstate, (b"123456789", 0))
351+
self.assertRaises(UnicodeDecodeError, decoder.setstate, (b"0" * 17, 0))
329352

330353
def test_setstate_invalid_designation(self):
331354
# gh-153603: an unknown charset designation in the state must not crash

‎Misc/NEWS.d/next/Library/2026-10-22-16-23-01.gh-issue-159739.CjkR5a.rst‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Fix HZ encoding of a tilde at an output buffer boundary and ISO-2022 decoding
2+
of split escape sequences. Reset ISO-2022 incremental codecs to their initial
3+
state. Keep multibyte codec tables and custom encoder error policies alive
4+
when Python code replaces them during use.

‎Modules/cjkcodecs/_codecs_cn.c‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -361,11 +361,13 @@ ENCODER(hz)
361361
NEXT_OUT(2);
362362
state->c[CN_STATE_OFFSET] = 0;
363363
}
364-
WRITEBYTE1((unsigned char)c);
365-
NEXT(1, 1);
366364
if (c == '~') {
367-
WRITEBYTE1('~');
368-
NEXT_OUT(1);
365+
WRITEBYTE2('~', '~');
366+
NEXT(1, 2);
367+
}
368+
else {
369+
WRITEBYTE1((unsigned char)c);
370+
NEXT(1, 1);
369371
}
370372
continue;
371373
}

‎Modules/cjkcodecs/_codecs_iso2022.c‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -174,9 +174,8 @@ ENCODER_RESET(iso2022)
174174
if (STATE_G0 != CHARSET_ASCII) {
175175
WRITEBYTE3(ESC, '(', 'B');
176176
NEXT_OUT(3);
177-
STATE_SETG0(CHARSET_ASCII);
178177
}
179-
return 0;
178+
return iso2022_encode_init(state, codec);
180179
}
181180

182181
ENCODER(iso2022)
@@ -311,9 +310,7 @@ DECODER_INIT(iso2022)
311310

312311
DECODER_RESET(iso2022)
313312
{
314-
STATE_SETG0(CHARSET_ASCII);
315-
STATE_CLEARFLAG(F_SHIFTED);
316-
return 0;
313+
return iso2022_decode_init(state, codec);
317314
}
318315

319316
static Py_ssize_t

‎Modules/cjkcodecs/multibytecodec.c‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -514,6 +514,11 @@ multibytecodec_encode(const MultibyteCodec *codec,
514514
if (datalen == 0 && !(flags & MBENC_RESET))
515515
return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES);
516516

517+
/* A callback may change a stateful encoder's errors attribute. */
518+
if (ERROR_ISCUSTOM(errors)) {
519+
Py_INCREF(errors);
520+
}
521+
517522
buf.excobj = NULL;
518523
buf.writer = NULL;
519524
buf.inobj = text; /* borrowed reference */
@@ -567,10 +572,12 @@ multibytecodec_encode(const MultibyteCodec *codec,
567572
if (inpos_t)
568573
*inpos_t = buf.inpos;
569574
Py_XDECREF(buf.excobj);
575+
ERROR_DECREF(errors);
570576
return PyBytesWriter_FinishWithPointer(buf.writer, buf.outbuf);
571577

572578
errorexit:
573579
Py_XDECREF(buf.excobj);
580+
ERROR_DECREF(errors);
574581
PyBytesWriter_Discard(buf.writer);
575582
return NULL;
576583
}
@@ -1101,7 +1108,7 @@ mbiencoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
11011108
self->codec->encinit(&self->state, self->codec) != 0)
11021109
goto errorexit;
11031110

1104-
Py_DECREF(codec);
1111+
self->codec_owner = codec;
11051112
return (PyObject *)self;
11061113

11071114
errorexit:
@@ -1122,6 +1129,7 @@ mbiencoder_traverse(PyObject *op, visitproc visit, void *arg)
11221129
MultibyteIncrementalEncoderObject *self = _MultibyteIncrementalEncoderObject_CAST(op);
11231130
if (ERROR_ISCUSTOM(self->errors))
11241131
Py_VISIT(self->errors);
1132+
Py_VISIT(self->codec_owner);
11251133
return 0;
11261134
}
11271135

@@ -1133,6 +1141,7 @@ mbiencoder_dealloc(PyObject *op)
11331141
PyObject_GC_UnTrack(self);
11341142
ERROR_DECREF(self->errors);
11351143
Py_CLEAR(self->pending);
1144+
Py_XDECREF(self->codec_owner);
11361145
tp->tp_free(self);
11371146
Py_DECREF(tp);
11381147
}
@@ -1384,7 +1393,7 @@ mbidecoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
13841393
self->codec->decinit(&self->state, self->codec) != 0)
13851394
goto errorexit;
13861395

1387-
Py_DECREF(codec);
1396+
self->codec_owner = codec;
13881397
return (PyObject *)self;
13891398

13901399
errorexit:
@@ -1405,6 +1414,7 @@ mbidecoder_traverse(PyObject *op, visitproc visit, void *arg)
14051414
MultibyteIncrementalDecoderObject *self = _MultibyteIncrementalDecoderObject_CAST(op);
14061415
if (ERROR_ISCUSTOM(self->errors))
14071416
Py_VISIT(self->errors);
1417+
Py_VISIT(self->codec_owner);
14081418
return 0;
14091419
}
14101420

@@ -1415,6 +1425,7 @@ mbidecoder_dealloc(PyObject *op)
14151425
PyTypeObject *tp = Py_TYPE(self);
14161426
PyObject_GC_UnTrack(self);
14171427
ERROR_DECREF(self->errors);
1428+
Py_XDECREF(self->codec_owner);
14181429
tp->tp_free(self);
14191430
Py_DECREF(tp);
14201431
}
@@ -1703,7 +1714,7 @@ mbstreamreader_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
17031714
self->codec->decinit(&self->state, self->codec) != 0)
17041715
goto errorexit;
17051716

1706-
Py_DECREF(codec);
1717+
self->codec_owner = codec;
17071718
return (PyObject *)self;
17081719

17091720
errorexit:
@@ -1725,6 +1736,7 @@ mbstreamreader_traverse(PyObject *op, visitproc visit, void *arg)
17251736
if (ERROR_ISCUSTOM(self->errors))
17261737
Py_VISIT(self->errors);
17271738
Py_VISIT(self->stream);
1739+
Py_VISIT(self->codec_owner);
17281740
return 0;
17291741
}
17301742

@@ -1736,6 +1748,7 @@ mbstreamreader_dealloc(PyObject *op)
17361748
PyObject_GC_UnTrack(self);
17371749
ERROR_DECREF(self->errors);
17381750
Py_XDECREF(self->stream);
1751+
Py_XDECREF(self->codec_owner);
17391752
tp->tp_free(self);
17401753
Py_DECREF(tp);
17411754
}
@@ -1927,7 +1940,7 @@ mbstreamwriter_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
19271940
self->codec->encinit(&self->state, self->codec) != 0)
19281941
goto errorexit;
19291942

1930-
Py_DECREF(codec);
1943+
self->codec_owner = codec;
19311944
return (PyObject *)self;
19321945

19331946
errorexit:
@@ -1949,6 +1962,7 @@ mbstreamwriter_traverse(PyObject *op, visitproc visit, void *arg)
19491962
if (ERROR_ISCUSTOM(self->errors))
19501963
Py_VISIT(self->errors);
19511964
Py_VISIT(self->stream);
1965+
Py_VISIT(self->codec_owner);
19521966
return 0;
19531967
}
19541968

@@ -1960,6 +1974,7 @@ mbstreamwriter_dealloc(PyObject *op)
19601974
PyObject_GC_UnTrack(self);
19611975
ERROR_DECREF(self->errors);
19621976
Py_XDECREF(self->stream);
1977+
Py_XDECREF(self->codec_owner);
19631978
tp->tp_free(self);
19641979
Py_DECREF(tp);
19651980
}

‎Modules/cjkcodecs/multibytecodec.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@ typedef struct {
7777
#define _MultibyteStatefulCodec_HEAD \
7878
PyObject_HEAD \
7979
const MultibyteCodec *codec; \
80+
PyObject *codec_owner; \
8081
MultibyteCodec_State state; \
8182
PyObject *errors;
8283
typedef struct {
@@ -91,7 +92,8 @@ typedef struct {
9192
_MultibyteStatefulEncoder_HEAD
9293
} MultibyteStatefulEncoderContext;
9394

94-
#define MAXDECPENDING 8
95+
/* ISO-2022 may retain an incomplete escape sequence up to 15 bytes. */
96+
#define MAXDECPENDING 16
9597
#define _MultibyteStatefulDecoder_HEAD \
9698
_MultibyteStatefulCodec_HEAD \
9799
unsigned char pending[MAXDECPENDING]; \

0 commit comments

Comments
 (0)