diff --git a/Lib/test/test_multibytecodec.py b/Lib/test/test_multibytecodec.py index 0b34c6e69fab6b..d4923a31aef835 100644 --- a/Lib/test/test_multibytecodec.py +++ b/Lib/test/test_multibytecodec.py @@ -39,6 +39,10 @@ def test_nullcoding(self): self.assertEqual(str(b'', enc), '') self.assertEqual(''.encode(enc), b'') + def test_hz_tilde_at_buffer_boundary(self): + text = 'a' + '中a' * 6 + '~' + self.assertEqual(text.encode('hz').decode('hz'), text) + def test_str_decode(self): for enc in ALL_CJKENCODINGS: self.assertEqual('abcd'.encode(enc), b'abcd') @@ -91,6 +95,12 @@ def test_decode_unicode(self): class Test_IncrementalEncoder(unittest.TestCase): + def test_iso2022_kr_reset(self): + encoder = codecs.getincrementalencoder('iso2022_kr')() + encoder.encode('가') + encoder.reset() + self.assertEqual(encoder.encode('가', final=True), '가'.encode('iso2022_kr')) + def test_stateless(self): # cp949 encoder isn't stateful at all. encoder = codecs.getincrementalencoder('cp949')() @@ -242,6 +252,19 @@ def test_subinterp(self): class Test_IncrementalDecoder(unittest.TestCase): + def test_iso2022_split_escape(self): + raw = b'\x1b$0000000B' + decoder = codecs.getincrementaldecoder('iso2022_jp')('replace') + decoded = decoder.decode(raw[:-1]) + decoder.decode(raw[-1:], final=True) + self.assertEqual(decoded, raw.decode('iso2022_jp', 'replace')) + + def test_iso2022_reset(self): + decoder = codecs.getincrementaldecoder('iso2022_jp')() + decoder.decode(b'\x1b%') + decoder.reset() + with self.assertRaises(UnicodeDecodeError): + decoder.decode(b'\xff', final=True) + def test_dbcs(self): # cp949 decoder is simple with only 1 or 2 bytes sequences. decoder = codecs.getincrementaldecoder('cp949')() @@ -325,7 +348,7 @@ def test_setstate_validates_input(self): self.assertRaises(TypeError, decoder.setstate, 123) self.assertRaises(TypeError, decoder.setstate, ("invalid", 0)) self.assertRaises(TypeError, decoder.setstate, (b"1234", "invalid")) - self.assertRaises(UnicodeDecodeError, decoder.setstate, (b"123456789", 0)) + self.assertRaises(UnicodeDecodeError, decoder.setstate, (b"0" * 17, 0)) def test_setstate_invalid_designation(self): # gh-153603: an unknown charset designation in the state must not crash diff --git a/Misc/NEWS.d/next/Library/2026-10-22-16-23-01.gh-issue-159739.CjkR5a.rst b/Misc/NEWS.d/next/Library/2026-10-22-16-23-01.gh-issue-159739.CjkR5a.rst new file mode 100644 index 00000000000000..2860dc6897ecc0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-22-16-23-01.gh-issue-159739.CjkR5a.rst @@ -0,0 +1,4 @@ +Fix HZ encoding of a tilde at an output buffer boundary and ISO-2022 decoding +of split escape sequences. Reset ISO-2022 incremental codecs to their initial +state. Keep multibyte codec tables and custom encoder error policies alive +when Python code replaces them during use. diff --git a/Modules/cjkcodecs/_codecs_cn.c b/Modules/cjkcodecs/_codecs_cn.c index dc89f1d899e36b..ede54d09f2875e 100644 --- a/Modules/cjkcodecs/_codecs_cn.c +++ b/Modules/cjkcodecs/_codecs_cn.c @@ -361,11 +361,13 @@ ENCODER(hz) NEXT_OUT(2); state->c[CN_STATE_OFFSET] = 0; } - WRITEBYTE1((unsigned char)c); - NEXT(1, 1); if (c == '~') { - WRITEBYTE1('~'); - NEXT_OUT(1); + WRITEBYTE2('~', '~'); + NEXT(1, 2); + } + else { + WRITEBYTE1((unsigned char)c); + NEXT(1, 1); } continue; } diff --git a/Modules/cjkcodecs/_codecs_iso2022.c b/Modules/cjkcodecs/_codecs_iso2022.c index 662c2e1188e817..38a8a230c7ae45 100644 --- a/Modules/cjkcodecs/_codecs_iso2022.c +++ b/Modules/cjkcodecs/_codecs_iso2022.c @@ -174,9 +174,8 @@ ENCODER_RESET(iso2022) if (STATE_G0 != CHARSET_ASCII) { WRITEBYTE3(ESC, '(', 'B'); NEXT_OUT(3); - STATE_SETG0(CHARSET_ASCII); } - return 0; + return iso2022_encode_init(state, codec); } ENCODER(iso2022) @@ -311,9 +310,7 @@ DECODER_INIT(iso2022) DECODER_RESET(iso2022) { - STATE_SETG0(CHARSET_ASCII); - STATE_CLEARFLAG(F_SHIFTED); - return 0; + return iso2022_decode_init(state, codec); } static Py_ssize_t diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index 5fb3ec45a38875..f513182cb70260 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -514,6 +514,11 @@ multibytecodec_encode(const MultibyteCodec *codec, if (datalen == 0 && !(flags & MBENC_RESET)) return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); + /* A callback may change a stateful encoder's errors attribute. */ + if (ERROR_ISCUSTOM(errors)) { + Py_INCREF(errors); + } + buf.excobj = NULL; buf.writer = NULL; buf.inobj = text; /* borrowed reference */ @@ -567,10 +572,12 @@ multibytecodec_encode(const MultibyteCodec *codec, if (inpos_t) *inpos_t = buf.inpos; Py_XDECREF(buf.excobj); + ERROR_DECREF(errors); return PyBytesWriter_FinishWithPointer(buf.writer, buf.outbuf); errorexit: Py_XDECREF(buf.excobj); + ERROR_DECREF(errors); PyBytesWriter_Discard(buf.writer); return NULL; } @@ -1101,7 +1108,7 @@ mbiencoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) self->codec->encinit(&self->state, self->codec) != 0) goto errorexit; - Py_DECREF(codec); + self->codec_owner = codec; return (PyObject *)self; errorexit: @@ -1122,6 +1129,7 @@ mbiencoder_traverse(PyObject *op, visitproc visit, void *arg) MultibyteIncrementalEncoderObject *self = _MultibyteIncrementalEncoderObject_CAST(op); if (ERROR_ISCUSTOM(self->errors)) Py_VISIT(self->errors); + Py_VISIT(self->codec_owner); return 0; } @@ -1133,6 +1141,7 @@ mbiencoder_dealloc(PyObject *op) PyObject_GC_UnTrack(self); ERROR_DECREF(self->errors); Py_CLEAR(self->pending); + Py_XDECREF(self->codec_owner); tp->tp_free(self); Py_DECREF(tp); } @@ -1384,7 +1393,7 @@ mbidecoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) self->codec->decinit(&self->state, self->codec) != 0) goto errorexit; - Py_DECREF(codec); + self->codec_owner = codec; return (PyObject *)self; errorexit: @@ -1405,6 +1414,7 @@ mbidecoder_traverse(PyObject *op, visitproc visit, void *arg) MultibyteIncrementalDecoderObject *self = _MultibyteIncrementalDecoderObject_CAST(op); if (ERROR_ISCUSTOM(self->errors)) Py_VISIT(self->errors); + Py_VISIT(self->codec_owner); return 0; } @@ -1415,6 +1425,7 @@ mbidecoder_dealloc(PyObject *op) PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); ERROR_DECREF(self->errors); + Py_XDECREF(self->codec_owner); tp->tp_free(self); Py_DECREF(tp); } @@ -1703,7 +1714,7 @@ mbstreamreader_new(PyTypeObject *type, PyObject *args, PyObject *kwds) self->codec->decinit(&self->state, self->codec) != 0) goto errorexit; - Py_DECREF(codec); + self->codec_owner = codec; return (PyObject *)self; errorexit: @@ -1725,6 +1736,7 @@ mbstreamreader_traverse(PyObject *op, visitproc visit, void *arg) if (ERROR_ISCUSTOM(self->errors)) Py_VISIT(self->errors); Py_VISIT(self->stream); + Py_VISIT(self->codec_owner); return 0; } @@ -1736,6 +1748,7 @@ mbstreamreader_dealloc(PyObject *op) PyObject_GC_UnTrack(self); ERROR_DECREF(self->errors); Py_XDECREF(self->stream); + Py_XDECREF(self->codec_owner); tp->tp_free(self); Py_DECREF(tp); } @@ -1927,7 +1940,7 @@ mbstreamwriter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) self->codec->encinit(&self->state, self->codec) != 0) goto errorexit; - Py_DECREF(codec); + self->codec_owner = codec; return (PyObject *)self; errorexit: @@ -1949,6 +1962,7 @@ mbstreamwriter_traverse(PyObject *op, visitproc visit, void *arg) if (ERROR_ISCUSTOM(self->errors)) Py_VISIT(self->errors); Py_VISIT(self->stream); + Py_VISIT(self->codec_owner); return 0; } @@ -1960,6 +1974,7 @@ mbstreamwriter_dealloc(PyObject *op) PyObject_GC_UnTrack(self); ERROR_DECREF(self->errors); Py_XDECREF(self->stream); + Py_XDECREF(self->codec_owner); tp->tp_free(self); Py_DECREF(tp); } diff --git a/Modules/cjkcodecs/multibytecodec.h b/Modules/cjkcodecs/multibytecodec.h index 5b85e2e3de316d..bacd6e1078b685 100644 --- a/Modules/cjkcodecs/multibytecodec.h +++ b/Modules/cjkcodecs/multibytecodec.h @@ -77,6 +77,7 @@ typedef struct { #define _MultibyteStatefulCodec_HEAD \ PyObject_HEAD \ const MultibyteCodec *codec; \ + PyObject *codec_owner; \ MultibyteCodec_State state; \ PyObject *errors; typedef struct { @@ -91,7 +92,8 @@ typedef struct { _MultibyteStatefulEncoder_HEAD } MultibyteStatefulEncoderContext; -#define MAXDECPENDING 8 +/* ISO-2022 may retain an incomplete escape sequence up to 15 bytes. */ +#define MAXDECPENDING 16 #define _MultibyteStatefulDecoder_HEAD \ _MultibyteStatefulCodec_HEAD \ unsigned char pending[MAXDECPENDING]; \