diff --git a/Lib/test/test_codecencodings_cn.py b/Lib/test/test_codecencodings_cn.py index 2a450718d5aae5..35a2d7fb1ebf29 100644 --- a/Lib/test/test_codecencodings_cn.py +++ b/Lib/test/test_codecencodings_cn.py @@ -92,5 +92,15 @@ class Test_HZ(multibytecodec_support.TestBase, unittest.TestCase): (b'~{Dc~\n:C~}', 'strict', None), # ~\n only in ASCII mode ) + def test_escape_at_buffer_boundary(self): + # '~~' must not be cut in two when the output buffer is enlarged. + for pairs in range(40): + for letters in range(3): + text = 'a\u4e2d' * pairs + 'x' * letters + '~{' + with self.subTest(pairs=pairs, letters=letters): + encoded = text.encode('hz') + self.assertEndsWith(encoded, b'~~{') + self.assertEqual(encoded.decode('hz'), text) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2026-10-22-08-15-57.gh-issue-160518.hZt1Ld.rst b/Misc/NEWS.d/next/Library/2026-10-22-08-15-57.gh-issue-160518.hZt1Ld.rst new file mode 100644 index 00000000000000..3a3c6805ec82ae --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-22-08-15-57.gh-issue-160518.hZt1Ld.rst @@ -0,0 +1,3 @@ +Fix the ``hz`` codec sometimes encoding ``~`` as a single ``~`` instead of +``~~`` in text that mixes ASCII and Chinese characters, which produced data +that did not decode back to the original text. diff --git a/Modules/cjkcodecs/_codecs_cn.c b/Modules/cjkcodecs/_codecs_cn.c index dc89f1d899e36b..ede54d09f2875e 100644 --- a/Modules/cjkcodecs/_codecs_cn.c +++ b/Modules/cjkcodecs/_codecs_cn.c @@ -361,11 +361,13 @@ ENCODER(hz) NEXT_OUT(2); state->c[CN_STATE_OFFSET] = 0; } - WRITEBYTE1((unsigned char)c); - NEXT(1, 1); if (c == '~') { - WRITEBYTE1('~'); - NEXT_OUT(1); + WRITEBYTE2('~', '~'); + NEXT(1, 2); + } + else { + WRITEBYTE1((unsigned char)c); + NEXT(1, 1); } continue; }