From 8e7de7b9988e7e2b20c80673ce96ccc3ab3cf2ce Mon Sep 17 00:00:00 2001 From: Vyron Vasileiadis Date: Wed, 29 Jul 2026 23:14:54 +0300 Subject: [PATCH 1/2] gh-154863: Fix iconv encoding of ISO-2022-CN-EXT returning empty bytes The flush that emits the pending shift sequence can report a nonreversible conversion, which the loop mistook for a substituted character. It then retried while still flushing, converted nothing and returned the empty output buffer. --- Lib/test/test_codecs.py | 23 +++++++++++++++++++++++ Objects/unicodeobject.c | 6 +++--- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 31704955df3e14..4a9240f2c3c153 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -3679,6 +3679,10 @@ def iconv_encoding_available(name): # Encodings iconv may provide but for which CPython has no built-in codec # (cp1047 is EBCDIC, i.e. not ASCII-compatible). _ICONV_ONLY = ['cp1047', 'cp1133', 'GEORGIAN-PS', 'ARMSCII-8'] +# Encodings that leave a shift state pending, so encoding ends with a flush. +_ICONV_SHIFT_STATE = [('ISO-2022-CN-EXT', 'ABC\u4e2dDEF'), + ('ISO-2022-CN', 'ABC\u4e2dDEF'), + ('ISO-2022-JP', 'ABC\u65e5DEF')] @unittest.skipUnless(hasattr(codecs, 'iconv_encode'), @@ -3812,6 +3816,25 @@ def test_encode_kinds(self): with self.subTest(text=text): self.assertEqual(text.encode('iconv:' + enc), text.encode(enc)) + def test_encode_shift_state_flush(self): + # Encoding ends with a flush that emits the pending shift sequence. Its + # return value counts nonreversible conversions, which some iconv + # implementations make positive for the flush itself (glibc does for + # ISO-2022-CN-EXT); that must not be read as a substituted character, + # which used to discard the whole output. + tested = False + for enc, text in _ICONV_SHIFT_STATE: + if not iconv_encoding_available(enc): + continue + tested = True + with self.subTest(encoding=enc): + data = codecs.iconv_encode(enc, text)[0] + self.assertNotEqual(data, b'') + self.assertEqual( + codecs.iconv_decode(enc, data, 'strict', True)[0], text) + if not tested: + self.skipTest('no shift-state iconv encoding is available') + def test_encode_surrogateescape(self): # A lone surrogate lives in the 2-byte kind and round-trips. enc = self.require('ASCII') diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index a4550c0f5f3363..eec02f662e7905 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -8462,6 +8462,9 @@ _PyUnicode_EncodeIconv(const char *encoding, PyObject *unicode, } if (ret != (size_t)-1) { + if (flushing) { + break; + } /* A positive result counts nonreversible conversions: iconv() substituted an unencodable character instead of failing with EILSEQ (musl and *BSD citrus do this). Treat it as unencodable @@ -8479,9 +8482,6 @@ _PyUnicode_EncodeIconv(const char *encoding, PyObject *unicode, out = out_before; up -= unit; } - else if (flushing) { - break; - } else if (careful && up < uend) { continue; } From b16c09f0d4003be217546f7e4a0d1557c14e880a Mon Sep 17 00:00:00 2001 From: Vyron Vasileiadis Date: Wed, 29 Jul 2026 23:33:23 +0300 Subject: [PATCH 2/2] Do not assume the platform iconv can represent the test character macOS substitutes '?' for the Chinese character in ISO-2022-CN instead of encoding it, so check only that the surrounding ASCII survives, which is what the fix is about. --- Lib/test/test_codecs.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 4a9240f2c3c153..f6f8089a065059 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -3818,20 +3818,28 @@ def test_encode_kinds(self): def test_encode_shift_state_flush(self): # Encoding ends with a flush that emits the pending shift sequence. Its - # return value counts nonreversible conversions, which some iconv - # implementations make positive for the flush itself (glibc does for - # ISO-2022-CN-EXT); that must not be read as a substituted character, - # which used to discard the whole output. + # return value counts nonreversible conversions, and some iconv + # implementations make it positive for the flush itself (glibc does for + # ISO-2022-CN-EXT). That must not be read as a substituted character: + # doing so discarded the whole output, ASCII included. + # + # Only the ASCII around the character is checked, not a full round-trip. + # An iconv that cannot represent the character either rejects it or + # substitutes for it silently, as macOS does for ISO-2022-CN. tested = False for enc, text in _ICONV_SHIFT_STATE: if not iconv_encoding_available(enc): continue - tested = True with self.subTest(encoding=enc): - data = codecs.iconv_encode(enc, text)[0] + try: + data = codecs.iconv_encode(enc, text)[0] + except UnicodeEncodeError: + continue + tested = True self.assertNotEqual(data, b'') - self.assertEqual( - codecs.iconv_decode(enc, data, 'strict', True)[0], text) + decoded = codecs.iconv_decode(enc, data, 'strict', True)[0] + self.assertStartsWith(decoded, 'ABC') + self.assertEndsWith(decoded, 'DEF') if not tested: self.skipTest('no shift-state iconv encoding is available')