gh-160307: Complete final multibyte incremental decoding

Lib/test/test_multibytecodec.py

@@ -3,6 +3,7 @@
 #   Unit test for multibytecodec itself
 #
 
+import _codecs
 import _multibytecodec
 import codecs
 import io
@@ -252,6 +253,91 @@ def test_subinterp(self):
 
 class Test_IncrementalDecoder(unittest.TestCase):
 
+    def test_final_decode_incomplete_sequence(self):
+        cases = [
+            ('euc_jp', b'\x8f\x00'),
+            ('euc_jis_2004', b'\x8f\x00'),
+            ('euc_jisx0213', b'\x8f\x00'),
+            ('gb18030', b'\x806'),
+            ('gb18030', b'\x81\x30\x81'),
+        ]
+        for encoding, data in cases:
+            for errors in ('surrogateescape', 'replace', 'ignore',
+                           'backslashreplace'):
+                expected = data.decode(encoding, errors)
+                for split in range(len(data) + 1):
+                    with self.subTest(encoding=encoding, data=data,
+                                      errors=errors, split=split):
+                        decoder = codecs.getincrementaldecoder(encoding)(errors)
+                        text = decoder.decode(data[:split])
+                        text += decoder.decode(data[split:], final=True)
+                        self.assertEqual(text, expected)
+                        self.assertEqual(decoder.getstate()[0], b'')
+                        self.assertEqual(decoder.decode(b'', final=True), '')
+
+    def test_final_decode_incomplete_sequence_strict(self):
+        for encoding in ('euc_jp', 'euc_jis_2004', 'euc_jisx0213', 'gb18030'):
+            with self.subTest(encoding=encoding):
+                decoder = codecs.getincrementaldecoder(encoding)()
+                self.assertEqual(decoder.decode(b'\x81'), '')
+                state = decoder.getstate()
+                with self.assertRaises(UnicodeDecodeError):
+                    decoder.decode(b'', final=True)
+                self.assertEqual(decoder.getstate(), state)
+                decoder.errors = 'surrogateescape'
+                self.assertEqual(decoder.decode(b'', final=True), '\udc81')
+                self.assertEqual(decoder.getstate()[0], b'')
+
+    def test_final_decode_partial_error_handler(self):
+        def replace_one(exc):
+            return ('?', exc.start + 1)
+
+        errors = 'test.multibytecodec.replace_one'
+        codecs.register_error(errors, replace_one)
+        self.addCleanup(_codecs._unregister_error, errors)
+        for encoding, data, expected in (
+            ('euc_jp', b'\x8f\x81', '??'),
+            ('gb18030', b'\x81\x30\x81', '?0?'),
+        ):
+            with self.subTest(encoding=encoding):
+                decoder = codecs.getincrementaldecoder(encoding)(errors)
+                self.assertEqual(decoder.decode(data, final=True), expected)
+                self.assertEqual(decoder.getstate()[0], b'')
+
+    def test_final_decode_partial_error_handler_failure(self):
+        def replace_one_then_raise(exc):
+            if exc.start:
+                raise exc
+            return ('?', exc.start + 1)
+
+        errors = 'test.multibytecodec.replace_one_then_raise'
+        codecs.register_error(errors, replace_one_then_raise)
+        self.addCleanup(_codecs._unregister_error, errors)
+        decoder = codecs.getincrementaldecoder('euc_jp')(errors)
+        self.assertEqual(decoder.decode(b'\x8f\x81'), '')
+        state = decoder.getstate()
+        with self.assertRaises(UnicodeDecodeError):
+            decoder.decode(b'', final=True)
+        self.assertEqual(decoder.getstate(), state)
+        decoder.errors = 'surrogateescape'
+        self.assertEqual(decoder.decode(b'', final=True), '\udc8f\udc81')
+        self.assertEqual(decoder.getstate()[0], b'')
+
+    def test_textio_incomplete_sequence(self):
+        for encoding, data in (
+            ('euc_jp', b'\x8f\x00'),
+            ('euc_jis_2004', b'\x8f\x00'),
+            ('euc_jisx0213', b'\x8f\x00'),
+            ('gb18030', b'\x806'),
+            ('gb18030', b'\x81\x30\x81'),
+        ):
+            with self.subTest(encoding=encoding, data=data):
+                with io.TextIOWrapper(io.BytesIO(data), encoding=encoding,
+                                      errors='surrogateescape') as stream:
+                    self.assertEqual(stream.read(),
+                                     data.decode(encoding, 'surrogateescape'))
+                    self.assertEqual(stream.read(), '')
+
     def test_iso2022_split_escape(self):
         raw = b'\x1b$0000000B'
         decoder = codecs.getincrementaldecoder('iso2022_jp')('replace')

Misc/NEWS.d/next/Library/2026-10-25-11-00-00.gh-issue-160307.Qe7mJ2.rst

@@ -0,0 +1,4 @@
+Fix incomplete final decoding with the ``surrogateescape`` error handler in
+the ``euc_jp``, ``euc_jis_2004``, ``euc_jisx0213`` and ``gb18030`` codecs.
+Trailing input after an escaped byte is now decoded instead of remaining
+buffered, including when reading through :class:`io.TextIOWrapper`.

Modules/cjkcodecs/multibytecodec.c

@@ -1217,14 +1217,17 @@ _multibytecodec_MultibyteIncrementalDecoder_decode_impl(MultibyteIncrementalDeco
     if (decoder_feed_buffer(STATEFUL_DCTX(self), &buf))
         goto errorexit;
 
-    if (final && buf.inbuf < buf.inbuf_end) {
+    while (final && buf.inbuf < buf.inbuf_end) {
         if (multibytecodec_decerror(self->codec, &self->state,
                         &buf, self->errors, MBERR_TOOFEW)) {
             /* recover the original pending buffer */
             memcpy(self->pending, wdata, origpending);
             self->pendingsize = origpending;
             goto errorexit;
         }
+        /* The error handler may have consumed only part of the input. */
+        if (decoder_feed_buffer(STATEFUL_DCTX(self), &buf))
+            goto errorexit;
     }
 
     if (buf.inbuf < buf.inbuf_end) { /* pending sequence still exists */