Issue #5640: Fix _multibytecodec so that CJK codecs don't repeat

error replacement returned by codec error callbacks twice in
IncrementalEncoder and StreamWriter.

diff --git a/Lib/test/test_multibytecodec.py b/Lib/test/test_multibytecodec.py
index 2f648675c270902b07ce6e8f5bbd9f4d2dd78151..1d9d9e35ebf2e3c253444f7889e0f4e95e1a75d7 100644 (file)
--- a/Lib/test/test_multibytecodec.py
+++ b/Lib/test/test_multibytecodec.py
@@ -112,6 +112,10 @@ class Test_IncrementalEncoder(unittest.TestCase):
         self.assertRaises(UnicodeEncodeError, encoder.encode, '\u0123')
         self.assertEqual(encoder.encode('', True), b'\xa9\xdc')
 
+    def test_issue5640(self):
+        encoder = codecs.getincrementalencoder('shift-jis')('backslashreplace')
+        self.assertEqual(encoder.encode('\xff'), b'\\xff')
+        self.assertEqual(encoder.encode('\n'), b'\n')
 
 class Test_IncrementalDecoder(unittest.TestCase):
 

diff --git a/Misc/NEWS b/Misc/NEWS
index 8e9fbf455b7c47356a86dc3ca4a97a8a86ba2139..c7b203dd88ff68523cf29022cd432ed4784811f6 100644 (file)
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -58,6 +58,10 @@ Core and Builtins
 Library
 -------
 
+- Issue #5640: Fix _multibytecodec so that CJK codecs don't repeat
+  error substitutions from non-strict codec error callbacks in
+  incrementalencoder and StreamWriter.
+
 - Issue #5656: Fix the coverage reporting when running the test suite with
   the -T argument.
 

diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c
index 9846465edc69b3425fca8eccc6b0c57d4e334ab0..c6b3492e51ed3c51828df68e00bec6ce3d3788b0 100644 (file)
--- a/Modules/cjkcodecs/multibytecodec.c
+++ b/Modules/cjkcodecs/multibytecodec.c
@@ -506,7 +506,6 @@ multibytecodec_encode(MultibyteCodec *codec,
                outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf);
                r = codec->encode(state, codec->config, &buf.inbuf, inleft,
                                  &buf.outbuf, outleft, flags);
-               *data = buf.inbuf;
                if ((r == 0) || (r == MBERR_TOOFEW && !(flags & MBENC_FLUSH)))
                        break;
                else if (multibytecodec_encerror(codec, state, &buf, errors,r))
@@ -536,6 +535,7 @@ multibytecodec_encode(MultibyteCodec *codec,
                if (_PyBytes_Resize(&buf.outobj, finalsize) == -1)
                        goto errorexit;
 
+       *data = buf.inbuf;
        Py_XDECREF(buf.excobj);
        return buf.outobj;