From 021d55ff745268299f8c3d487aac7f12a01ec294 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 21 Feb 2015 01:19:17 +0200 Subject: [PATCH] Issue #23215: Multibyte codecs with custom error handlers that ignores errors consumed too much memory and raised SystemError or MemoryError. Original patch by Aleksi Torhamo. --- Lib/test/test_multibytecodec.py | 7 +++++++ Misc/NEWS | 4 ++++ Modules/cjkcodecs/multibytecodec.c | 25 ++++++++++++++----------- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_multibytecodec.py b/Lib/test/test_multibytecodec.py index c38df8d51d..5b61e7e857 100644 --- a/Lib/test/test_multibytecodec.py +++ b/Lib/test/test_multibytecodec.py @@ -43,6 +43,13 @@ class Test_MultibyteCodec(unittest.TestCase): self.assertRaises(IndexError, dec, 'apple\x92ham\x93spam', 'test.cjktest') + def test_errorcallback_custom_ignore(self): + # Issue #23215: MemoryError with custom error handlers and multibyte codecs + data = 100 * unichr(0xdc00) + codecs.register_error("test.ignore", codecs.ignore_errors) + for enc in ALL_CJKENCODINGS: + self.assertEqual(data.encode(enc, "test.ignore"), b'') + def test_codingspec(self): for enc in ALL_CJKENCODINGS: code = '# coding: {}\n'.format(enc) diff --git a/Misc/NEWS b/Misc/NEWS index 8766956297..51aa4e9667 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -18,6 +18,10 @@ Core and Builtins Library ------- +- Issue #23215: Multibyte codecs with custom error handlers that ignores errors + consumed too much memory and raised SystemError or MemoryError. + Original patch by Aleksi Torhamo. + - Issue #5700: io.FileIO() called flush() after closing the file. flush() was not called in close() if closefd=False. diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index 14fed3ef38..3f75fe74ca 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -170,8 +170,10 @@ expand_encodebuffer(MultibyteEncodeBuffer *buf, Py_ssize_t esize) orgsize = PyString_GET_SIZE(buf->outobj); incsize = (esize < (orgsize >> 1) ? (orgsize >> 1) | 1 : esize); - if (orgsize > PY_SSIZE_T_MAX - incsize) + if (orgsize > PY_SSIZE_T_MAX - incsize) { + PyErr_NoMemory(); return -1; + } if (_PyString_Resize(&buf->outobj, orgsize + incsize) == -1) return -1; @@ -182,11 +184,11 @@ expand_encodebuffer(MultibyteEncodeBuffer *buf, Py_ssize_t esize) return 0; } -#define REQUIRE_ENCODEBUFFER(buf, s) { \ - if ((s) < 1 || (buf)->outbuf + (s) > (buf)->outbuf_end) \ +#define REQUIRE_ENCODEBUFFER(buf, s) do { \ + if ((s) < 0 || (s) > (buf)->outbuf_end - (buf)->outbuf) \ if (expand_encodebuffer(buf, s) == -1) \ goto errorexit; \ -} +} while(0) static int expand_decodebuffer(MultibyteDecodeBuffer *buf, Py_ssize_t esize) @@ -205,11 +207,11 @@ expand_decodebuffer(MultibyteDecodeBuffer *buf, Py_ssize_t esize) return 0; } -#define REQUIRE_DECODEBUFFER(buf, s) { \ - if ((s) < 1 || (buf)->outbuf + (s) > (buf)->outbuf_end) \ +#define REQUIRE_DECODEBUFFER(buf, s) do { \ + if ((s) < 0 || (s) > (buf)->outbuf_end - (buf)->outbuf) \ if (expand_decodebuffer(buf, s) == -1) \ goto errorexit; \ -} +} while(0) /** @@ -327,10 +329,11 @@ multibytecodec_encerror(MultibyteCodec *codec, } retstrsize = PyString_GET_SIZE(retstr); - REQUIRE_ENCODEBUFFER(buf, retstrsize); - - memcpy(buf->outbuf, PyString_AS_STRING(retstr), retstrsize); - buf->outbuf += retstrsize; + if (retstrsize > 0) { + REQUIRE_ENCODEBUFFER(buf, retstrsize); + memcpy(buf->outbuf, PyString_AS_STRING(retstr), retstrsize); + buf->outbuf += retstrsize; + } newpos = PyInt_AsSsize_t(PyTuple_GET_ITEM(retobj, 1)); if (newpos < 0 && !PyErr_Occurred()) -- 2.50.1