bpo-34087: Fix buffer overflow in int(s) and similar functions (GH-8274)

`_PyUnicode_TransformDecimalAndSpaceToASCII()` missed trailing NUL char.
It caused buffer overflow in `_Py_string_to_number_with_underscores()`.

This bug is introduced in 9b6c60cb.

diff --git a/Lib/test/test_complex.py b/Lib/test/test_complex.py
index 2d883c5348ea6fa8d4ef50391b4cf2779dad7d90..21c6eaed60540c2b613b1b657d69468e7f786950 100644 (file)
--- a/Lib/test/test_complex.py
+++ b/Lib/test/test_complex.py
@@ -345,6 +345,9 @@ class ComplexTest(unittest.TestCase):
         self.assertEqual(type(complex("1"*500)), complex)
         # check whitespace processing
         self.assertEqual(complex('\N{EM SPACE}(\N{EN SPACE}1+1j ) '), 1+1j)
+        # Invalid unicode string
+        # See bpo-34087
+        self.assertRaises(ValueError, complex, '\u3053\u3093\u306b\u3061\u306f')
 
         class EvilExc(Exception):
             pass

diff --git a/Lib/test/test_float.py b/Lib/test/test_float.py
index 17174dd295dfcc8b2c015dc5aaac45187a64d936..06ea90c207f56c98c0b0bd6c5389b0ca5f25c72f 100644 (file)
--- a/Lib/test/test_float.py
+++ b/Lib/test/test_float.py
@@ -60,6 +60,9 @@ class GeneralFloatCases(unittest.TestCase):
         # extra long strings should not be a problem
         float(b'.' + b'1'*1000)
         float('.' + '1'*1000)
+        # Invalid unicode string
+        # See bpo-34087
+        self.assertRaises(ValueError, float, '\u3053\u3093\u306b\u3061\u306f')
 
     def test_underscores(self):
         for lit in VALID_UNDERSCORE_LITERALS:

diff --git a/Lib/test/test_long.py b/Lib/test/test_long.py
index cc48259e35fb2c810709242a1431312401ead86b..8472889d48badee25062c2e64359d967c1910f83 100644 (file)
--- a/Lib/test/test_long.py
+++ b/Lib/test/test_long.py
@@ -373,6 +373,10 @@ class LongTest(unittest.TestCase):
         for base in invalid_bases:
             self.assertRaises(ValueError, int, '42', base)
 
+        # Invalid unicode string
+        # See bpo-34087
+        self.assertRaises(ValueError, int, '\u3053\u3093\u306b\u3061\u306f')
+
 
     def test_conversion(self):
 

diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-07-13-22-09-55.bpo-34087.I1Bxfc.rst b/Misc/NEWS.d/next/Core and Builtins/2018-07-13-22-09-55.bpo-34087.I1Bxfc.rst
new file mode 100644 (file)
index 0000000..5147395
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2018-07-13-22-09-55.bpo-34087.I1Bxfc.rst
@@ -0,0 +1 @@
+Fix buffer overflow while converting unicode to numeric values.

diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index 80d1bba1e9b407a876b1f37cdabc037f6d73a7e0..2b06f15f6c65e90778ef1a4d9e532c0688c814e8 100644 (file)
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -9072,6 +9072,7 @@ _PyUnicode_TransformDecimalAndSpaceToASCII(PyObject *unicode)
             int decimal = Py_UNICODE_TODECIMAL(ch);
             if (decimal < 0) {
                 out[i] = '?';
+                out[i+1] = '\0';
                 _PyUnicode_LENGTH(result) = i + 1;
                 break;
             }
@@ -9079,6 +9080,7 @@ _PyUnicode_TransformDecimalAndSpaceToASCII(PyObject *unicode)
         }
     }
 
+    assert(_PyUnicode_CheckConsistency(result, 1));
     return result;
 }
 

diff --git a/Python/pystrtod.c b/Python/pystrtod.c
index 3546d44c84248b2d080115864d93c721f01374bc..461e8dcb5e0c4a879202234e748eecef9c2c2337 100644 (file)
--- a/Python/pystrtod.c
+++ b/Python/pystrtod.c
@@ -391,6 +391,8 @@ _Py_string_to_number_with_underscores(
     char *dup, *end;
     PyObject *result;
 
+    assert(s[orig_len] == '\0');
+
     if (strchr(s, '_') == NULL) {
         return innerfunc(s, orig_len, arg);
     }