This patch adds a new Python C API called PyString_AsStringAndSize()

which implements the automatic conversion from Unicode to a string
object using the default encoding.

The new API is then put to use to have eval() and exec accept
Unicode objects as code parameter. This closes bugs #110924
and #113890.

As side-effect, the traditional C APIs PyString_Size() and
PyString_AsString() will also accept Unicode objects as
parameters.

diff --git a/Doc/api/api.tex b/Doc/api/api.tex
index 43624487f2b148e8206cea338b1c9a0c131e84d1..9acc8e85646a12cf4894a861c706c97c1ff4ca07 100644 (file)
--- a/Doc/api/api.tex
+++ b/Doc/api/api.tex
@@ -2105,6 +2105,23 @@ Macro form of \cfunction{PyString_AsString()} but without error
 checking.
 \end{cfuncdesc}
 
+\begin{cfuncdesc}{int}{PyString_AsStringAndSize}{PyObject *obj,
+                                                 char **buffer,
+                                                 int *length}
+Returns a null-terminated representation of the contents of the object
+\var{obj} through the output variables \var{buffer} and \var{length}.
+
+The function accepts both string and Unicode objects as input. For
+Unicode objects it returns the default encoded version of the object.
+If \var{length} is set to \NULL{}, the resulting buffer may not contain
+null characters; if it does, the function returns -1 and a
+TypeError is raised.
+
+The buffer refers to an internal string buffer of \var{obj}, not a
+copy. The data must not be modified in any way.  It must not be
+de-allocated.
+\end{cfuncdesc}
+
 \begin{cfuncdesc}{void}{PyString_Concat}{PyObject **string,
                                          PyObject *newpart}
 Creates a new string object in \var{*string} containing the

diff --git a/Doc/api/refcounts.dat b/Doc/api/refcounts.dat
index 0a80d67ba68032bbd925ee0da9de1735e65e7feb..f3ef0ac3708713297e07203d77e5d70c09acb282 100644 (file)
--- a/Doc/api/refcounts.dat
+++ b/Doc/api/refcounts.dat
@@ -760,6 +760,11 @@ PyString_AS_STRING:PyObject*:string:0:
 PyString_AsString:char*:::
 PyString_AsString:PyObject*:string:0:
 
+PyString_AsStringAndSize:int:::
+PyString_AsStringAndSize:PyObject*:obj:0:
+PyString_AsStringAndSize:char**:buffer::
+PyString_AsStringAndSize:int*:length::
+
 PyString_Check:int:::
 PyString_Check:PyObject*:o:0:
 

diff --git a/Include/stringobject.h b/Include/stringobject.h
index 7afd347c856eb38d3e8df169246cd4ef73c5de83..3bba7bc1f32f19ec5812f16a35b9c97f950d39d1 100644 (file)
--- a/Include/stringobject.h
+++ b/Include/stringobject.h
@@ -103,6 +103,21 @@ extern DL_IMPORT(PyObject*) PyString_AsEncodedString(
     const char *errors         /* error handling */
     );
 
+/* Provides access to the internal data buffer and size of a string
+   object or the default encoded version of an Unicode object. Passing
+   NULL as *len parameter will force the string buffer to be
+   0-terminated (passing a string with embedded NULL characters will
+   cause an exception).  */
+
+extern DL_IMPORT(int) PyString_AsStringAndSize(
+    register PyObject *obj,    /* string or Unicode object */
+    register char **s,         /* pointer to buffer variable */
+    register int *len          /* pointer to length variable or NULL
+                                  (only possible for 0-terminated
+                                  strings) */
+    );
+    
+
 #ifdef __cplusplus
 }
 #endif

diff --git a/Lib/test/test_b1.py b/Lib/test/test_b1.py
index f8dfe4725378588a8fcec3102895bc33d4991f9d..24c52795be5f24a11dd5a3991273066fd72daccf 100644 (file)
--- a/Lib/test/test_b1.py
+++ b/Lib/test/test_b1.py
@@ -161,6 +161,18 @@ if eval('b', globals, locals) <> 200:
     raise TestFailed, "eval(3)"
 if eval('c', globals, locals) <> 300:
     raise TestFailed, "eval(4)"
+if eval(u'1+1') <> 2: raise TestFailed, 'eval(u\'1+1\')'
+if eval(u' 1+1\n') <> 2: raise TestFailed, 'eval(u\' 1+1\\n\')'
+globals = {'a': 1, 'b': 2}
+locals = {'b': 200, 'c': 300}
+if eval(u'a', globals) <> 1:
+    raise TestFailed, "eval(1) == %s" % eval(u'a', globals)
+if eval(u'a', globals, locals) <> 1:
+    raise TestFailed, "eval(2)"
+if eval(u'b', globals, locals) <> 200:
+    raise TestFailed, "eval(3)"
+if eval(u'c', globals, locals) <> 300:
+    raise TestFailed, "eval(4)"
 
 print 'execfile'
 z = 0

diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py
index ef7c09b9a57fe8ace098772bd4a1b89a3353a10c..68cae81f5076f4076ba05e651ddd888121a2dae6 100644 (file)
--- a/Lib/test/test_grammar.py
+++ b/Lib/test/test_grammar.py
@@ -355,6 +355,13 @@ def f():
        del z
        exec 'z=1+1'
        if z <> 2: raise TestFailed, 'exec \'z=1+1\''
+       z = None
+       del z
+       exec u'z=1+1\n'
+       if z <> 2: raise TestFailed, 'exec u\'z=1+1\'\\n'
+       del z
+       exec u'z=1+1'
+       if z <> 2: raise TestFailed, 'exec u\'z=1+1\''
 f()
 g = {}
 exec 'z = 1' in g

diff --git a/Objects/stringobject.c b/Objects/stringobject.c
index eee355173a7b9bb1905137ccaaf37fcf781c9855..cadca1685889228c5d0a38b260689f666d7c07ea 100644 (file)
--- a/Objects/stringobject.c
+++ b/Objects/stringobject.c
@@ -239,24 +239,80 @@ string_dealloc(PyObject *op)
        PyObject_DEL(op);
 }
 
+static int
+string_getsize(register PyObject *op)
+{
+       char *s;
+       int len;
+       if (PyString_AsStringAndSize(op, &s, &len))
+               return -1;
+       return len;
+}
+
+static /*const*/ char *
+string_getbuffer(register PyObject *op)
+{
+       char *s;
+       int len;
+       if (PyString_AsStringAndSize(op, &s, &len))
+               return NULL;
+       return s;
+}
+
 int
 PyString_Size(register PyObject *op)
 {
-       if (!PyString_Check(op)) {
-               PyErr_BadInternalCall();
-               return -1;
-       }
+       if (!PyString_Check(op))
+               return string_getsize(op);
        return ((PyStringObject *)op) -> ob_size;
 }
 
 /*const*/ char *
 PyString_AsString(register PyObject *op)
 {
-       if (!PyString_Check(op)) {
+       if (!PyString_Check(op))
+               return string_getbuffer(op);
+       return ((PyStringObject *)op) -> ob_sval;
+}
+
+/* Internal API needed by PyString_AsStringAndSize(): */
+extern 
+PyObject *_PyUnicode_AsDefaultEncodedString(PyObject *unicode,
+                                           const char *errors);
+
+int
+PyString_AsStringAndSize(register PyObject *obj,
+                        register char **s,
+                        register int *len)
+{
+       if (s == NULL) {
                PyErr_BadInternalCall();
-               return NULL;
+               return -1;
        }
-       return ((PyStringObject *)op) -> ob_sval;
+
+       if (!PyString_Check(obj)) {
+               if (PyUnicode_Check(obj)) {
+                       obj = _PyUnicode_AsDefaultEncodedString(obj, NULL);
+                       if (obj == NULL)
+                               return -1;
+               }
+               else {
+                       PyErr_Format(PyExc_TypeError,
+                                    "expected string or Unicode object, "
+                                    "%.200s found", obj->ob_type->tp_name);
+                       return -1;
+               }
+       }
+
+       *s = PyString_AS_STRING(obj);
+       if (len != NULL)
+               *len = PyString_GET_SIZE(obj);
+       else if ((int)strlen(*s) != PyString_GET_SIZE(obj)) {
+               PyErr_SetString(PyExc_TypeError,
+                               "expected string without null bytes");
+               return -1;
+       }
+       return 0;
 }
 
 /* Methods */

diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c
index 3eac8d52d9a53ddcd02876ca2f8d02ec4a8af825..88656ca630f09cec1d8f2e5589b365b1b97206f1 100644 (file)
--- a/Python/bltinmodule.c
+++ b/Python/bltinmodule.c
@@ -748,17 +748,14 @@ builtin_eval(PyObject *self, PyObject *args)
        }
        if (PyCode_Check(cmd))
                return PyEval_EvalCode((PyCodeObject *) cmd, globals, locals);
-       if (!PyString_Check(cmd)) {
+       if (!PyString_Check(cmd) &&
+           !PyUnicode_Check(cmd)) {
                PyErr_SetString(PyExc_TypeError,
                           "eval() argument 1 must be string or code object");
                return NULL;
        }
-       str = PyString_AsString(cmd);
-       if (strlen(str) != (size_t)PyString_Size(cmd)) {
-               PyErr_SetString(PyExc_ValueError,
-                          "embedded '\\0' in string arg");
+       if (PyString_AsStringAndSize(cmd, &str, NULL))
                return NULL;
-       }
        while (*str == ' ' || *str == '\t')
                str++;
        return PyRun_String(str, Py_eval_input, globals, locals);

diff --git a/Python/ceval.c b/Python/ceval.c
index 09ae132c2fe188c7681125b0a2780de67ecf69d6..491a73bdf1d8c3f40c8cca9af5f90b2b3509053a 100644 (file)
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -3042,6 +3042,7 @@ exec_statement(PyFrameObject *f, PyObject *prog, PyObject *globals,
        else if (locals == Py_None)
                locals = globals;
        if (!PyString_Check(prog) &&
+           !PyUnicode_Check(prog) &&
            !PyCode_Check(prog) &&
            !PyFile_Check(prog)) {
                PyErr_SetString(PyExc_TypeError,
@@ -3064,13 +3065,10 @@ exec_statement(PyFrameObject *f, PyObject *prog, PyObject *globals,
                v = PyRun_File(fp, name, Py_file_input, globals, locals);
        }
        else {
-               char *s = PyString_AsString(prog);
-               if (strlen(s) != (size_t)PyString_Size(prog)) {
-                       PyErr_SetString(PyExc_ValueError,
-                                       "embedded '\\0' in exec string");
+               char *str;
+               if (PyString_AsStringAndSize(prog, &str, NULL))
                        return -1;
-               }
-               v = PyRun_String(s, Py_file_input, globals, locals);
+               v = PyRun_String(str, Py_file_input, globals, locals);
        }
        if (plain)
                PyFrame_LocalsToFast(f, 0);