From: Benjamin Peterson Date: Sun, 2 Sep 2012 20:37:09 +0000 (-0400) Subject: merge 3.2 X-Git-Tag: v3.3.1rc1~818^2^2~149 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=d40528fe9a27c4656a2b3edeef2b71b3a5afe000;p=python merge 3.2 --- d40528fe9a27c4656a2b3edeef2b71b3a5afe000 diff --cc Python/ast.c index aca5b09f03,4ae9d75d03..43c18f4aa7 --- a/Python/ast.c +++ b/Python/ast.c @@@ -525,50 -46,33 +525,50 @@@ static PyObject *parsestrplus(struct co #define COMP_LISTCOMP 1 #define COMP_SETCOMP 2 +static int +init_normalization(struct compiling *c) +{ + PyObject *m = PyImport_ImportModuleNoBlock("unicodedata"); + if (!m) + return 0; + c->c_normalize = PyObject_GetAttrString(m, "normalize"); + Py_DECREF(m); + if (!c->c_normalize) + return 0; + c->c_normalize_args = Py_BuildValue("(sN)", "NFKC", Py_None); + PyTuple_SET_ITEM(c->c_normalize_args, 1, NULL); + if (!c->c_normalize_args) { + Py_CLEAR(c->c_normalize); + return 0; + } + return 1; +} + static identifier - new_identifier(const char* n, struct compiling *c) -new_identifier(const char *n, PyArena *arena) ++new_identifier(const char *n, struct compiling *c) { - PyObject* id = PyUnicode_DecodeUTF8(n, strlen(n), NULL); + PyObject *id = PyUnicode_DecodeUTF8(n, strlen(n), NULL); - Py_UNICODE *u; if (!id) return NULL; - u = PyUnicode_AS_UNICODE(id); + /* PyUnicode_DecodeUTF8 should always return a ready string. */ + assert(PyUnicode_IS_READY(id)); /* Check whether there are non-ASCII characters in the identifier; if so, normalize to NFKC. */ - for (; *u; u++) { - if (*u >= 128) { - PyObject *m = PyImport_ImportModuleNoBlock("unicodedata"); - PyObject *id2; - if (!m) - return NULL; - id2 = PyObject_CallMethod(m, "normalize", "sO", "NFKC", id); - Py_DECREF(m); - if (!id2) - return NULL; + if (!PyUnicode_IS_ASCII(id)) { + PyObject *id2; + if (!c->c_normalize && !init_normalization(c)) { Py_DECREF(id); - id = id2; - break; + return NULL; } + PyTuple_SET_ITEM(c->c_normalize_args, 1, id); + id2 = PyObject_Call(c->c_normalize, c->c_normalize_args, NULL); + Py_DECREF(id); + if (!id2) + return NULL; + id = id2; } PyUnicode_InternInPlace(&id); - PyArena_AddPyObject(arena, id); + PyArena_AddPyObject(c->c_arena, id); return id; }