#define COMP_LISTCOMP 1
#define COMP_SETCOMP 2
+static int
+init_normalization(struct compiling *c)
+{
+ PyObject *m = PyImport_ImportModuleNoBlock("unicodedata");
+ if (!m)
+ return 0;
+ c->c_normalize = PyObject_GetAttrString(m, "normalize");
+ Py_DECREF(m);
+ if (!c->c_normalize)
+ return 0;
+ c->c_normalize_args = Py_BuildValue("(sN)", "NFKC", Py_None);
+ PyTuple_SET_ITEM(c->c_normalize_args, 1, NULL);
+ if (!c->c_normalize_args) {
+ Py_CLEAR(c->c_normalize);
+ return 0;
+ }
+ return 1;
+}
+
static identifier
- new_identifier(const char* n, struct compiling *c)
-new_identifier(const char *n, PyArena *arena)
++new_identifier(const char *n, struct compiling *c)
{
- PyObject* id = PyUnicode_DecodeUTF8(n, strlen(n), NULL);
+ PyObject *id = PyUnicode_DecodeUTF8(n, strlen(n), NULL);
- Py_UNICODE *u;
if (!id)
return NULL;
- u = PyUnicode_AS_UNICODE(id);
+ /* PyUnicode_DecodeUTF8 should always return a ready string. */
+ assert(PyUnicode_IS_READY(id));
/* Check whether there are non-ASCII characters in the
identifier; if so, normalize to NFKC. */
- for (; *u; u++) {
- if (*u >= 128) {
- PyObject *m = PyImport_ImportModuleNoBlock("unicodedata");
- PyObject *id2;
- if (!m)
- return NULL;
- id2 = PyObject_CallMethod(m, "normalize", "sO", "NFKC", id);
- Py_DECREF(m);
- if (!id2)
- return NULL;
+ if (!PyUnicode_IS_ASCII(id)) {
+ PyObject *id2;
+ if (!c->c_normalize && !init_normalization(c)) {
Py_DECREF(id);
- id = id2;
- break;
+ return NULL;
}
+ PyTuple_SET_ITEM(c->c_normalize_args, 1, id);
+ id2 = PyObject_Call(c->c_normalize, c->c_normalize_args, NULL);
+ Py_DECREF(id);
+ if (!id2)
+ return NULL;
+ id = id2;
}
PyUnicode_InternInPlace(&id);
- PyArena_AddPyObject(arena, id);
+ PyArena_AddPyObject(c->c_arena, id);
return id;
}