From 8daa49ee34667899af5687b322603fc71d848d3b Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sat, 3 Apr 2010 22:55:48 +0000 Subject: [PATCH] Merged revisions 79723 via svnmerge from svn+ssh://pythondev@svn.python.org/python/trunk ........ r79723 | benjamin.peterson | 2010-04-03 17:48:51 -0500 (Sat, 03 Apr 2010) | 1 line ensure that the locale does not affect the tokenization of identifiers ........ --- Misc/NEWS | 2 ++ Parser/tokenizer.c | 12 +++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS b/Misc/NEWS index a15fa91754..e042975d78 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -12,6 +12,8 @@ What's New in Python 3.2 Alpha 1? Core and Builtins ----------------- +- Ensure that tokenization of identifiers is not affected by locale. + - Issue #1222585: Added LDCXXSHARED for C++ support. Patch by Arfrever. - Raise a TypeError when trying to delete a T_STRING_INPLACE struct member. diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 0e2dc3a52d..be2940cd67 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -179,6 +179,16 @@ decode_str(const char *str, int exec_input, struct tok_state *tok) #else /* PGEN */ +/* Ensure that the locale does not interfere with tokenization. */ + +static int +ascii_isalnum(int c) +{ + return (('a' <= c && c <= 'z') || + ('A' <= c && c <= 'Z') || + ('0' <= c && c <= '9')); +} + static char * error_ret(struct tok_state *tok) /* XXX */ { @@ -245,7 +255,7 @@ get_coding_spec(const char *s, Py_ssize_t size) } while (t[0] == '\x20' || t[0] == '\t'); begin = t; - while (isalnum(Py_CHARMASK(t[0])) || + while (ascii_isalnum(Py_CHARMASK(t[0])) || t[0] == '-' || t[0] == '_' || t[0] == '.') t++; -- 2.40.0