Merged revisions 79723 via svnmerge from

svn+ssh://pythondev@svn.python.org/python/trunk

........
  r79723 | benjamin.peterson | 2010-04-03 17:48:51 -0500 (Sat, 03 Apr 2010) | 1 line

  ensure that the locale does not affect the tokenization of identifiers
........

diff --git a/Misc/NEWS b/Misc/NEWS
index a15fa917546a75fa6ffd2b65ae8cfd84ccabf490..e042975d78a3445dd18a223f91ec13bf2d4f1e1c 100644 (file)
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -12,6 +12,8 @@ What's New in Python 3.2 Alpha 1?
 Core and Builtins
 -----------------
 
+- Ensure that tokenization of identifiers is not affected by locale.
+
 - Issue #1222585: Added LDCXXSHARED for C++ support. Patch by Arfrever.
 
 - Raise a TypeError when trying to delete a T_STRING_INPLACE struct member.

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 0e2dc3a52d099c4509dc96b37f581fbc2d73bc24..be2940cd67ff9e555598d4a82b51d21e9a22cb39 100644 (file)
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -179,6 +179,16 @@ decode_str(const char *str, int exec_input, struct tok_state *tok)
 
 #else /* PGEN */
 
+/* Ensure that the locale does not interfere with tokenization. */
+
+static int
+ascii_isalnum(int c)
+{
+       return (('a' <= c && c <= 'z') ||
+               ('A' <= c && c <= 'Z') ||
+               ('0' <= c && c <= '9'));
+}
+
 static char *
 error_ret(struct tok_state *tok) /* XXX */
 {
@@ -245,7 +255,7 @@ get_coding_spec(const char *s, Py_ssize_t size)
                        } while (t[0] == '\x20' || t[0] == '\t');
 
                        begin = t;
-                       while (isalnum(Py_CHARMASK(t[0])) ||
+                       while (ascii_isalnum(Py_CHARMASK(t[0])) ||
                               t[0] == '-' || t[0] == '_' || t[0] == '.')
                                t++;