Merged revisions 79725 via svnmerge from

svn+ssh://pythondev@svn.python.org/python/trunk

........
  r79725 | benjamin.peterson | 2010-04-03 18:03:35 -0500 (Sat, 03 Apr 2010) | 4 lines

  use our own locale independent ctype macros

  requires building pyctype.o into pgen
........

diff --git a/Makefile.pre.in b/Makefile.pre.in
index 898c4f0ace1243208ab7328c4403ba2fa431a21d..c704b145fe97654a74574fbfb43d3035a87dba5e 100644 (file)
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -239,6 +239,7 @@ PARSER_OBJS=        $(POBJS) Parser/myreadline.o Parser/tokenizer.o
 PGOBJS=                \
                Objects/obmalloc.o \
                Python/mysnprintf.o \
+                Python/pyctype.o \
                Parser/tokenizer_pgen.o \
                Parser/printgrammar.o \
                Parser/pgenmain.o

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index be2940cd67ff9e555598d4a82b51d21e9a22cb39..77fec7400857428cb688a3b7c602f542f4aee9e3 100644 (file)
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -179,16 +179,6 @@ decode_str(const char *str, int exec_input, struct tok_state *tok)
 
 #else /* PGEN */
 
-/* Ensure that the locale does not interfere with tokenization. */
-
-static int
-ascii_isalnum(int c)
-{
-       return (('a' <= c && c <= 'z') ||
-               ('A' <= c && c <= 'Z') ||
-               ('0' <= c && c <= '9'));
-}
-
 static char *
 error_ret(struct tok_state *tok) /* XXX */
 {
@@ -255,7 +245,7 @@ get_coding_spec(const char *s, Py_ssize_t size)
                        } while (t[0] == '\x20' || t[0] == '\t');
 
                        begin = t;
-                       while (ascii_isalnum(Py_CHARMASK(t[0])) ||
+                       while (Py_ISALNUM(t[0]) ||
                               t[0] == '-' || t[0] == '_' || t[0] == '.')
                                t++;