use our own locale independent ctype macros

requires building pyctype.o into pgen

diff --git a/Makefile.pre.in b/Makefile.pre.in
index 36b0a56bd154dab9249c2b1e0dcc677a54aa0fdc..a332b6b793d17cbd756193342e3c3ed8844625d8 100644 (file)
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -230,6 +230,7 @@ PARSER_OBJS=        $(POBJS) Parser/myreadline.o Parser/tokenizer.o
 PGOBJS=                \
                Objects/obmalloc.o \
                Python/mysnprintf.o \
+                Python/pyctype.o \
                Parser/tokenizer_pgen.o \
                Parser/printgrammar.o \
                Parser/pgenmain.o

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index fbbd0bc7fb3288e2c4d128414fc11ac38d3d3cbb..495182fc8611f380a8edad2345dffbbc6a9a09b3 100644 (file)
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -92,22 +92,6 @@ char *_PyParser_TokenNames[] = {
        "<N_TOKENS>"
 };
 
-
-/* Ensure that the locale does not interfere with tokenization. */
-
-static int
-ascii_isalpha(int c)
-{
-       return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z');
-}
-
-static int
-ascii_isalnum(int c)
-{
-       return ascii_isalpha(c) || ('0' <= c && c <= '9');
-}
-
-
 /* Create and initialize a new tok_state structure */
 
 static struct tok_state *
@@ -245,7 +229,7 @@ get_coding_spec(const char *s, Py_ssize_t size)
                        } while (t[0] == '\x20' || t[0] == '\t');
 
                        begin = t;
-                       while (ascii_isalnum(Py_CHARMASK(t[0])) ||
+                       while (Py_ISALNUM(t[0]) ||
                               t[0] == '-' || t[0] == '_' || t[0] == '.')
                                t++;
 
@@ -1355,7 +1339,7 @@ tok_get(register struct tok_state *tok, char **p_start, char **p_end)
        }
 
        /* Identifier (most frequent token!) */
-       if (ascii_isalpha(c) || c == '_') {
+       if (Py_ISALPHA(c) || c == '_') {
                /* Process r"", u"" and ur"" */
                switch (c) {
                case 'b':
@@ -1381,7 +1365,7 @@ tok_get(register struct tok_state *tok, char **p_start, char **p_end)
                                goto letter_quote;
                        break;
                }
-               while (ascii_isalnum(c) || c == '_') {
+               while (Py_ISALNUM(c) || c == '_') {
                        c = tok_nextc(tok);
                }
                tok_backup(tok, c);