From: Benjamin Peterson <benjamin@python.org>
Date: Sat, 3 Apr 2010 23:03:35 +0000 (+0000)
Subject: use our own locale independent ctype macros
X-Git-Tag: v2.7b1~83
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=88623d76b49a553445fff037bc9cf2e79a24ceef;p=python

use our own locale independent ctype macros

requires building pyctype.o into pgen
---

diff --git a/Makefile.pre.in b/Makefile.pre.in
index 36b0a56bd1..a332b6b793 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -230,6 +230,7 @@ PARSER_OBJS=	$(POBJS) Parser/myreadline.o Parser/tokenizer.o
 PGOBJS=		\
 		Objects/obmalloc.o \
 		Python/mysnprintf.o \
+                Python/pyctype.o \
 		Parser/tokenizer_pgen.o \
 		Parser/printgrammar.o \
 		Parser/pgenmain.o
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index fbbd0bc7fb..495182fc86 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -92,22 +92,6 @@ char *_PyParser_TokenNames[] = {
 	"<N_TOKENS>"
 };
 
-
-/* Ensure that the locale does not interfere with tokenization. */
-
-static int
-ascii_isalpha(int c)
-{
-	return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z');
-}
-
-static int
-ascii_isalnum(int c)
-{
-	return ascii_isalpha(c) || ('0' <= c && c <= '9');
-}
-
-
 /* Create and initialize a new tok_state structure */
 
 static struct tok_state *
@@ -245,7 +229,7 @@ get_coding_spec(const char *s, Py_ssize_t size)
 			} while (t[0] == '\x20' || t[0] == '\t');
 
 			begin = t;
-			while (ascii_isalnum(Py_CHARMASK(t[0])) ||
+			while (Py_ISALNUM(t[0]) ||
 			       t[0] == '-' || t[0] == '_' || t[0] == '.')
 				t++;
 
@@ -1355,7 +1339,7 @@ tok_get(register struct tok_state *tok, char **p_start, char **p_end)
 	}
 
 	/* Identifier (most frequent token!) */
-	if (ascii_isalpha(c) || c == '_') {
+	if (Py_ISALPHA(c) || c == '_') {
 		/* Process r"", u"" and ur"" */
 		switch (c) {
 		case 'b':
@@ -1381,7 +1365,7 @@ tok_get(register struct tok_state *tok, char **p_start, char **p_end)
 				goto letter_quote;
 			break;
 		}
-		while (ascii_isalnum(c) || c == '_') {
+		while (Py_ISALNUM(c) || c == '_') {
 			c = tok_nextc(tok);
 		}
 		tok_backup(tok, c);