From: Guido van Rossum <guido@python.org>
Date: Mon, 19 Jan 1998 22:07:46 +0000 (+0000)
Subject: tok_nextc() should return unsigned characters, to avoid mistaking
X-Git-Tag: v1.5.1~936
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=cf57d8b8c92143342b4d978c6134d307114fbf44;p=python

tok_nextc() should return unsigned characters, to avoid mistaking
'\377' for EOF.
---

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 3dc6c82799..134f00f252 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -46,6 +46,14 @@ extern char *PyOS_Readline Py_PROTO((char *));
 /* Don't ever change this -- it would break the portability of Python code */
 #define TABSIZE 8
 
+/* Convert a possibly signed character to a nonnegative int */
+/* XXX This assumes characters are 8 bits wide */
+#ifdef __CHAR_UNSIGNED__
+#define Py_CHARMASK(c)		(c)
+#else
+#define Py_CHARMASK(c)		((c) & 0xff)
+#endif
+
 /* Forward */
 static struct tok_state *tok_new Py_PROTO((void));
 static int tok_nextc Py_PROTO((struct tok_state *tok));
@@ -178,7 +186,7 @@ tok_nextc(tok)
 {
 	for (;;) {
 		if (tok->cur != tok->inp) {
-			return *tok->cur++; /* Fast path */
+			return Py_CHARMASK(*tok->cur++); /* Fast path */
 		}
 		if (tok->done != E_OK)
 			return EOF;
@@ -197,7 +205,7 @@ tok_nextc(tok)
 				tok->buf = tok->cur;
 			tok->lineno++;
 			tok->inp = end;
-			return *tok->cur++;
+			return Py_CHARMASK(*tok->cur++);
 		}
 		if (tok->prompt != NULL) {
 			char *new = PyOS_Readline(tok->prompt);