#3367 from Kristjan Valur Jonsson:

If a PyTokenizer_FromString() is called with an empty string, the
tokenizer's line_start member never gets initialized.  Later, it is
compared with the token pointer 'a' in parsetok.c:193 and that behavior
can result in undefined behavior.

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 1d0a4aa3f23b438dc5c17185bbd973130582f436..73ff92a19939abb7573a95a0823b050fabf31f50 100644 (file)
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -1117,7 +1117,7 @@ tok_get(register struct tok_state *tok, char **p_start, char **p_end)
        register int c;
        int blankline;
 
-       *p_start = *p_end = NULL;
+       tok->line_start = *p_start = *p_end = NULL;
   nextline:
        tok->start = NULL;
        blankline = 0;