Marc-Andre Lemburg: add new string token types u"..." and ur"..."

(Unicode and raw Unicode).

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 08b6ee9eec60af78241d3a194efc3a9d3d7b51d0..ff564ee7ca13d9eb6d6c205a4448fc4897b3970d 100644 (file)
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -591,12 +591,22 @@ PyTokenizer_Get(tok, p_start, p_end)
        
        /* Identifier (most frequent token!) */
        if (isalpha(c) || c == '_') {
+               /* Process r"", u"" and ur"" */
                switch (c) {
                case 'r':
                case 'R':
                        c = tok_nextc(tok);
                        if (c == '"' || c == '\'')
                                goto letter_quote;
+                       break;
+               case 'u':
+               case 'U':
+                       c = tok_nextc(tok);
+                       if (c == 'r' || c == 'R')
+                               c = tok_nextc(tok);
+                       if (c == '"' || c == '\'')
+                               goto letter_quote;
+                       break;
                }
                while (isalnum(c) || c == '_') {
                        c = tok_nextc(tok);