From: Ezio Melotti <ezio.melotti@gmail.com>
Date: Sat, 3 Nov 2012 15:46:51 +0000 (+0200)
Subject: #16152: merge with 3.2.
X-Git-Tag: v3.3.1rc1~696
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=fafa8b7797034aa799157172392a5b7721e1854a;p=python

#16152: merge with 3.2.
---

fafa8b7797034aa799157172392a5b7721e1854a
diff --cc Lib/test/test_tokenize.py
index 213e9b46ce,f9652ce7b2..b4a58f0db2
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@@ -1036,79 -941,7 +1036,83 @@@ class TestTokenize(TestCase)
  
          self.assertTrue(encoding_used, encoding)
  
 +    def assertExactTypeEqual(self, opstr, *optypes):
 +        tokens = list(tokenize(BytesIO(opstr.encode('utf-8')).readline))
 +        num_optypes = len(optypes)
 +        self.assertEqual(len(tokens), 2 + num_optypes)
 +        self.assertEqual(token.tok_name[tokens[0].exact_type],
 +                         token.tok_name[ENCODING])
 +        for i in range(num_optypes):
 +            self.assertEqual(token.tok_name[tokens[i + 1].exact_type],
 +                             token.tok_name[optypes[i]])
 +        self.assertEqual(token.tok_name[tokens[1 + num_optypes].exact_type],
 +                         token.tok_name[token.ENDMARKER])
 +
 +    def test_exact_type(self):
 +        self.assertExactTypeEqual('()', token.LPAR, token.RPAR)
 +        self.assertExactTypeEqual('[]', token.LSQB, token.RSQB)
 +        self.assertExactTypeEqual(':', token.COLON)
 +        self.assertExactTypeEqual(',', token.COMMA)
 +        self.assertExactTypeEqual(';', token.SEMI)
 +        self.assertExactTypeEqual('+', token.PLUS)
 +        self.assertExactTypeEqual('-', token.MINUS)
 +        self.assertExactTypeEqual('*', token.STAR)
 +        self.assertExactTypeEqual('/', token.SLASH)
 +        self.assertExactTypeEqual('|', token.VBAR)
 +        self.assertExactTypeEqual('&', token.AMPER)
 +        self.assertExactTypeEqual('<', token.LESS)
 +        self.assertExactTypeEqual('>', token.GREATER)
 +        self.assertExactTypeEqual('=', token.EQUAL)
 +        self.assertExactTypeEqual('.', token.DOT)
 +        self.assertExactTypeEqual('%', token.PERCENT)
 +        self.assertExactTypeEqual('{}', token.LBRACE, token.RBRACE)
 +        self.assertExactTypeEqual('==', token.EQEQUAL)
 +        self.assertExactTypeEqual('!=', token.NOTEQUAL)
 +        self.assertExactTypeEqual('<=', token.LESSEQUAL)
 +        self.assertExactTypeEqual('>=', token.GREATEREQUAL)
 +        self.assertExactTypeEqual('~', token.TILDE)
 +        self.assertExactTypeEqual('^', token.CIRCUMFLEX)
 +        self.assertExactTypeEqual('<<', token.LEFTSHIFT)
 +        self.assertExactTypeEqual('>>', token.RIGHTSHIFT)
 +        self.assertExactTypeEqual('**', token.DOUBLESTAR)
 +        self.assertExactTypeEqual('+=', token.PLUSEQUAL)
 +        self.assertExactTypeEqual('-=', token.MINEQUAL)
 +        self.assertExactTypeEqual('*=', token.STAREQUAL)
 +        self.assertExactTypeEqual('/=', token.SLASHEQUAL)
 +        self.assertExactTypeEqual('%=', token.PERCENTEQUAL)
 +        self.assertExactTypeEqual('&=', token.AMPEREQUAL)
 +        self.assertExactTypeEqual('|=', token.VBAREQUAL)
 +        self.assertExactTypeEqual('^=', token.CIRCUMFLEXEQUAL)
 +        self.assertExactTypeEqual('^=', token.CIRCUMFLEXEQUAL)
 +        self.assertExactTypeEqual('<<=', token.LEFTSHIFTEQUAL)
 +        self.assertExactTypeEqual('>>=', token.RIGHTSHIFTEQUAL)
 +        self.assertExactTypeEqual('**=', token.DOUBLESTAREQUAL)
 +        self.assertExactTypeEqual('//', token.DOUBLESLASH)
 +        self.assertExactTypeEqual('//=', token.DOUBLESLASHEQUAL)
 +        self.assertExactTypeEqual('@', token.AT)
 +
 +        self.assertExactTypeEqual('a**2+b**2==c**2',
 +                                  NAME, token.DOUBLESTAR, NUMBER,
 +                                  token.PLUS,
 +                                  NAME, token.DOUBLESTAR, NUMBER,
 +                                  token.EQEQUAL,
 +                                  NAME, token.DOUBLESTAR, NUMBER)
 +        self.assertExactTypeEqual('{1, 2, 3}',
 +                                  token.LBRACE,
 +                                  token.NUMBER, token.COMMA,
 +                                  token.NUMBER, token.COMMA,
 +                                  token.NUMBER,
 +                                  token.RBRACE)
 +        self.assertExactTypeEqual('^(x & 0x1)',
 +                                  token.CIRCUMFLEX,
 +                                  token.LPAR,
 +                                  token.NAME, token.AMPER, token.NUMBER,
 +                                  token.RPAR)
 +
++    def test_pathological_trailing_whitespace(self):
++        # See http://bugs.python.org/issue16152
++        self.assertExactTypeEqual('@          ', token.AT)
+ 
  __test__ = {"doctests" : doctests, 'decistmt': decistmt}
  
  def test_main():
diff --cc Lib/tokenize.py
index d669412db3,29c9e29b30..cbf91ef222
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@@ -158,11 -104,11 +158,11 @@@ PlainToken = group(Number, Funny, Strin
  Token = Ignore + PlainToken
  
  # First (or only) line of ' or " string.
 -ContStr = group(r"[bB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
 +ContStr = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
                  group("'", r'\\\r?\n'),
 -                r'[bB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
 +                StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
                  group('"', r'\\\r?\n'))
- PseudoExtras = group(r'\\\r?\n', Comment, Triple)
+ PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple)
  PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
  
  def _compile(expr):
diff --cc Misc/ACKS
index d80ead042e,1095733839..ee1ce675c6
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@@ -75,12 -70,12 +75,13 @@@ Cesar Eduardo Barro
  Des Barry
  Ulf Bartelt
  Don Bashford
 +Pior Bastida
  Nick Bastin
+ Ned Batchelder
  Jeff Bauer
 -Mike Bayer
  Michael R Bax
  Anthony Baxter
 +Mike Bayer
  Samuel L. Bayer
  Donald Beaudry
  David Beazley