Improve the repr for the TokenInfo named tuple.

diff --git a/Doc/library/tokenize.rst b/Doc/library/tokenize.rst
index 446d3bbde985fa7bb8d80fe3701d38a9cee4ca32..dbd01c478d594ba7740200229c64d0f336681343 100644 (file)
--- a/Doc/library/tokenize.rst
+++ b/Doc/library/tokenize.rst
@@ -111,7 +111,7 @@ function it uses to do this is available:
                 return fp.read()
 
 
-Example of a script re-writer that transforms float literals into Decimal
+Example of a script rewriter that transforms float literals into Decimal
 objects::
 
     from tokenize import tokenize, untokenize, NUMBER, STRING, NAME, OP

diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index aa4b246246d35e69562cd515a5e79db336ccb0c7..c97ed8a634bcdda72e1542323083c0dd2080536d 100644 (file)
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -63,7 +63,9 @@ class TokenInfo(tuple):
         return result
 
     def __repr__(self):
-        return 'TokenInfo(type=%r, string=%r, start=%r, end=%r, line=%r)' % self
+        typ = self[0]
+        return 'TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)' % \
+               ((('%d (%s)' % (typ, tok_name[typ])),) + self[1:])
 
     def _asdict(self):
         'Return a new dict which maps field names to their values'
@@ -550,3 +552,28 @@ def _tokenize(readline, encoding):
 # library that expect to be able to use tokenize with strings
 def generate_tokens(readline):
     return _tokenize(readline, None)
+
+if __name__ == "__main__":
+    # Quick sanity check
+    s = b'''def parseline(self, line):
+            """Parse the line into a command name and a string containing
+            the arguments.  Returns a tuple containing (command, args, line).
+            'command' and 'args' may be None if the line couldn't be parsed.
+            """
+            line = line.strip()
+            if not line:
+                return None, None, line
+            elif line[0] == '?':
+                line = 'help ' + line[1:]
+            elif line[0] == '!':
+                if hasattr(self, 'do_shell'):
+                    line = 'shell ' + line[1:]
+                else:
+                    return None, None, line
+            i, n = 0, len(line)
+            while i < n and line[i] in self.identchars: i = i+1
+            cmd, arg = line[:i], line[i:].strip()
+            return cmd, arg, line
+    '''
+    for tok in tokenize(iter(s.splitlines()).__next__):
+        print(tok)