Added PEP 3127 support to tokenize (with tests); added PEP 3127 to NEWS.

diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index d6cfb65d3329874d4ac4b71602b61e61a9c4c04f..cbfafa8d1401191d4ecb282d6f49c4b772bdb974 100644 (file)
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -4,7 +4,7 @@ Tests for the tokenize module.
     >>> import glob, random, sys
 
 The tests can be really simple. Given a small fragment of source
-code, print out a table with thokens. The ENDMARK is omitted for
+code, print out a table with tokens. The ENDMARK is omitted for
 brevity.
 
     >>> dump_tokens("1 + 1")
@@ -106,7 +106,7 @@ Some error-handling code
     ...           "else:   print 'Loaded'\\n")
     True
 
-Balancing contunuation
+Balancing continuation
 
     >>> roundtrip("a = (3,4, \\n"
     ...           "5,6)\\n"
@@ -126,6 +126,14 @@ Ordinary integers and binary operators
     NUMBER     '0xff'        (1, 0) (1, 4)
     OP         '<='          (1, 5) (1, 7)
     NUMBER     '255'         (1, 8) (1, 11)
+    >>> dump_tokens("0b10 <= 255")
+    NUMBER     '0b10'        (1, 0) (1, 4)
+    OP         '<='          (1, 5) (1, 7)
+    NUMBER     '255'         (1, 8) (1, 11)
+    >>> dump_tokens("0o123 <= 0123")
+    NUMBER     '0o123'       (1, 0) (1, 5)
+    OP         '<='          (1, 6) (1, 8)
+    NUMBER     '0123'        (1, 9) (1, 13)
     >>> dump_tokens("01234567 > ~0x15")
     NUMBER     '01234567'    (1, 0) (1, 8)
     OP         '>'           (1, 9) (1, 10)

diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index 1c9394436818853f4a8390e293aa9c37331399d2..76ab4309508892849464febef4d4fcd2c89baf55 100644 (file)
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -51,9 +51,10 @@ Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
 Name = r'[a-zA-Z_]\w*'
 
 Hexnumber = r'0[xX][\da-fA-F]+[lL]?'
-Octnumber = r'0[0-7]*[lL]?'
+Octnumber = r'(0[oO][0-7]+)|(0[0-7]*)[lL]?'
+Binnumber = r'0[bB][01]+[lL]?'
 Decnumber = r'[1-9]\d*[lL]?'
-Intnumber = group(Hexnumber, Octnumber, Decnumber)
+Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)
 Exponent = r'[eE][-+]?\d+'
 Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent)
 Expfloat = r'\d+' + Exponent

diff --git a/Misc/NEWS b/Misc/NEWS
index 1963ad5950e6958571a4483699a9195cb2d01b56..1fc6691f216f4ae2942bac6dcd176379f0207f9c 100644 (file)
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -12,6 +12,10 @@ What's New in Python 2.6 alpha 2?
 Core and builtins
 -----------------
 
+- PEP 3127: octal literals now start with "0o". Old-style octal literals
+  are still valid. There are binary literals with a prefix of "0b".
+  This also affects int(x, 0).
+
 - Issue #1779871: Gnu gcc can now build Python on OS X because the
   flags -Wno-long-double, -no-cpp-precomp, and -mno-fused-madd are no
   longer passed.