use floor division and add a test that exercises the tabsize codepath

diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index ae5f410b7a134155079a5e86a74a783275f107e0..e5371cfc42a8daf1a9d1c45c80416845b5b5c0db 100644 (file)
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -508,6 +508,23 @@ pass the '-ucompiler' option to process the full directory.
     ...         break
     ... else: True
     True
+
+Evil tabs
+    >>> dump_tokens("def f():\\n\\tif x\\n        \tpass")
+    NAME       'def'         (1, 0) (1, 3)
+    NAME       'f'           (1, 4) (1, 5)
+    OP         '('           (1, 5) (1, 6)
+    OP         ')'           (1, 6) (1, 7)
+    OP         ':'           (1, 7) (1, 8)
+    NEWLINE    '\\n'          (1, 8) (1, 9)
+    INDENT     '\\t'          (2, 0) (2, 1)
+    NAME       'if'          (2, 1) (2, 3)
+    NAME       'x'           (2, 4) (2, 5)
+    NEWLINE    '\\n'          (2, 5) (2, 6)
+    INDENT     '         '   (3, 0) (3, 9)
+    NAME       'pass'        (3, 9) (3, 13)
+    DEDENT     ''            (4, 0) (4, 0)
+    DEDENT     ''            (4, 0) (4, 0)
 """
 
 

diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index ad3cf9d1626a18804a96b8ed3a22f39a56f8772f..686ad66f6dd1a0813fba4b3b01ff1c6aa34185c8 100644 (file)
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -319,7 +319,7 @@ def generate_tokens(readline):
                 if line[pos] == ' ':
                     column = column + 1
                 elif line[pos] == '\t':
-                    column = (column/tabsize + 1)*tabsize
+                    column = (column//tabsize + 1)*tabsize
                 elif line[pos] == '\f':
                     column = 0
                 else: