if not line: break
column = 0
while pos < max: # measure leading whitespace
- if line[pos] == ' ': column = column + 1
- elif line[pos] == '\t': column = (column/tabsize + 1)*tabsize
- elif line[pos] == '\f': column = 0
- else: break
+ if line[pos] == ' ':
+ column = column + 1
+ elif line[pos] == '\t':
+ column = (column/tabsize + 1)*tabsize
+ elif line[pos] == '\f':
+ column = 0
+ else:
+ break
pos = pos + 1
- if pos == max: break
+ if pos == max:
+ break
if line[pos] in '#\r\n': # skip comments or blank lines
if line[pos] == '#':
elif initial == '\\': # continued stmt
continued = 1
else:
- if initial in '([{': parenlev = parenlev + 1
- elif initial in ')]}': parenlev = parenlev - 1
+ if initial in '([{':
+ parenlev = parenlev + 1
+ elif initial in ')]}':
+ parenlev = parenlev - 1
yield (OP, token, spos, epos, line)
else:
yield (ERRORTOKEN, line[pos],
if __name__ == '__main__': # testing
import sys
- if len(sys.argv) > 1: tokenize(open(sys.argv[1]).readline)
- else: tokenize(sys.stdin.readline)
+ if len(sys.argv) > 1:
+ tokenize(open(sys.argv[1]).readline)
+ else:
+ tokenize(sys.stdin.readline)