import_name< 'import' imp=any >
"""
+ def start_tree(self, tree, name):
+ super(FixImport, self).start_tree(tree, name)
+ self.skip = "absolute_import" in tree.future_features
+
def transform(self, node, results):
+ if self.skip:
+ return
imp = results['imp']
if node.type == syms.import_from:
self.warning(node, "absolute and local imports together")
return
- new = FromImport('.', [imp])
+ new = FromImport(u".", [imp])
new.prefix = node.prefix
return new
def probably_a_local_import(self, imp_name):
- imp_name = imp_name.split('.', 1)[0]
+ if imp_name.startswith(u"."):
+ # Relative imports are certainly not local imports.
+ return False
+ imp_name = imp_name.split(u".", 1)[0]
base_path = dirname(self.filename)
base_path = join(base_path, imp_name)
# If there is no __init__.py next to the file its not in a package
# so can't be a relative import.
- if not exists(join(dirname(base_path), '__init__.py')):
+ if not exists(join(dirname(base_path), "__init__.py")):
return False
- for ext in ['.py', sep, '.pyc', '.so', '.sl', '.pyd']:
+ for ext in [".py", sep, ".pyc", ".so", ".sl", ".pyd"]:
if exists(base_path + ext):
return True
return False
func = "'(' func=any ')'"
PATTERN = """
power< module='operator'
- trailer< '.' {methods} > trailer< {func} > >
+ trailer< '.' %(methods)s > trailer< %(func)s > >
|
- power< {methods} trailer< {func} > >
- """.format(methods=methods, func=func)
+ power< %(methods)s trailer< %(func)s > >
+ """ % dict(methods=methods, func=func)
def transform(self, node, results):
method = results["method"][0]
used in that module.
"""
-from .. import pytree
-from .. import fixer_base
-from ..fixer_util import Name, Attr, touch_import
+from lib2to3 import fixer_base
+from lib2to3.fixer_util import touch_import
class FixSysExc(fixer_base.BaseFix):
# This order matches the ordering of sys.exc_info().
- exc_info = ["exc_type", "exc_value", "exc_traceback"]
+ exc_info = [u"exc_type", u"exc_value", u"exc_traceback"]
PATTERN = """
power< 'sys' trailer< dot='.' attribute=(%s) > >
""" % '|'.join("'%s'" % e for e in exc_info)
def transform(self, node, results):
sys_attr = results["attribute"][0]
- index = Number(self.exc_info.index(sys_attr.value))
+ index = Number(unicode(self.exc_info.index(sys_attr.value)))
call = Call(Name(u"exc_info"), prefix=sys_attr.prefix)
attr = Attr(Name(u"sys"), call)
end = Newline()
else:
start = 0
- indent = "; "
+ indent = u"; "
end = pytree.Leaf(token.INDENT, u"")
# We need access to self for new_name(), and making this a method
if d is None:
d = {}
for i, obj in enumerate(param_list):
- trailer = [Subscript(Number(i))]
+ trailer = [Subscript(Number(unicode(i)))]
if isinstance(obj, list):
map_to_index(obj, trailer, d=d)
else:
rest=any* >
"""
+ def start_tree(self, tree, filename):
+ super(FixXrange, self).start_tree(tree, filename)
+ self.transformed_xranges = set()
+
+ def finish_tree(self, tree, filename):
+ self.transformed_xranges = None
+
def transform(self, node, results):
name = results["name"]
if name.value == u"xrange":
def transform_xrange(self, node, results):
name = results["name"]
name.replace(Name(u"range", prefix=name.prefix))
+ # This prevents the new range call from being wrapped in a list later.
+ self.transformed_xranges.add(id(node))
def transform_range(self, node, results):
- if not self.in_special_context(node):
+ if (id(node) not in self.transformed_xranges and
+ not self.in_special_context(node)):
range_call = Call(Name(u"range"), [results["args"].clone()])
# Encase the range call in list().
list_call = Call(Name(u"list"), [range_call],
Main program for 2to3.
"""
+from __future__ import with_statement
+
import sys
import os
import difflib
if self.show_diffs:
diff_lines = diff_texts(old, new, filename)
try:
- for line in diff_lines:
- print line
+ if self.output_lock is not None:
+ with self.output_lock:
+ for line in diff_lines:
+ print line
+ sys.stdout.flush()
+ else:
+ for line in diff_lines:
+ print line
except UnicodeEncodeError:
warn("couldn't encode %s's diff for your terminal" %
(filename,))
parser.add_option("-x", "--nofix", action="append", default=[],
help="Prevent a fixer from being run.")
parser.add_option("-l", "--list-fixes", action="store_true",
- help="List available transformations (fixes/fix_*.py)")
+ help="List available transformations")
parser.add_option("-p", "--print-function", action="store_true",
help="Modify the grammar so that print() is a function")
parser.add_option("-v", "--verbose", action="store_true",
tokens = tokenize_wrapper(input)
try:
root = self.driver.parse_tokens(tokens, debug=debug)
- except parse.ParseError as e:
+ except parse.ParseError, e:
raise PatternSyntaxError(str(e))
return self.compile_node(root)
"generate_tokens", "untokenize"]
del token
+try:
+ bytes
+except NameError:
+ # Support bytes type in Python <= 2.5, so 2to3 turns itself into
+ # valid Python 3 code.
+ bytes = str
+
def group(*choices): return '(' + '|'.join(choices) + ')'
def any(*choices): return group(*choices) + '*'
def maybe(*choices): return group(*choices) + '?'
try:
return readline()
except StopIteration:
- return b''
+ return bytes()
def find_cookie(line):
try:
for node in child.post_order():
yield node
- @property
- def prefix(self):
+ def _prefix_getter(self):
"""
The whitespace and comments preceding this node in the input.
"""
return ""
return self.children[0].prefix
- @prefix.setter
- def prefix(self, prefix):
+ def _prefix_setter(self, prefix):
if self.children:
self.children[0].prefix = prefix
+ prefix = property(_prefix_getter, _prefix_setter)
+
def set_child(self, i, child):
"""
Equivalent to 'node.children[i] = child'. This method also sets the
"""Return a pre-order iterator for the tree."""
yield self
- @property
- def prefix(self):
+ def _prefix_getter(self):
"""
The whitespace and comments preceding this token in the input.
"""
return self._prefix
- @prefix.setter
- def prefix(self, prefix):
+ def _prefix_setter(self, prefix):
self.changed()
self._prefix = prefix
+ prefix = property(_prefix_getter, _prefix_setter)
def convert(gr, raw_node):
"""
provides infrastructure to write your own refactoring tool.
"""
+from __future__ import with_statement
+
__author__ = "Guido van Rossum <guido@python.org>"
_to_system_newlines = _identity
-def _detect_future_print(source):
+def _detect_future_features(source):
have_docstring = False
gen = tokenize.generate_tokens(StringIO.StringIO(source).readline)
def advance():
- tok = next(gen)
+ tok = gen.next()
return tok[0], tok[1]
ignore = frozenset((token.NEWLINE, tokenize.NL, token.COMMENT))
+ features = set()
try:
while True:
tp, value = advance()
have_docstring = True
elif tp == token.NAME and value == u"from":
tp, value = advance()
- if tp != token.NAME and value != u"__future__":
+ if tp != token.NAME or value != u"__future__":
break
tp, value = advance()
- if tp != token.NAME and value != u"import":
+ if tp != token.NAME or value != u"import":
break
tp, value = advance()
if tp == token.OP and value == u"(":
tp, value = advance()
while tp == token.NAME:
- if value == u"print_function":
- return True
+ features.add(value)
tp, value = advance()
- if tp != token.OP and value != u",":
+ if tp != token.OP or value != u",":
break
tp, value = advance()
else:
break
except StopIteration:
pass
- return False
+ return frozenset(features)
class FixerError(Exception):
An AST corresponding to the refactored input stream; None if
there were errors during the parse.
"""
- if _detect_future_print(data):
+ features = _detect_future_features(data)
+ if "print_function" in features:
self.driver.grammar = pygram.python_grammar_no_print_statement
try:
tree = self.driver.parse_string(data)
return
finally:
self.driver.grammar = self.grammar
+ tree.future_features = features
self.log_debug("Refactoring %s", name)
self.refactor_tree(tree, name)
return tree
def __init__(self, *args, **kwargs):
super(MultiprocessRefactoringTool, self).__init__(*args, **kwargs)
self.queue = None
+ self.output_lock = None
def refactor(self, items, write=False, doctests_only=False,
num_processes=1):
if self.queue is not None:
raise RuntimeError("already doing multiple processes")
self.queue = multiprocessing.JoinableQueue()
+ self.output_lock = multiprocessing.Lock()
processes = [multiprocessing.Process(target=self._child)
for i in xrange(num_processes)]
try:
for call in fixer_util.consuming_calls:
self.unchanged("a = %s(range(10))" % call)
+class Test_xrange_with_reduce(FixerTestCase):
+
+ def setUp(self):
+ super(Test_xrange_with_reduce, self).setUp(["xrange", "reduce"])
+
+ def test_double_transform(self):
+ b = """reduce(x, xrange(5))"""
+ a = """from functools import reduce
+reduce(x, range(5))"""
+ self.check(b, a)
+
class Test_raw_input(FixerTestCase):
fixer = "raw_input"
self.files_checked.append(name)
return self.always_exists or (name in self.present_files)
- from ..fixes import fix_import
+ from lib2to3.fixes import fix_import
fix_import.exists = fake_exists
def tearDown(self):
self.present_files = set(["bar.py"])
self.unchanged(s)
+ def test_with_absolute_import_enabled(self):
+ s = "from __future__ import absolute_import\nimport bar"
+ self.always_exists = False
+ self.present_files = set(["__init__.py", "bar.py"])
+ self.unchanged(s)
+
def test_in_package(self):
b = "import bar"
a = "from . import bar"
self.present_files = set(["__init__.py", "bar" + os.path.sep])
self.check(b, a)
+ def test_already_relative_import(self):
+ s = "from . import bar"
+ self.unchanged(s)
+
def test_comments_and_indent(self):
b = "import bar # Foo"
a = "from . import bar # Foo"
test_grammar.py files from both Python 2 and Python 3.
"""
+from __future__ import with_statement
+
# Testing imports
from . import support
from .support import driver, test_dir
# Python imports
import os
-import io
import sys
# Local imports
encoding = tokenize.detect_encoding(fp.readline)[0]
self.assertTrue(encoding is not None,
"can't detect encoding for %s" % filepath)
- with io.open(filepath, "r", encoding=encoding) as fp:
+ with open(filepath, "r") as fp:
source = fp.read()
+ source = source.decode(encoding)
tree = driver.parse_string(source)
new = unicode(tree)
if diff(filepath, new, encoding):
def diff(fn, result, encoding):
- f = io.open("@", "w", encoding=encoding)
+ f = open("@", "w")
try:
- f.write(result)
+ f.write(result.encode(encoding))
finally:
f.close()
try:
especially when debugging a test.
"""
+from __future__ import with_statement
+
+import sys
import warnings
# Testing imports
"""Unit tests for nodes (Base, Leaf, Node)."""
- def test_deprecated_prefix_methods(self):
- l = pytree.Leaf(100, "foo")
- with warnings.catch_warnings(record=True) as w:
- warnings.simplefilter("always", DeprecationWarning)
- self.assertEqual(l.get_prefix(), "")
- l.set_prefix("hi")
- self.assertEqual(l.prefix, "hi")
- self.assertEqual(len(w), 2)
- for warning in w:
- self.assertTrue(warning.category is DeprecationWarning)
- self.assertEqual(str(w[0].message), "get_prefix() is deprecated; " \
- "use the prefix property")
- self.assertEqual(str(w[1].message), "set_prefix() is deprecated; " \
- "use the prefix property")
+ if sys.version_info >= (2,6):
+ # warnings.catch_warnings is new in 2.6.
+ def test_deprecated_prefix_methods(self):
+ l = pytree.Leaf(100, "foo")
+ with warnings.catch_warnings(record=True) as w:
+ warnings.simplefilter("always", DeprecationWarning)
+ self.assertEqual(l.get_prefix(), "")
+ l.set_prefix("hi")
+ self.assertEqual(l.prefix, "hi")
+ self.assertEqual(len(w), 2)
+ for warning in w:
+ self.assertTrue(warning.category is DeprecationWarning)
+ self.assertEqual(str(w[0].message), "get_prefix() is deprecated; " \
+ "use the prefix property")
+ self.assertEqual(str(w[1].message), "set_prefix() is deprecated; " \
+ "use the prefix property")
def test_instantiate_base(self):
if __debug__:
Unit tests for refactor.py.
"""
+from __future__ import with_statement
+
import sys
import os
import codecs
self.assertEqual(full_names,
["myfixes.fix_" + name for name in contents])
- def test_detect_future_print(self):
- run = refactor._detect_future_print
- self.assertFalse(run(""))
- self.assertTrue(run("from __future__ import print_function"))
- self.assertFalse(run("from __future__ import generators"))
- self.assertFalse(run("from __future__ import generators, feature"))
- input = "from __future__ import generators, print_function"
- self.assertTrue(run(input))
- input ="from __future__ import print_function, generators"
- self.assertTrue(run(input))
- input = "from __future__ import (print_function,)"
- self.assertTrue(run(input))
- input = "from __future__ import (generators, print_function)"
- self.assertTrue(run(input))
- input = "from __future__ import (generators, nested_scopes)"
- self.assertFalse(run(input))
- input = """from __future__ import generators
+ def test_detect_future_features(self):
+ run = refactor._detect_future_features
+ fs = frozenset
+ empty = fs()
+ self.assertEqual(run(""), empty)
+ self.assertEqual(run("from __future__ import print_function"),
+ fs(("print_function",)))
+ self.assertEqual(run("from __future__ import generators"),
+ fs(("generators",)))
+ self.assertEqual(run("from __future__ import generators, feature"),
+ fs(("generators", "feature")))
+ inp = "from __future__ import generators, print_function"
+ self.assertEqual(run(inp), fs(("generators", "print_function")))
+ inp ="from __future__ import print_function, generators"
+ self.assertEqual(run(inp), fs(("print_function", "generators")))
+ inp = "from __future__ import (print_function,)"
+ self.assertEqual(run(inp), fs(("print_function",)))
+ inp = "from __future__ import (generators, print_function)"
+ self.assertEqual(run(inp), fs(("generators", "print_function")))
+ inp = "from __future__ import (generators, nested_scopes)"
+ self.assertEqual(run(inp), fs(("generators", "nested_scopes")))
+ inp = """from __future__ import generators
from __future__ import print_function"""
- self.assertTrue(run(input))
- self.assertFalse(run("from"))
- self.assertFalse(run("from 4"))
- self.assertFalse(run("from x"))
- self.assertFalse(run("from x 5"))
- self.assertFalse(run("from x im"))
- self.assertFalse(run("from x import"))
- self.assertFalse(run("from x import 4"))
- input = "'docstring'\nfrom __future__ import print_function"
- self.assertTrue(run(input))
- input = "'docstring'\n'somng'\nfrom __future__ import print_function"
- self.assertFalse(run(input))
- input = "# comment\nfrom __future__ import print_function"
- self.assertTrue(run(input))
- input = "# comment\n'doc'\nfrom __future__ import print_function"
- self.assertTrue(run(input))
- input = "class x: pass\nfrom __future__ import print_function"
- self.assertFalse(run(input))
+ self.assertEqual(run(inp), fs(("generators", "print_function")))
+ invalid = ("from",
+ "from 4",
+ "from x",
+ "from x 5",
+ "from x im",
+ "from x import",
+ "from x import 4",
+ )
+ for inp in invalid:
+ self.assertEqual(run(inp), empty)
+ inp = "'docstring'\nfrom __future__ import print_function"
+ self.assertEqual(run(inp), fs(("print_function",)))
+ inp = "'docstring'\n'somng'\nfrom __future__ import print_function"
+ self.assertEqual(run(inp), empty)
+ inp = "# comment\nfrom __future__ import print_function"
+ self.assertEqual(run(inp), fs(("print_function",)))
+ inp = "# comment\n'doc'\nfrom __future__ import print_function"
+ self.assertEqual(run(inp), fs(("print_function",)))
+ inp = "class x: pass\nfrom __future__ import print_function"
+ self.assertEqual(run(inp), empty)
def test_get_headnode_dict(self):
class NoneFix(fixer_base.BaseFix):
projects / python / commitdiff