:program:`python -m compileall <directory> -r 0` is equivalent to
:program:`python -m compileall <directory> -l`.
+.. cmdoption:: -j N
+
+ Use *N* workers to compile the files within the given directory.
+ If ``0`` is used, then the result of :func:`os.cpu_count()`
+ will be used.
.. versionchanged:: 3.2
Added the ``-i``, ``-b`` and ``-h`` options.
.. versionchanged:: 3.5
- Added the ``-r`` option.
+ Added the ``-j`` and ``-r`` options.
+
There is no command-line option to control the optimization level used by the
:func:`compile` function, because the Python interpreter itself already
Public functions
----------------
-.. function:: compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, quiet=False, legacy=False, optimize=-1)
+.. function:: compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, quiet=False, legacy=False, optimize=-1, workers=1)
Recursively descend the directory tree named by *dir*, compiling all :file:`.py`
files along the way.
*optimize* specifies the optimization level for the compiler. It is passed to
the built-in :func:`compile` function.
+ The argument *workers* specifies how many workers are used to
+ compile files in parallel. The default is to not use multiple workers.
+ If the platform can't use multiple workers and *workers* argument is given,
+ then a :exc:`NotImplementedError` will be raised.
+ If *workers* is lower than ``0``, a :exc:`ValueError` will be raised.
+
.. versionchanged:: 3.2
Added the *legacy* and *optimize* parameter.
+ .. versionchanged:: 3.5
+ Added the *workers* parameter.
+
.. function:: compile_file(fullname, ddir=None, force=False, rx=None, quiet=False, legacy=False, optimize=-1)
Improved Modules
================
+compileall
+----------
+
+* :func:`compileall.compile_dir` and :mod:`compileall`'s command-line interface
+ can now do parallel bytecode compilation.
+ (Contributed by Claudiu Popa in :issue:`16104`).
+
doctest
-------
import py_compile
import struct
-__all__ = ["compile_dir","compile_file","compile_path"]
+try:
+ from concurrent.futures import ProcessPoolExecutor
+except ImportError:
+ ProcessPoolExecutor = None
+from functools import partial
-def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None,
- quiet=False, legacy=False, optimize=-1):
- """Byte-compile all modules in the given directory tree.
-
- Arguments (only dir is required):
+__all__ = ["compile_dir","compile_file","compile_path"]
- dir: the directory to byte-compile
- maxlevels: maximum recursion level (default 10)
- ddir: the directory that will be prepended to the path to the
- file as it is compiled into each byte-code file.
- force: if True, force compilation, even if timestamps are up-to-date
- quiet: if True, be quiet during compilation
- legacy: if True, produce legacy pyc paths instead of PEP 3147 paths
- optimize: optimization level or -1 for level of the interpreter
- """
+def _walk_dir(dir, ddir=None, maxlevels=10, quiet=False):
if not quiet:
print('Listing {!r}...'.format(dir))
try:
print("Can't list {!r}".format(dir))
names = []
names.sort()
- success = 1
for name in names:
if name == '__pycache__':
continue
else:
dfile = None
if not os.path.isdir(fullname):
- if not compile_file(fullname, ddir, force, rx, quiet,
- legacy, optimize):
- success = 0
+ yield fullname
elif (maxlevels > 0 and name != os.curdir and name != os.pardir and
os.path.isdir(fullname) and not os.path.islink(fullname)):
- if not compile_dir(fullname, maxlevels - 1, dfile, force, rx,
- quiet, legacy, optimize):
+ yield from _walk_dir(fullname, ddir=dfile,
+ maxlevels=maxlevels - 1, quiet=quiet)
+
+def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None,
+ quiet=False, legacy=False, optimize=-1, workers=1):
+ """Byte-compile all modules in the given directory tree.
+
+ Arguments (only dir is required):
+
+ dir: the directory to byte-compile
+ maxlevels: maximum recursion level (default 10)
+ ddir: the directory that will be prepended to the path to the
+ file as it is compiled into each byte-code file.
+ force: if True, force compilation, even if timestamps are up-to-date
+ quiet: if True, be quiet during compilation
+ legacy: if True, produce legacy pyc paths instead of PEP 3147 paths
+ optimize: optimization level or -1 for level of the interpreter
+ workers: maximum number of parallel workers
+ """
+ files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels,
+ ddir=ddir)
+ success = 1
+ if workers is not None and workers != 1:
+ if workers < 0:
+ raise ValueError('workers must be greater or equal to 0')
+ if ProcessPoolExecutor is None:
+ raise NotImplementedError('multiprocessing support not available')
+
+ workers = workers or None
+ with ProcessPoolExecutor(max_workers=workers) as executor:
+ results = executor.map(partial(compile_file,
+ ddir=ddir, force=force,
+ rx=rx, quiet=quiet,
+ legacy=legacy,
+ optimize=optimize),
+ files)
+ success = min(results, default=1)
+ else:
+ for file in files:
+ if not compile_file(file, ddir, force, rx, quiet,
+ legacy, optimize):
success = 0
return success
help=('zero or more file and directory names '
'to compile; if no arguments given, defaults '
'to the equivalent of -l sys.path'))
- args = parser.parse_args()
+ parser.add_argument('-j', '--workers', default=1,
+ type=int, help='Run compileall concurrently')
+ args = parser.parse_args()
compile_dests = args.compile_dest
if (args.ddir and (len(compile_dests) != 1
print("Error reading file list {}".format(args.flist))
return False
+ if args.workers is not None:
+ args.workers = args.workers or None
+
success = True
try:
if compile_dests:
else:
if not compile_dir(dest, maxlevels, args.ddir,
args.force, args.rx, args.quiet,
- args.legacy):
+ args.legacy, workers=args.workers):
success = False
return success
else:
import unittest
import io
+from unittest import mock, skipUnless
+try:
+ from concurrent.futures import ProcessPoolExecutor
+ _have_multiprocessing = True
+except ImportError:
+ _have_multiprocessing = False
+
from test import support, script_helper
class CompileallTests(unittest.TestCase):
debug_override=not optimize)
self.assertTrue(os.path.isfile(cached3))
+ @mock.patch('compileall.ProcessPoolExecutor')
+ def test_compile_pool_called(self, pool_mock):
+ compileall.compile_dir(self.directory, quiet=True, workers=5)
+ self.assertTrue(pool_mock.called)
+
+ def test_compile_workers_non_positive(self):
+ with self.assertRaisesRegex(ValueError,
+ "workers must be greater or equal to 0"):
+ compileall.compile_dir(self.directory, workers=-1)
+
+ @mock.patch('compileall.ProcessPoolExecutor')
+ def test_compile_workers_cpu_count(self, pool_mock):
+ compileall.compile_dir(self.directory, quiet=True, workers=0)
+ self.assertEqual(pool_mock.call_args[1]['max_workers'], None)
+
+ @mock.patch('compileall.ProcessPoolExecutor')
+ @mock.patch('compileall.compile_file')
+ def test_compile_one_worker(self, compile_file_mock, pool_mock):
+ compileall.compile_dir(self.directory, quiet=True)
+ self.assertFalse(pool_mock.called)
+ self.assertTrue(compile_file_mock.called)
+
+ @mock.patch('compileall.ProcessPoolExecutor', new=None)
+ def test_compile_missing_multiprocessing(self):
+ with self.assertRaisesRegex(NotImplementedError,
+ "multiprocessing support not available"):
+ compileall.compile_dir(self.directory, quiet=True, workers=5)
class EncodingTest(unittest.TestCase):
"""Issue 6716: compileall should escape source code when printing errors
out = self.assertRunOK('badfilename')
self.assertRegex(out, b"Can't list 'badfilename'")
+ @skipUnless(_have_multiprocessing, "requires multiprocessing")
+ def test_workers(self):
+ bar2fn = script_helper.make_script(self.directory, 'bar2', '')
+ files = []
+ for suffix in range(5):
+ pkgdir = os.path.join(self.directory, 'foo{}'.format(suffix))
+ os.mkdir(pkgdir)
+ fn = script_helper.make_script(pkgdir, '__init__', '')
+ files.append(script_helper.make_script(pkgdir, 'bar2', ''))
+
+ self.assertRunOK(self.directory, '-j', '0')
+ self.assertCompiled(bar2fn)
+ for file in files:
+ self.assertCompiled(file)
+
+ @mock.patch('compileall.compile_dir')
+ def test_workers_available_cores(self, compile_dir):
+ with mock.patch("sys.argv",
+ new=[sys.executable, self.directory, "-j0"]):
+ compileall.main()
+ self.assertTrue(compile_dir.called)
+ self.assertEqual(compile_dir.call_args[-1]['workers'], None)
+
if __name__ == "__main__":
unittest.main()
projects / python / commitdiff