bpo-36786: Run compileall in parallel during "make install" (GH-13078)

diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst
index 5e08616e9347b0cb24ba7242e74288da12353521..bb5000a0736cccb2eccbaea2f0f8b4cf693009db 100644 (file)
--- a/Doc/library/compileall.rst
+++ b/Doc/library/compileall.rst
@@ -158,7 +158,8 @@ Public functions
    The argument *workers* specifies how many workers are used to
    compile files in parallel. The default is to not use multiple workers.
    If the platform can't use multiple workers and *workers* argument is given,
-   then sequential compilation will be used as a fallback.  If *workers* is
+   then sequential compilation will be used as a fallback.  If *workers*
+   is 0, the number of cores in the system is used.  If *workers* is
    lower than ``0``, a :exc:`ValueError` will be raised.
 
    *invalidation_mode* should be a member of the
@@ -184,6 +185,9 @@ Public functions
    .. versionchanged:: 3.7
       The *invalidation_mode* parameter was added.
 
+   .. versionchanged:: 3.8
+      Setting *workers* to 0 now chooses the optimal number of cores.
+
 .. function:: compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, invalidation_mode=py_compile.PycInvalidationMode.TIMESTAMP)
 
    Compile the file with path *fullname*. Return a true value if the file

diff --git a/Lib/compileall.py b/Lib/compileall.py
index aa65c6b904e79adc750e44bf56fd871d4b4dd617..49306d9dabbc51437d2e5d066633a7c4795d2a42 100644 (file)
--- a/Lib/compileall.py
+++ b/Lib/compileall.py
@@ -67,20 +67,20 @@ def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None,
     invalidation_mode: how the up-to-dateness of the pyc will be checked
     """
     ProcessPoolExecutor = None
-    if workers is not None:
-        if workers < 0:
-            raise ValueError('workers must be greater or equal to 0')
-        elif workers != 1:
-            try:
-                # Only import when needed, as low resource platforms may
-                # fail to import it
-                from concurrent.futures import ProcessPoolExecutor
-            except ImportError:
-                workers = 1
+    if workers < 0:
+        raise ValueError('workers must be greater or equal to 0')
+    if workers != 1:
+        try:
+            # Only import when needed, as low resource platforms may
+            # fail to import it
+            from concurrent.futures import ProcessPoolExecutor
+        except ImportError:
+            workers = 1
     files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels,
                       ddir=ddir)
     success = True
-    if workers is not None and workers != 1 and ProcessPoolExecutor is not None:
+    if workers != 1 and ProcessPoolExecutor is not None:
+        # If workers == 0, let ProcessPoolExecutor choose
         workers = workers or None
         with ProcessPoolExecutor(max_workers=workers) as executor:
             results = executor.map(partial(compile_file,
@@ -290,9 +290,6 @@ def main():
                 print("Error reading file list {}".format(args.flist))
             return False
 
-    if args.workers is not None:
-        args.workers = args.workers or None
-
     if args.invalidation_mode:
         ivl_mode = args.invalidation_mode.replace('-', '_').upper()
         invalidation_mode = py_compile.PycInvalidationMode[ivl_mode]

diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py
index 8e584602de36138dd1fbbd7b5378f3dc9ea0808d..04f6e1e049dde1a446014400068945862688d8aa 100644 (file)
--- a/Lib/test/test_compileall.py
+++ b/Lib/test/test_compileall.py
@@ -575,7 +575,7 @@ class CommandLineTestsBase:
                         new=[sys.executable, self.directory, "-j0"]):
             compileall.main()
             self.assertTrue(compile_dir.called)
-            self.assertEqual(compile_dir.call_args[-1]['workers'], None)
+            self.assertEqual(compile_dir.call_args[-1]['workers'], 0)
 
 
 class CommmandLineTestsWithSourceEpoch(CommandLineTestsBase,

diff --git a/Makefile.pre.in b/Makefile.pre.in
index 75eb66be3c01354e1c73e9ed9c3afd2e18ce96da..4924dedc357c722cd155fd54577939c34d1c4486 100644 (file)
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -1432,30 +1432,30 @@ libinstall:     build_all $(srcdir)/Modules/xxmodule.c
        fi
        -PYTHONPATH=$(DESTDIR)$(LIBDEST)  $(RUNSHARED) \
                $(PYTHON_FOR_BUILD) -Wi $(DESTDIR)$(LIBDEST)/compileall.py \
-               -d $(LIBDEST) -f \
+               -j0 -d $(LIBDEST) -f \
                -x 'bad_coding|badsyntax|site-packages|lib2to3/tests/data' \
                $(DESTDIR)$(LIBDEST)
        -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \
                $(PYTHON_FOR_BUILD) -Wi -O $(DESTDIR)$(LIBDEST)/compileall.py \
-               -d $(LIBDEST) -f \
+               -j0 -d $(LIBDEST) -f \
                -x 'bad_coding|badsyntax|site-packages|lib2to3/tests/data' \
                $(DESTDIR)$(LIBDEST)
        -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \
                $(PYTHON_FOR_BUILD) -Wi -OO $(DESTDIR)$(LIBDEST)/compileall.py \
-               -d $(LIBDEST) -f \
+               -j0 -d $(LIBDEST) -f \
                -x 'bad_coding|badsyntax|site-packages|lib2to3/tests/data' \
                $(DESTDIR)$(LIBDEST)
        -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \
                $(PYTHON_FOR_BUILD) -Wi $(DESTDIR)$(LIBDEST)/compileall.py \
-               -d $(LIBDEST)/site-packages -f \
+               -j0 -d $(LIBDEST)/site-packages -f \
                -x badsyntax $(DESTDIR)$(LIBDEST)/site-packages
        -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \
                $(PYTHON_FOR_BUILD) -Wi -O $(DESTDIR)$(LIBDEST)/compileall.py \
-               -d $(LIBDEST)/site-packages -f \
+               -j0 -d $(LIBDEST)/site-packages -f \
                -x badsyntax $(DESTDIR)$(LIBDEST)/site-packages
        -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \
                $(PYTHON_FOR_BUILD) -Wi -OO $(DESTDIR)$(LIBDEST)/compileall.py \
-               -d $(LIBDEST)/site-packages -f \
+               -j0 -d $(LIBDEST)/site-packages -f \
                -x badsyntax $(DESTDIR)$(LIBDEST)/site-packages
        -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \
                $(PYTHON_FOR_BUILD) -m lib2to3.pgen2.driver $(DESTDIR)$(LIBDEST)/lib2to3/Grammar.txt

diff --git a/Misc/NEWS.d/next/Build/2019-05-03-21-08-06.bpo-36786.gOLFbD.rst b/Misc/NEWS.d/next/Build/2019-05-03-21-08-06.bpo-36786.gOLFbD.rst
new file mode 100644 (file)
index 0000000..0fcda43
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2019-05-03-21-08-06.bpo-36786.gOLFbD.rst
@@ -0,0 +1 @@
+"make install" now runs compileall in parallel.