modules (currently limited to C extensions, should accommodate C++
extensions ASAP)."""
-import sys, os, re
+import contextlib
+import os
+import re
+import sys
from distutils.core import Command
from distutils.errors import *
from distutils.sysconfig import customize_compiler, get_python_version
"forcibly build everything (ignore file timestamps)"),
('compiler=', 'c',
"specify the compiler type"),
+ ('parallel=', 'j',
+ "number of parallel build jobs"),
('swig-cpp', None,
"make SWIG create C++ files (default is C)"),
('swig-opts=', None,
self.swig_cpp = None
self.swig_opts = None
self.user = None
+ self.parallel = None
def finalize_options(self):
from distutils import sysconfig
('compiler', 'compiler'),
('debug', 'debug'),
('force', 'force'),
+ ('parallel', 'parallel'),
('plat_name', 'plat_name'),
)
self.library_dirs.append(user_lib)
self.rpath.append(user_lib)
+ if isinstance(self.parallel, str):
+ try:
+ self.parallel = int(self.parallel)
+ except ValueError:
+ raise DistutilsOptionError("parallel should be an integer")
+
def run(self):
from distutils.ccompiler import new_compiler
def build_extensions(self):
# First, sanity-check the 'extensions' list
self.check_extensions_list(self.extensions)
+ if self.parallel:
+ self._build_extensions_parallel()
+ else:
+ self._build_extensions_serial()
+
+ def _build_extensions_parallel(self):
+ workers = self.parallel
+ if self.parallel is True:
+ workers = os.cpu_count() # may return None
+ try:
+ from concurrent.futures import ThreadPoolExecutor
+ except ImportError:
+ workers = None
+
+ if workers is None:
+ self._build_extensions_serial()
+ return
+ with ThreadPoolExecutor(max_workers=workers) as executor:
+ futures = [executor.submit(self.build_extension, ext)
+ for ext in self.extensions]
+ for ext, fut in zip(self.extensions, futures):
+ with self._filter_build_errors(ext):
+ fut.result()
+
+ def _build_extensions_serial(self):
for ext in self.extensions:
- try:
+ with self._filter_build_errors(ext):
self.build_extension(ext)
- except (CCompilerError, DistutilsError, CompileError) as e:
- if not ext.optional:
- raise
- self.warn('building extension "%s" failed: %s' %
- (ext.name, e))
+
+ @contextlib.contextmanager
+ def _filter_build_errors(self, ext):
+ try:
+ yield
+ except (CCompilerError, DistutilsError, CompileError) as e:
+ if not ext.optional:
+ raise
+ self.warn('building extension "%s" failed: %s' %
+ (ext.name, e))
def build_extension(self, ext):
sources = ext.sources
from distutils.command import build_ext
build_ext.USER_BASE = site.USER_BASE
+ def build_ext(self, *args, **kwargs):
+ return build_ext(*args, **kwargs)
+
def test_build_ext(self):
global ALREADY_TESTED
copy_xxmodule_c(self.tmp_dir)
xx_ext = Extension('xx', [xx_c])
dist = Distribution({'name': 'xx', 'ext_modules': [xx_ext]})
dist.package_dir = self.tmp_dir
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
fixup_build_ext(cmd)
cmd.build_lib = self.tmp_dir
cmd.build_temp = self.tmp_dir
def test_solaris_enable_shared(self):
dist = Distribution({'name': 'xx'})
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
old = sys.platform
sys.platform = 'sunos' # fooling finalize_options
def test_user_site(self):
import site
dist = Distribution({'name': 'xx'})
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
# making sure the user option is there
options = [name for name, short, lable in
# with the optional argument.
modules = [Extension('foo', ['xxx'], optional=False)]
dist = Distribution({'name': 'xx', 'ext_modules': modules})
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.ensure_finalized()
self.assertRaises((UnknownFileError, CompileError),
cmd.run) # should raise an error
modules = [Extension('foo', ['xxx'], optional=True)]
dist = Distribution({'name': 'xx', 'ext_modules': modules})
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.ensure_finalized()
cmd.run() # should pass
# etc.) are in the include search path.
modules = [Extension('foo', ['xxx'], optional=False)]
dist = Distribution({'name': 'xx', 'ext_modules': modules})
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.finalize_options()
from distutils import sysconfig
# make sure cmd.libraries is turned into a list
# if it's a string
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.libraries = 'my_lib, other_lib lastlib'
cmd.finalize_options()
self.assertEqual(cmd.libraries, ['my_lib', 'other_lib', 'lastlib'])
# make sure cmd.library_dirs is turned into a list
# if it's a string
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.library_dirs = 'my_lib_dir%sother_lib_dir' % os.pathsep
cmd.finalize_options()
self.assertIn('my_lib_dir', cmd.library_dirs)
# make sure rpath is turned into a list
# if it's a string
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.rpath = 'one%stwo' % os.pathsep
cmd.finalize_options()
self.assertEqual(cmd.rpath, ['one', 'two'])
# make sure define is turned into 2-tuples
# strings if they are ','-separated strings
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.define = 'one,two'
cmd.finalize_options()
self.assertEqual(cmd.define, [('one', '1'), ('two', '1')])
# make sure undef is turned into a list of
# strings if they are ','-separated strings
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.undef = 'one,two'
cmd.finalize_options()
self.assertEqual(cmd.undef, ['one', 'two'])
# make sure swig_opts is turned into a list
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.swig_opts = None
cmd.finalize_options()
self.assertEqual(cmd.swig_opts, [])
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.swig_opts = '1 2'
cmd.finalize_options()
self.assertEqual(cmd.swig_opts, ['1', '2'])
def test_check_extensions_list(self):
dist = Distribution()
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.finalize_options()
#'extensions' option must be a list of Extension instances
def test_get_source_files(self):
modules = [Extension('foo', ['xxx'], optional=False)]
dist = Distribution({'name': 'xx', 'ext_modules': modules})
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.ensure_finalized()
self.assertEqual(cmd.get_source_files(), ['xxx'])
# should not be overriden by a compiler instance
# when the command is run
dist = Distribution()
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.compiler = 'unix'
cmd.ensure_finalized()
cmd.run()
ext = Extension('foo', [c_file], optional=False)
dist = Distribution({'name': 'xx',
'ext_modules': [ext]})
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
fixup_build_ext(cmd)
cmd.ensure_finalized()
self.assertEqual(len(cmd.get_outputs()), 1)
#etree_ext = Extension('lxml.etree', [etree_c])
#dist = Distribution({'name': 'lxml', 'ext_modules': [etree_ext]})
dist = Distribution()
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.inplace = 1
cmd.distribution.package_dir = {'': 'src'}
cmd.distribution.packages = ['lxml', 'lxml.html']
'ext_modules': [deptarget_ext]
})
dist.package_dir = self.tmp_dir
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.build_lib = self.tmp_dir
cmd.build_temp = self.tmp_dir
self.fail("Wrong deployment target during compilation")
+class ParallelBuildExtTestCase(BuildExtTestCase):
+
+ def build_ext(self, *args, **kwargs):
+ build_ext = super().build_ext(*args, **kwargs)
+ build_ext.parallel = True
+ return build_ext
+
+
def test_suite():
- return unittest.makeSuite(BuildExtTestCase)
+ suite = unittest.TestSuite()
+ suite.addTest(unittest.makeSuite(BuildExtTestCase))
+ suite.addTest(unittest.makeSuite(ParallelBuildExtTestCase))
+ return suite
if __name__ == '__main__':
- support.run_unittest(test_suite())
+ support.run_unittest(__name__)
py_cflags_nodist = sysconfig.get_config_var('PY_CFLAGS_NODIST')
sysconfig.get_config_vars()['CFLAGS'] = cflags + ' ' + py_cflags_nodist
+class Dummy:
+ """Hack for parallel build"""
+ ProcessPoolExecutor = None
+sys.modules['concurrent.futures.process'] = Dummy
+
def get_platform():
# cross build
if "_PYTHON_HOST_PLATFORM" in os.environ:
build_ext.__init__(self, dist)
self.failed = []
self.failed_on_import = []
+ if '-j' in os.environ.get('MAKEFLAGS', ''):
+ self.parallel = True
def build_extensions(self):
build_ext.build_extensions(self)
+ for ext in self.extensions:
+ self.check_extension_import(ext)
+
longest = max([len(e.name) for e in self.extensions])
if self.failed or self.failed_on_import:
all_failed = self.failed + self.failed_on_import
(ext.name, sys.exc_info()[1]))
self.failed.append(ext.name)
return
+
+ def check_extension_import(self, ext):
+ # Don't try to import an extension that has failed to compile
+ if ext.name in self.failed:
+ self.announce(
+ 'WARNING: skipping import check for failed build "%s"' %
+ ext.name, level=1)
+ return
+
# Workaround for Mac OS X: The Carbon-based modules cannot be
# reliably imported into a command-line Python
if 'Carbon' in ext.extra_link_args:
projects / python / commitdiff