1 """distutils.command.build_ext
3 Implements the Distutils 'build_ext' command, for building extension
4 modules (currently limited to C extensions, should accommodate C++
11 from ..core import Command
12 from ..errors import (
18 DistutilsPlatformError,
20 from ..sysconfig import customize_compiler, get_python_version
21 from ..sysconfig import get_config_h_filename
22 from ..dep_util import newer_group
23 from ..extension import Extension
24 from ..util import get_platform
25 from distutils._log import log
26 from . import py37compat
28 from site import USER_BASE
30 # An extension name is just a dot-separated list of Python NAMEs (ie.
31 # the same as a fully-qualified module name).
32 extension_name_re = re.compile(r'^[a-zA-Z_][a-zA-Z_0-9]*(\.[a-zA-Z_][a-zA-Z_0-9]*)*$')
36 from ..ccompiler import show_compilers
41 class build_ext(Command):
43 description = "build C/C++ extensions (compile/link to build directory)"
45 # XXX thoughts on how to deal with complex command-line options like
46 # these, i.e. how to make it so fancy_getopt can suck them off the
47 # command line and make it look like setup.py defined the appropriate
48 # lists of tuples of what-have-you.
49 # - each command needs a callback to process its command-line options
50 # - Command.__init__() needs access to its share of the whole
51 # command line (must ultimately come from
52 # Distribution.parse_command_line())
53 # - it then calls the current command class' option-parsing
54 # callback to deal with weird options like -D, which have to
55 # parse the option text and churn out some custom data
57 # - that data structure (in this case, a list of 2-tuples)
58 # will then be present in the command object by the time
59 # we get to finalize_options() (i.e. the constructor
60 # takes care of both command-line and client options
61 # in between initialize_options() and finalize_options())
63 sep_by = " (separated by '%s')" % os.pathsep
65 ('build-lib=', 'b', "directory for compiled extension modules"),
66 ('build-temp=', 't', "directory for temporary files (build by-products)"),
70 "platform name to cross-compile for, if supported "
71 "(default: %s)" % get_platform(),
76 "ignore build-lib and put compiled extensions into the source "
77 + "directory alongside your pure Python modules",
82 "list of directories to search for header files" + sep_by,
84 ('define=', 'D', "C preprocessor macros to define"),
85 ('undef=', 'U', "C preprocessor macros to undefine"),
86 ('libraries=', 'l', "external C libraries to link with"),
90 "directories to search for external C libraries" + sep_by,
92 ('rpath=', 'R', "directories to search for shared C libraries at runtime"),
93 ('link-objects=', 'O', "extra explicit link objects to include in the link"),
94 ('debug', 'g', "compile/link with debugging information"),
95 ('force', 'f', "forcibly build everything (ignore file timestamps)"),
96 ('compiler=', 'c', "specify the compiler type"),
97 ('parallel=', 'j', "number of parallel build jobs"),
98 ('swig-cpp', None, "make SWIG create C++ files (default is C)"),
99 ('swig-opts=', None, "list of SWIG command line options"),
100 ('swig=', None, "path to the SWIG executable"),
101 ('user', None, "add user include, library and rpath"),
104 boolean_options = ['inplace', 'debug', 'force', 'swig-cpp', 'user']
107 ('help-compiler', None, "list available compilers", show_compilers),
110 def initialize_options(self):
111 self.extensions = None
112 self.build_lib = None
113 self.plat_name = None
114 self.build_temp = None
118 self.include_dirs = None
121 self.libraries = None
122 self.library_dirs = None
124 self.link_objects = None
130 self.swig_opts = None
134 def finalize_options(self): # noqa: C901
135 from distutils import sysconfig
137 self.set_undefined_options(
139 ('build_lib', 'build_lib'),
140 ('build_temp', 'build_temp'),
141 ('compiler', 'compiler'),
144 ('parallel', 'parallel'),
145 ('plat_name', 'plat_name'),
148 if self.package is None:
149 self.package = self.distribution.ext_package
151 self.extensions = self.distribution.ext_modules
153 # Make sure Python's include directories (for Python.h, pyconfig.h,
154 # etc.) are in the include search path.
155 py_include = sysconfig.get_python_inc()
156 plat_py_include = sysconfig.get_python_inc(plat_specific=1)
157 if self.include_dirs is None:
158 self.include_dirs = self.distribution.include_dirs or []
159 if isinstance(self.include_dirs, str):
160 self.include_dirs = self.include_dirs.split(os.pathsep)
162 # If in a virtualenv, add its include directory
164 if sys.exec_prefix != sys.base_exec_prefix:
165 self.include_dirs.append(os.path.join(sys.exec_prefix, 'include'))
167 # Put the Python "system" include dir at the end, so that
168 # any local include dirs take precedence.
169 self.include_dirs.extend(py_include.split(os.path.pathsep))
170 if plat_py_include != py_include:
171 self.include_dirs.extend(plat_py_include.split(os.path.pathsep))
173 self.ensure_string_list('libraries')
174 self.ensure_string_list('link_objects')
176 # Life is easier if we're not forever checking for None, so
177 # simplify these options to empty lists if unset
178 if self.libraries is None:
180 if self.library_dirs is None:
181 self.library_dirs = []
182 elif isinstance(self.library_dirs, str):
183 self.library_dirs = self.library_dirs.split(os.pathsep)
185 if self.rpath is None:
187 elif isinstance(self.rpath, str):
188 self.rpath = self.rpath.split(os.pathsep)
190 # for extensions under windows use different directories
191 # for Release and Debug builds.
192 # also Python's library directory must be appended to library_dirs
194 # the 'libs' directory is for binary installs - we assume that
195 # must be the *native* platform. But we don't really support
196 # cross-compiling via a binary install anyway, so we let it go.
197 self.library_dirs.append(os.path.join(sys.exec_prefix, 'libs'))
198 if sys.base_exec_prefix != sys.prefix: # Issue 16116
199 self.library_dirs.append(os.path.join(sys.base_exec_prefix, 'libs'))
201 self.build_temp = os.path.join(self.build_temp, "Debug")
203 self.build_temp = os.path.join(self.build_temp, "Release")
205 # Append the source distribution include and library directories,
206 # this allows distutils on windows to work in the source tree
207 self.include_dirs.append(os.path.dirname(get_config_h_filename()))
208 self.library_dirs.append(sys.base_exec_prefix)
210 # Use the .lib files for the correct architecture
211 if self.plat_name == 'win32':
215 suffix = self.plat_name[4:]
216 new_lib = os.path.join(sys.exec_prefix, 'PCbuild')
218 new_lib = os.path.join(new_lib, suffix)
219 self.library_dirs.append(new_lib)
221 # For extensions under Cygwin, Python's library directory must be
222 # appended to library_dirs
223 if sys.platform[:6] == 'cygwin':
224 if not sysconfig.python_build:
225 # building third party extensions
226 self.library_dirs.append(
228 sys.prefix, "lib", "python" + get_python_version(), "config"
232 # building python standard extensions
233 self.library_dirs.append('.')
235 # For building extensions with a shared Python library,
236 # Python's library directory must be appended to library_dirs
237 # See Issues: #1600860, #4366
238 if sysconfig.get_config_var('Py_ENABLE_SHARED'):
239 if not sysconfig.python_build:
240 # building third party extensions
241 self.library_dirs.append(sysconfig.get_config_var('LIBDIR'))
243 # building python standard extensions
244 self.library_dirs.append('.')
246 # The argument parsing will result in self.define being a string, but
247 # it has to be a list of 2-tuples. All the preprocessor symbols
248 # specified by the 'define' option will be set to '1'. Multiple
249 # symbols can be separated with commas.
252 defines = self.define.split(',')
253 self.define = [(symbol, '1') for symbol in defines]
255 # The option for macros to undefine is also a string from the
256 # option parsing, but has to be a list. Multiple symbols can also
257 # be separated with commas here.
259 self.undef = self.undef.split(',')
261 if self.swig_opts is None:
264 self.swig_opts = self.swig_opts.split(' ')
266 # Finally add the user include and library directories if requested
268 user_include = os.path.join(USER_BASE, "include")
269 user_lib = os.path.join(USER_BASE, "lib")
270 if os.path.isdir(user_include):
271 self.include_dirs.append(user_include)
272 if os.path.isdir(user_lib):
273 self.library_dirs.append(user_lib)
274 self.rpath.append(user_lib)
276 if isinstance(self.parallel, str):
278 self.parallel = int(self.parallel)
280 raise DistutilsOptionError("parallel should be an integer")
282 def run(self): # noqa: C901
283 from ..ccompiler import new_compiler
285 # 'self.extensions', as supplied by setup.py, is a list of
286 # Extension instances. See the documentation for Extension (in
287 # distutils.extension) for details.
289 # For backwards compatibility with Distutils 0.8.2 and earlier, we
290 # also allow the 'extensions' list to be a list of tuples:
291 # (ext_name, build_info)
292 # where build_info is a dictionary containing everything that
293 # Extension instances do except the name, with a few things being
294 # differently named. We convert these 2-tuples to Extension
295 # instances as needed.
297 if not self.extensions:
300 # If we were asked to build any C/C++ libraries, make sure that the
301 # directory where we put them is in the library search path for
302 # linking extensions.
303 if self.distribution.has_c_libraries():
304 build_clib = self.get_finalized_command('build_clib')
305 self.libraries.extend(build_clib.get_library_names() or [])
306 self.library_dirs.append(build_clib.build_clib)
308 # Setup the CCompiler object that we'll use to do all the
309 # compiling and linking
310 self.compiler = new_compiler(
311 compiler=self.compiler,
312 verbose=self.verbose,
313 dry_run=self.dry_run,
316 customize_compiler(self.compiler)
317 # If we are cross-compiling, init the compiler now (if we are not
318 # cross-compiling, init would not hurt, but people may rely on
319 # late initialization of compiler even if they shouldn't...)
320 if os.name == 'nt' and self.plat_name != get_platform():
321 self.compiler.initialize(self.plat_name)
323 # And make sure that any compile/link-related options (which might
324 # come from the command-line or from the setup script) are set in
325 # that CCompiler object -- that way, they automatically apply to
326 # all compiling and linking done here.
327 if self.include_dirs is not None:
328 self.compiler.set_include_dirs(self.include_dirs)
329 if self.define is not None:
330 # 'define' option is a list of (name,value) tuples
331 for (name, value) in self.define:
332 self.compiler.define_macro(name, value)
333 if self.undef is not None:
334 for macro in self.undef:
335 self.compiler.undefine_macro(macro)
336 if self.libraries is not None:
337 self.compiler.set_libraries(self.libraries)
338 if self.library_dirs is not None:
339 self.compiler.set_library_dirs(self.library_dirs)
340 if self.rpath is not None:
341 self.compiler.set_runtime_library_dirs(self.rpath)
342 if self.link_objects is not None:
343 self.compiler.set_link_objects(self.link_objects)
345 # Now actually compile and link everything.
346 self.build_extensions()
348 def check_extensions_list(self, extensions): # noqa: C901
349 """Ensure that the list of extensions (presumably provided as a
350 command option 'extensions') is valid, i.e. it is a list of
351 Extension objects. We also support the old-style list of 2-tuples,
352 where the tuples are (ext_name, build_info), which are converted to
353 Extension instances here.
355 Raise DistutilsSetupError if the structure is invalid anywhere;
356 just returns otherwise.
358 if not isinstance(extensions, list):
359 raise DistutilsSetupError(
360 "'ext_modules' option must be a list of Extension instances"
363 for i, ext in enumerate(extensions):
364 if isinstance(ext, Extension):
365 continue # OK! (assume type-checking done
366 # by Extension constructor)
368 if not isinstance(ext, tuple) or len(ext) != 2:
369 raise DistutilsSetupError(
370 "each element of 'ext_modules' option must be an "
371 "Extension instance or 2-tuple"
374 ext_name, build_info = ext
377 "old-style (ext_name, build_info) tuple found in "
378 "ext_modules for extension '%s' "
379 "-- please convert to Extension instance",
383 if not (isinstance(ext_name, str) and extension_name_re.match(ext_name)):
384 raise DistutilsSetupError(
385 "first element of each tuple in 'ext_modules' "
386 "must be the extension name (a string)"
389 if not isinstance(build_info, dict):
390 raise DistutilsSetupError(
391 "second element of each tuple in 'ext_modules' "
392 "must be a dictionary (build info)"
395 # OK, the (ext_name, build_info) dict is type-safe: convert it
396 # to an Extension instance.
397 ext = Extension(ext_name, build_info['sources'])
399 # Easy stuff: one-to-one mapping from dict elements to
400 # instance attributes.
406 'extra_compile_args',
409 val = build_info.get(key)
411 setattr(ext, key, val)
413 # Medium-easy stuff: same syntax/semantics, different names.
414 ext.runtime_library_dirs = build_info.get('rpath')
415 if 'def_file' in build_info:
417 "'def_file' element of build info dict " "no longer supported"
420 # Non-trivial stuff: 'macros' split into 'define_macros'
421 # and 'undef_macros'.
422 macros = build_info.get('macros')
424 ext.define_macros = []
425 ext.undef_macros = []
427 if not (isinstance(macro, tuple) and len(macro) in (1, 2)):
428 raise DistutilsSetupError(
429 "'macros' element of build info dict "
430 "must be 1- or 2-tuple"
433 ext.undef_macros.append(macro[0])
434 elif len(macro) == 2:
435 ext.define_macros.append(macro)
439 def get_source_files(self):
440 self.check_extensions_list(self.extensions)
443 # Wouldn't it be neat if we knew the names of header files too...
444 for ext in self.extensions:
445 filenames.extend(ext.sources)
448 def get_outputs(self):
449 # Sanity check the 'extensions' list -- can't assume this is being
450 # done in the same run as a 'build_extensions()' call (in fact, we
451 # can probably assume that it *isn't*!).
452 self.check_extensions_list(self.extensions)
454 # And build the list of output (built) filenames. Note that this
455 # ignores the 'inplace' flag, and assumes everything goes in the
458 for ext in self.extensions:
459 outputs.append(self.get_ext_fullpath(ext.name))
462 def build_extensions(self):
463 # First, sanity-check the 'extensions' list
464 self.check_extensions_list(self.extensions)
466 self._build_extensions_parallel()
468 self._build_extensions_serial()
470 def _build_extensions_parallel(self):
471 workers = self.parallel
472 if self.parallel is True:
473 workers = os.cpu_count() # may return None
475 from concurrent.futures import ThreadPoolExecutor
480 self._build_extensions_serial()
483 with ThreadPoolExecutor(max_workers=workers) as executor:
485 executor.submit(self.build_extension, ext) for ext in self.extensions
487 for ext, fut in zip(self.extensions, futures):
488 with self._filter_build_errors(ext):
491 def _build_extensions_serial(self):
492 for ext in self.extensions:
493 with self._filter_build_errors(ext):
494 self.build_extension(ext)
496 @contextlib.contextmanager
497 def _filter_build_errors(self, ext):
500 except (CCompilerError, DistutilsError, CompileError) as e:
503 self.warn('building extension "{}" failed: {}'.format(ext.name, e))
505 def build_extension(self, ext):
506 sources = ext.sources
507 if sources is None or not isinstance(sources, (list, tuple)):
508 raise DistutilsSetupError(
509 "in 'ext_modules' option (extension '%s'), "
510 "'sources' must be present and must be "
511 "a list of source filenames" % ext.name
513 # sort to make the resulting .so file build reproducible
514 sources = sorted(sources)
516 ext_path = self.get_ext_fullpath(ext.name)
517 depends = sources + ext.depends
518 if not (self.force or newer_group(depends, ext_path, 'newer')):
519 log.debug("skipping '%s' extension (up-to-date)", ext.name)
522 log.info("building '%s' extension", ext.name)
524 # First, scan the sources for SWIG definition files (.i), run
525 # SWIG on 'em to create .c files, and modify the sources list
527 sources = self.swig_sources(sources, ext)
529 # Next, compile the source code to object files.
531 # XXX not honouring 'define_macros' or 'undef_macros' -- the
532 # CCompiler API needs to change to accommodate this, and I
533 # want to do one thing at a time!
535 # Two possible sources for extra compiler arguments:
536 # - 'extra_compile_args' in Extension object
537 # - CFLAGS environment variable (not particularly
538 # elegant, but people seem to expect it and I
540 # The environment variable should take precedence, and
541 # any sensible compiler will give precedence to later
542 # command line args. Hence we combine them in order:
543 extra_args = ext.extra_compile_args or []
545 macros = ext.define_macros[:]
546 for undef in ext.undef_macros:
547 macros.append((undef,))
549 objects = self.compiler.compile(
551 output_dir=self.build_temp,
553 include_dirs=ext.include_dirs,
555 extra_postargs=extra_args,
559 # XXX outdated variable, kept here in case third-part code
561 self._built_objects = objects[:]
563 # Now link the object files together into a "shared object" --
564 # of course, first we have to figure out all the other things
565 # that go into the mix.
566 if ext.extra_objects:
567 objects.extend(ext.extra_objects)
568 extra_args = ext.extra_link_args or []
570 # Detect target language, if not provided
571 language = ext.language or self.compiler.detect_language(sources)
573 self.compiler.link_shared_object(
576 libraries=self.get_libraries(ext),
577 library_dirs=ext.library_dirs,
578 runtime_library_dirs=ext.runtime_library_dirs,
579 extra_postargs=extra_args,
580 export_symbols=self.get_export_symbols(ext),
582 build_temp=self.build_temp,
583 target_lang=language,
586 def swig_sources(self, sources, extension):
587 """Walk the list of source files in 'sources', looking for SWIG
588 interface (.i) files. Run SWIG on all that are found, and
589 return a modified 'sources' list with SWIG source files replaced
590 by the generated C (or C++) files.
596 # XXX this drops generated C/C++ files into the source tree, which
597 # is fine for developers who want to distribute the generated
598 # source -- but there should be an option to put SWIG output in
602 log.warning("--swig-cpp is deprecated - use --swig-opts=-c++")
606 or ('-c++' in self.swig_opts)
607 or ('-c++' in extension.swig_opts)
613 for source in sources:
614 (base, ext) = os.path.splitext(source)
615 if ext == ".i": # SWIG interface file
616 new_sources.append(base + '_wrap' + target_ext)
617 swig_sources.append(source)
618 swig_targets[source] = new_sources[-1]
620 new_sources.append(source)
625 swig = self.swig or self.find_swig()
626 swig_cmd = [swig, "-python"]
627 swig_cmd.extend(self.swig_opts)
629 swig_cmd.append("-c++")
631 # Do not override commandline arguments
632 if not self.swig_opts:
633 for o in extension.swig_opts:
636 for source in swig_sources:
637 target = swig_targets[source]
638 log.info("swigging %s to %s", source, target)
639 self.spawn(swig_cmd + ["-o", target, source])
644 """Return the name of the SWIG executable. On Unix, this is
645 just "swig" -- it should be in the PATH. Tries a bit harder on
648 if os.name == "posix":
650 elif os.name == "nt":
651 # Look for SWIG in its standard installation directory on
652 # Windows (or so I presume!). If we find it there, great;
653 # if not, act like Unix and assume it's in the PATH.
654 for vers in ("1.3", "1.2", "1.1"):
655 fn = os.path.join("c:\\swig%s" % vers, "swig.exe")
656 if os.path.isfile(fn):
661 raise DistutilsPlatformError(
662 "I don't know how to find (much less run) SWIG "
663 "on platform '%s'" % os.name
666 # -- Name generators -----------------------------------------------
667 # (extension names, filenames, whatever)
668 def get_ext_fullpath(self, ext_name):
669 """Returns the path of the filename for a given extension.
671 The file is located in `build_lib` or directly in the package
674 fullname = self.get_ext_fullname(ext_name)
675 modpath = fullname.split('.')
676 filename = self.get_ext_filename(modpath[-1])
679 # no further work needed
681 # build_dir/package/path/filename
682 filename = os.path.join(*modpath[:-1] + [filename])
683 return os.path.join(self.build_lib, filename)
685 # the inplace option requires to find the package directory
686 # using the build_py command for that
687 package = '.'.join(modpath[0:-1])
688 build_py = self.get_finalized_command('build_py')
689 package_dir = os.path.abspath(build_py.get_package_dir(package))
692 # package_dir/filename
693 return os.path.join(package_dir, filename)
695 def get_ext_fullname(self, ext_name):
696 """Returns the fullname of a given extension name.
698 Adds the `package.` prefix"""
699 if self.package is None:
702 return self.package + '.' + ext_name
704 def get_ext_filename(self, ext_name):
705 r"""Convert the name of an extension (eg. "foo.bar") into the name
706 of the file from which it will be loaded (eg. "foo/bar.so", or
709 from ..sysconfig import get_config_var
711 ext_path = ext_name.split('.')
712 ext_suffix = get_config_var('EXT_SUFFIX')
713 return os.path.join(*ext_path) + ext_suffix
715 def get_export_symbols(self, ext):
716 """Return the list of symbols that a shared extension has to
717 export. This either uses 'ext.export_symbols' or, if it's not
718 provided, "PyInit_" + module_name. Only relevant on Windows, where
719 the .pyd file (DLL) must export the module "PyInit_" function.
721 name = ext.name.split('.')[-1]
723 # Unicode module name support as defined in PEP-489
724 # https://www.python.org/dev/peps/pep-0489/#export-hook-name
726 except UnicodeEncodeError:
727 suffix = 'U_' + name.encode('punycode').replace(b'-', b'_').decode('ascii')
731 initfunc_name = "PyInit" + suffix
732 if initfunc_name not in ext.export_symbols:
733 ext.export_symbols.append(initfunc_name)
734 return ext.export_symbols
736 def get_libraries(self, ext): # noqa: C901
737 """Return the list of libraries to link against when building a
738 shared extension. On most platforms, this is just 'ext.libraries';
739 on Windows, we add the Python library (eg. python20.dll).
741 # The python library is always needed on Windows. For MSVC, this
742 # is redundant, since the library is mentioned in a pragma in
743 # pyconfig.h that MSVC groks. The other Windows compilers all seem
744 # to need it mentioned explicitly, though, so that's what we do.
745 # Append '_d' to the python import library on debug builds.
746 if sys.platform == "win32":
747 from .._msvccompiler import MSVCCompiler
749 if not isinstance(self.compiler, MSVCCompiler):
750 template = "python%d%d"
752 template = template + '_d'
753 pythonlib = template % (
754 sys.hexversion >> 24,
755 (sys.hexversion >> 16) & 0xFF,
757 # don't extend ext.libraries, it may be shared with other
758 # extensions, it is a reference to the original list
759 return ext.libraries + [pythonlib]
761 # On Android only the main executable and LD_PRELOADs are considered
762 # to be RTLD_GLOBAL, all the dependencies of the main executable
763 # remain RTLD_LOCAL and so the shared libraries must be linked with
764 # libpython when python is built with a shared python library (issue
766 # On Cygwin (and if required, other POSIX-like platforms based on
767 # Windows like MinGW) it is simply necessary that all symbols in
768 # shared libraries are resolved at link time.
769 from ..sysconfig import get_config_var
771 link_libpython = False
772 if get_config_var('Py_ENABLE_SHARED'):
773 # A native build on an Android device or on Cygwin
774 if hasattr(sys, 'getandroidapilevel'):
775 link_libpython = True
776 elif sys.platform == 'cygwin':
777 link_libpython = True
778 elif '_PYTHON_HOST_PLATFORM' in os.environ:
779 # We are cross-compiling for one of the relevant platforms
780 if get_config_var('ANDROID_API_LEVEL') != 0:
781 link_libpython = True
782 elif get_config_var('MACHDEP') == 'cygwin':
783 link_libpython = True
786 ldversion = get_config_var('LDVERSION')
787 return ext.libraries + ['python' + ldversion]
789 return ext.libraries + py37compat.pythonlib()