.. method:: set_data(self, path, data)
Optional abstract method which writes the specified bytes to a file
- path. When writing to the path fails because the path is read-only, do
- not propagate the exception.
+ path. Any intermediate directories which do not exist are to be created
+ automatically.
+
+ When writing to the path fails because the path is read-only
+ (:attr:`errno.EACCES`), do not propagate the exception.
.. method:: get_code(self, fullname)
"""
if 'PYTHONCASEOK' in os.environ:
return True
- elif check in os.listdir(directory):
+ elif check in os.listdir(directory if directory else os.getcwd()):
return True
return False
def _path_join(*args):
"""Replacement for os.path.join."""
return path_sep.join(x[:-len(path_sep)] if x.endswith(path_sep) else x
- for x in args)
+ for x in args if x)
def _path_exists(path):
# XXX Could also expose Modules/getpath.c:isdir()
def _path_isdir(path):
"""Replacement for os.path.isdir."""
+ if not path:
+ path = _os.getcwd()
return _path_is_mode_type(path, 0o040000)
new.__dict__.update(old.__dict__)
+code_type = type(_wrap.__code__)
+
# Finder/loader utility code ##################################################
def set_package(fxn):
the second argument.
"""
- def decorated(self, fullname):
+ def decorated(self, fullname, *args, **kwargs):
module = sys.modules.get(fullname)
is_reload = bool(module)
if not is_reload:
module = imp.new_module(fullname)
sys.modules[fullname] = module
try:
- return fxn(self, module)
+ return fxn(self, module, *args, **kwargs)
except:
if not is_reload:
del sys.modules[fullname]
return imp.is_frozen_package(fullname)
-class SourceLoader:
+class _LoaderBasics:
+
+ """Base class of common code needed by both SourceLoader and
+ _SourcelessFileLoader."""
+
+ def is_package(self, fullname):
+ """Concrete implementation of InspectLoader.is_package by checking if
+ the path returned by get_filename has a filename of '__init__.py'."""
+ filename = self.get_filename(fullname).rpartition(path_sep)[2]
+ return filename.rsplit('.', 1)[0] == '__init__'
+
+ def _bytes_from_bytecode(self, fullname, data, source_mtime):
+ """Return the marshalled bytes from bytecode, verifying the magic
+ number and timestamp alon the way.
+
+ If source_mtime is None then skip the timestamp check.
+
+ """
+ magic = data[:4]
+ raw_timestamp = data[4:8]
+ if len(magic) != 4 or magic != imp.get_magic():
+ raise ImportError("bad magic number in {}".format(fullname))
+ elif len(raw_timestamp) != 4:
+ raise EOFError("bad timestamp in {}".format(fullname))
+ elif source_mtime is not None:
+ if marshal._r_long(raw_timestamp) != source_mtime:
+ raise ImportError("bytecode is stale for {}".format(fullname))
+ # Can't return the code object as errors from marshal loading need to
+ # propagate even when source is available.
+ return data[8:]
+
+ @module_for_loader
+ def _load_module(self, module, *, sourceless=False):
+ """Helper for load_module able to handle either source or sourceless
+ loading."""
+ name = module.__name__
+ code_object = self.get_code(name)
+ module.__file__ = self.get_filename(name)
+ if not sourceless:
+ module.__cached__ = imp.cache_from_source(module.__file__)
+ else:
+ module.__cached__ = module.__file__
+ module.__package__ = name
+ if self.is_package(name):
+ module.__path__ = [module.__file__.rsplit(path_sep, 1)[0]]
+ else:
+ module.__package__ = module.__package__.rpartition('.')[0]
+ module.__loader__ = self
+ exec(code_object, module.__dict__)
+ return module
+
+
+class SourceLoader(_LoaderBasics):
def path_mtime(self, path:str) -> int:
"""Optional method that returns the modification time for the specified
"""
raise NotImplementedError
- def is_package(self, fullname):
- """Concrete implementation of InspectLoader.is_package by checking if
- the path returned by get_filename has a filename of '__init__.py'."""
- filename = self.get_filename(fullname).rsplit(path_sep, 1)[1]
- return filename.rsplit('.', 1)[0] == '__init__'
def get_source(self, fullname):
"""Concrete implementation of InspectLoader.get_source."""
except IOError:
pass
else:
- magic = data[:4]
- raw_timestamp = data[4:8]
- if (len(magic) == 4 and len(raw_timestamp) == 4 and
- magic == imp.get_magic() and
- marshal._r_long(raw_timestamp) == source_mtime):
- return marshal.loads(data[8:])
+ try:
+ bytes_data = self._bytes_from_bytecode(fullname, data,
+ source_mtime)
+ except (ImportError, EOFError):
+ pass
+ else:
+ found = marshal.loads(bytes_data)
+ if isinstance(found, code_type):
+ return found
+ else:
+ msg = "Non-code object in {}"
+ raise ImportError(msg.format(bytecode_path))
source_bytes = self.get_data(source_path)
code_object = compile(source_bytes, source_path, 'exec',
dont_inherit=True)
pass
return code_object
- @module_for_loader
- def load_module(self, module):
+ def load_module(self, fullname):
"""Concrete implementation of Loader.load_module.
Requires ExecutionLoader.get_filename and ResourceLoader.get_data to be
get_code uses/writes bytecode.
"""
- name = module.__name__
- code_object = self.get_code(name)
- module.__file__ = self.get_filename(name)
- module.__cached__ = imp.cache_from_source(module.__file__)
- module.__package__ = name
- is_package = self.is_package(name)
- if is_package:
- module.__path__ = [module.__file__.rsplit(path_sep, 1)[0]]
- else:
- module.__package__ = module.__package__.rpartition('.')[0]
- module.__loader__ = self
- exec(code_object, module.__dict__)
- return module
-
+ return self._load_module(fullname)
-class _SourceFileLoader(SourceLoader):
- """Concrete implementation of SourceLoader.
+class _FileLoader:
- NOT A PUBLIC CLASS! Do not expect any API stability from this class, so DO
- NOT SUBCLASS IN YOUR OWN CODE!
-
- """
+ """Base file loader class which implements the loader protocol methods that
+ require file system usage."""
def __init__(self, fullname, path):
+ """Cache the module name and the path to the file found by the
+ finder."""
self._name = fullname
self._path = path
"""Return the path to the source file as found by the finder."""
return self._path
+ def get_data(self, path):
+ """Return the data from path as raw bytes."""
+ with _closing(_io.FileIO(path, 'r')) as file:
+ return file.read()
+
+
+class _SourceFileLoader(_FileLoader, SourceLoader):
+
+ """Concrete implementation of SourceLoader using the file system."""
+
def path_mtime(self, path):
"""Return the modification time for the path."""
return int(_os.stat(path).st_mtime)
- def set_data(self, data, path):
+ def set_data(self, path, data):
"""Write bytes data to a file."""
try:
- with _closing(_io.FileIO(bytecode_path, 'w')) as file:
+ with _closing(_io.FileIO(path, 'wb')) as file:
file.write(data)
except IOError as exc:
- if exc.errno != errno.EACCES:
+ if exc.errno == errno.ENOENT:
+ directory, _, filename = path.rpartition(path_sep)
+ sub_directories = []
+ while not _path_isdir(directory):
+ directory, _, sub_dir = directory.rpartition(path_sep)
+ sub_directories.append(sub_dir)
+ for part in reversed(sub_directories):
+ directory = _path_join(directory, part)
+ try:
+ _os.mkdir(directory)
+ except IOError as exc:
+ if exc.errno != errno.EACCES:
+ raise
+ else:
+ return
+ return self.set_data(path, data)
+ elif exc.errno != errno.EACCES:
raise
-class PyLoader:
-
- """Loader base class for Python source code.
-
- Subclasses need to implement the methods:
-
- - source_path
- - get_data
- - is_package
-
- """
-
- @module_for_loader
- def load_module(self, module):
- """Initialize the module."""
- name = module.__name__
- code_object = self.get_code(module.__name__)
- module.__file__ = self.get_filename(name)
- if self.is_package(name):
- module.__path__ = [module.__file__.rsplit(path_sep, 1)[0]]
- module.__package__ = module.__name__
- if not hasattr(module, '__path__'):
- module.__package__ = module.__package__.rpartition('.')[0]
- module.__loader__ = self
- exec(code_object, module.__dict__)
- return module
-
- def get_filename(self, fullname):
- """Return the path to the source file, else raise ImportError."""
- path = self.source_path(fullname)
- if path is not None:
- return path
- else:
- raise ImportError("no source path available for "
- "{0!r}".format(fullname))
-
- def get_code(self, fullname):
- """Get a code object from source."""
- source_path = self.source_path(fullname)
- if source_path is None:
- message = "a source path must exist to load {0}".format(fullname)
- raise ImportError(message)
- source = self.get_data(source_path)
- return compile(source, source_path, 'exec', dont_inherit=True)
-
- # Never use in implementing import! Imports code within the method.
- def get_source(self, fullname):
- """Return the source code for a module.
-
- self.source_path() and self.get_data() are used to implement this
- method.
-
- """
- path = self.source_path(fullname)
- if path is None:
- return None
- try:
- source_bytes = self.get_data(path)
- except IOError:
- return ImportError("source not available through get_data()")
- import io
- import tokenize
- encoding = tokenize.detect_encoding(io.BytesIO(source_bytes).readline)
- return source_bytes.decode(encoding[0])
-
-
-class PyPycLoader(PyLoader):
-
- """Loader base class for Python source and bytecode.
+class _SourcelessFileLoader(_FileLoader, _LoaderBasics):
- Requires implementing the methods needed for PyLoader as well as
- source_mtime, bytecode_path, and write_bytecode.
+ """Loader which handles sourceless file imports."""
- """
-
- def get_filename(self, fullname):
- """Return the source or bytecode file path."""
- path = self.source_path(fullname)
- if path is not None:
- return path
- path = self.bytecode_path(fullname)
- if path is not None:
- return path
- raise ImportError("no source or bytecode path available for "
- "{0!r}".format(fullname))
+ def load_module(self, fullname):
+ return self._load_module(fullname, sourceless=True)
def get_code(self, fullname):
- """Get a code object from source or bytecode."""
- # XXX Care enough to make sure this call does not happen if the magic
- # number is bad?
- source_timestamp = self.source_mtime(fullname)
- # Try to use bytecode if it is available.
- bytecode_path = self.bytecode_path(fullname)
- if bytecode_path:
- data = self.get_data(bytecode_path)
- try:
- magic = data[:4]
- if len(magic) < 4:
- raise ImportError("bad magic number in {}".format(fullname))
- raw_timestamp = data[4:8]
- if len(raw_timestamp) < 4:
- raise EOFError("bad timestamp in {}".format(fullname))
- pyc_timestamp = marshal._r_long(raw_timestamp)
- bytecode = data[8:]
- # Verify that the magic number is valid.
- if imp.get_magic() != magic:
- raise ImportError("bad magic number in {}".format(fullname))
- # Verify that the bytecode is not stale (only matters when
- # there is source to fall back on.
- if source_timestamp:
- if pyc_timestamp < source_timestamp:
- raise ImportError("bytecode is stale")
- except (ImportError, EOFError):
- # If source is available give it a shot.
- if source_timestamp is not None:
- pass
- else:
- raise
- else:
- # Bytecode seems fine, so try to use it.
- # XXX If the bytecode is ill-formed, would it be beneficial to
- # try for using source if available and issue a warning?
- return marshal.loads(bytecode)
- elif source_timestamp is None:
- raise ImportError("no source or bytecode available to create code "
- "object for {0!r}".format(fullname))
- # Use the source.
- code_object = super().get_code(fullname)
- # Generate bytecode and write it out.
- if not sys.dont_write_bytecode:
- data = bytearray(imp.get_magic())
- data.extend(marshal._w_long(source_timestamp))
- data.extend(marshal.dumps(code_object))
- self.write_bytecode(fullname, data)
- return code_object
-
-
-class _PyFileLoader(PyLoader):
-
- """Load a Python source file."""
-
- def __init__(self, name, path, is_pkg):
- self._name = name
- self._is_pkg = is_pkg
- # Figure out the base path based on whether it was source or bytecode
- # that was found.
- try:
- self._base_path = _path_without_ext(path, imp.PY_SOURCE)
- except ValueError:
- self._base_path = _path_without_ext(path, imp.PY_COMPILED)
-
- def _find_path(self, ext_type):
- """Find a path from the base path and the specified extension type that
- exists, returning None if one is not found."""
- for suffix in _suffix_list(ext_type):
- path = self._base_path + suffix
- if _path_exists(path):
- return path
+ path = self.get_filename(fullname)
+ data = self.get_data(path)
+ bytes_data = self._bytes_from_bytecode(fullname, data, None)
+ found = marshal.loads(bytes_data)
+ if isinstance(found, code_type):
+ return found
else:
- return None
-
- @_check_name
- def source_path(self, fullname):
- """Return the path to an existing source file for the module, or None
- if one cannot be found."""
- # Not a property so that it is easy to override.
- return self._find_path(imp.PY_SOURCE)
+ raise ImportError("Non-code object in {}".format(path))
- def get_data(self, path):
- """Return the data from path as raw bytes."""
- return _io.FileIO(path, 'r').read() # Assuming bytes.
-
- @_check_name
- def is_package(self, fullname):
- """Return a boolean based on whether the module is a package.
-
- Raises ImportError (like get_source) if the loader cannot handle the
- package.
-
- """
- return self._is_pkg
-
-
-class _PyPycFileLoader(PyPycLoader, _PyFileLoader):
-
- """Load a module from a source or bytecode file."""
-
- def _find_path(self, ext_type):
- """Return PEP 3147 path if ext_type is PY_COMPILED, otherwise
- super()._find_path() is called."""
- if ext_type == imp.PY_COMPILED:
- # We don't really care what the extension on self._base_path is,
- # as long as it has exactly one dot.
- source_path = self._base_path + '.py'
- pycache_path = imp.cache_from_source(source_path)
- legacy_path = self._base_path + '.pyc'
- # The rule is: if the source file exists, then Python always uses
- # the __pycache__/foo.<tag>.pyc file. If the source file does not
- # exist, then Python uses the legacy path.
- pyc_path = (pycache_path
- if _path_exists(source_path)
- else legacy_path)
- return (pyc_path if _path_exists(pyc_path) else None)
- return super()._find_path(ext_type)
-
- @_check_name
- def source_mtime(self, name):
- """Return the modification time of the source for the specified
- module."""
- source_path = self.source_path(name)
- if not source_path:
- return None
- return int(_os.stat(source_path).st_mtime)
-
- @_check_name
- def bytecode_path(self, fullname):
- """Return the path to a bytecode file, or None if one does not
- exist."""
- # Not a property for easy overriding.
- return self._find_path(imp.PY_COMPILED)
-
- @_check_name
- def write_bytecode(self, name, data):
- """Write out 'data' for the specified module, returning a boolean
- signifying if the write-out actually occurred.
-
- Raises ImportError (just like get_source) if the specified module
- cannot be handled by the loader.
-
- """
- bytecode_path = self.bytecode_path(name)
- if not bytecode_path:
- source_path = self.source_path(name)
- bytecode_path = imp.cache_from_source(source_path)
- # Ensure that the __pycache__ directory exists. We can't use
- # os.path.dirname() here.
- dirname, sep, basename = bytecode_path.rpartition(path_sep)
- try:
- _os.mkdir(dirname)
- except OSError as error:
- if error.errno != errno.EEXIST:
- raise
- try:
- # Assuming bytes.
- with _closing(_io.FileIO(bytecode_path, 'w')) as bytecode_file:
- bytecode_file.write(data)
- return True
- except IOError as exc:
- if exc.errno == errno.EACCES:
- return False
- else:
- raise
+ def get_source(self, fullname):
+ """Return None as there is no source code."""
+ return None
class _ExtensionFileLoader:
"""
- def __init__(self, name, path, is_pkg):
+ def __init__(self, name, path):
"""Initialize the loader.
If is_pkg is True then an exception is raised as extension modules
"""
self._name = name
self._path = path
- if is_pkg:
- raise ValueError("extension modules cannot be packages")
@_check_name
@set_package
return None
-class _ChainedFinder:
-
- """Finder that sequentially calls other finders."""
-
- def __init__(self, *finders):
- self._finders = finders
-
- def find_module(self, fullname, path=None):
- for finder in self._finders:
- result = finder.find_module(fullname, path)
- if result:
- return result
- else:
- return None
-
-
class _FileFinder:
- """Base class for file finders.
-
- Subclasses are expected to define the following attributes:
-
- * _suffixes
- Sequence of file suffixes whose order will be followed.
+ """File-based finder.
- * _possible_package
- True if importer should check for packages.
-
- * _loader
- A callable that takes the module name, a file path, and whether
- the path points to a package and returns a loader for the module
- found at that path.
+ Constructor takes a list of objects detailing what file extensions their
+ loader supports along with whether it can be used for a package.
"""
- def __init__(self, path_entry):
- """Initialize an importer for the passed-in sys.path entry (which is
- assumed to have already been verified as an existing directory).
-
- Can be used as an entry on sys.path_hook.
-
- """
- absolute_path = _path_absolute(path_entry)
- if not _path_isdir(absolute_path):
- raise ImportError("only directories are supported")
- self._path_entry = absolute_path
-
- def find_module(self, fullname, path=None):
+ def __init__(self, path, *details):
+ """Initialize with finder details."""
+ packages = []
+ modules = []
+ for detail in details:
+ modules.extend((suffix, detail.loader) for suffix in detail.suffixes)
+ if detail.supports_packages:
+ packages.extend((suffix, detail.loader)
+ for suffix in detail.suffixes)
+ self.packages = packages
+ self.modules = modules
+ self.path = path
+
+ def find_module(self, fullname):
+ """Try to find a loader for the specified module."""
tail_module = fullname.rpartition('.')[2]
- package_directory = None
- if self._possible_package:
- for ext in self._suffixes:
- package_directory = _path_join(self._path_entry, tail_module)
- init_filename = '__init__' + ext
- package_init = _path_join(package_directory, init_filename)
- if (_path_isfile(package_init) and
- _case_ok(self._path_entry, tail_module) and
- _case_ok(package_directory, init_filename)):
- return self._loader(fullname, package_init, True)
- for ext in self._suffixes:
- file_name = tail_module + ext
- file_path = _path_join(self._path_entry, file_name)
- if (_path_isfile(file_path) and
- _case_ok(self._path_entry, file_name)):
- return self._loader(fullname, file_path, False)
- else:
- # Raise a warning if it matches a directory w/o an __init__ file.
- if (package_directory is not None and
- _path_isdir(package_directory) and
- _case_ok(self._path_entry, tail_module)):
- _warnings.warn("Not importing directory %s: missing __init__"
- % package_directory, ImportWarning)
- return None
-
-
-class _PyFileFinder(_FileFinder):
-
- """Importer for source/bytecode files."""
-
- _possible_package = True
- _loader = _PyFileLoader
-
- def __init__(self, path_entry):
- # Lack of imp during class creation means _suffixes is set here.
- # Make sure that Python source files are listed first! Needed for an
- # optimization by the loader.
- self._suffixes = _suffix_list(imp.PY_SOURCE)
- super().__init__(path_entry)
-
-
-class _PyPycFileFinder(_PyFileFinder):
+ base_path = _path_join(self.path, tail_module)
+ if _path_isdir(base_path) and _case_ok(self.path, tail_module):
+ for suffix, loader in self.packages:
+ init_filename = '__init__' + suffix
+ full_path = _path_join(base_path, init_filename)
+ if (_path_isfile(full_path) and
+ _case_ok(base_path, init_filename)):
+ return loader(fullname, full_path)
+ else:
+ msg = "Not importing directory {}: missing __init__"
+ _warnings.warn(msg.format(base_path), ImportWarning)
+ for suffix, loader in self.modules:
+ mod_filename = tail_module + suffix
+ full_path = _path_join(self.path, mod_filename)
+ if _path_isfile(full_path) and _case_ok(self.path, mod_filename):
+ return loader(fullname, full_path)
+ return None
- """Finder for source and bytecode files."""
+class _SourceFinderDetails:
- _loader = _PyPycFileLoader
+ loader = _SourceFileLoader
+ supports_packages = True
- def __init__(self, path_entry):
- super().__init__(path_entry)
- self._suffixes += _suffix_list(imp.PY_COMPILED)
+ def __init__(self):
+ self.suffixes = _suffix_list(imp.PY_SOURCE)
+class _SourcelessFinderDetails:
+ loader = _SourcelessFileLoader
+ supports_packages = True
+ def __init__(self):
+ self.suffixes = _suffix_list(imp.PY_COMPILED)
-class _ExtensionFileFinder(_FileFinder):
- """Importer for extension files."""
+class _ExtensionFinderDetails:
- _possible_package = False
- _loader = _ExtensionFileLoader
+ loader = _ExtensionFileLoader
+ supports_packages = False
- def __init__(self, path_entry):
- # Assigning to _suffixes here instead of at the class level because
- # imp is not imported at the time of class creation.
- self._suffixes = _suffix_list(imp.C_EXTENSION)
- super().__init__(path_entry)
+ def __init__(self):
+ self.suffixes = _suffix_list(imp.C_EXTENSION)
# Import itself ###############################################################
-def _chained_path_hook(*path_hooks):
- """Create a closure which sequentially checks path hooks to see which ones
- (if any) can work with a path."""
- def path_hook(entry):
- """Check to see if 'entry' matches any of the enclosed path hooks."""
- finders = []
- for hook in path_hooks:
- try:
- finder = hook(entry)
- except ImportError:
- continue
- else:
- finders.append(finder)
- if not finders:
- raise ImportError("no finder found")
- else:
- return _ChainedFinder(*finders)
-
- return path_hook
+def _file_path_hook(path):
+ """If the path is a directory, return a file-based finder."""
+ if _path_isdir(path):
+ return _FileFinder(path, _ExtensionFinderDetails(),
+ _SourceFinderDetails(),
+ _SourcelessFinderDetails())
+ else:
+ raise ImportError("only directories are supported")
-_DEFAULT_PATH_HOOK = _chained_path_hook(_ExtensionFileFinder, _PyPycFileFinder)
+_DEFAULT_PATH_HOOK = _file_path_hook
class _DefaultPathFinder(PathFinder):
else:
return path
-PyLoader.register(_bootstrap.PyLoader)
-
class PyPycLoader(PyLoader):
self.write_bytecode(fullname, data)
return code_object
-
@abc.abstractmethod
def source_mtime(self, fullname:str) -> int:
"""Abstract method which when implemented should return the
bytecode for the module, returning a boolean representing whether the
bytecode was written or not."""
raise NotImplementedError
-
-PyPycLoader.register(_bootstrap.PyPycLoader)
good_name = ext_util.NAME
bad_name = good_name.upper()
assert good_name != bad_name
- finder = _bootstrap._ExtensionFileFinder(ext_util.PATH)
+ finder = _bootstrap._FileFinder(ext_util.PATH,
+ _bootstrap._ExtensionFinderDetails())
return finder.find_module(bad_name)
def test_case_sensitive(self):
"""Test the finder for extension modules."""
def find_module(self, fullname):
- importer = _bootstrap._ExtensionFileFinder(util.PATH)
+ importer = _bootstrap._FileFinder(util.PATH,
+ _bootstrap._ExtensionFinderDetails())
return importer.find_module(fullname)
def test_module(self):
def load_module(self, fullname):
loader = _bootstrap._ExtensionFileLoader(ext_util.NAME,
- ext_util.FILEPATH, False)
+ ext_util.FILEPATH)
return loader.load_module(fullname)
def test_module(self):
# XXX Should it only work for directories containing an extension module?
def hook(self, entry):
- return _bootstrap._ExtensionFileFinder(entry)
+ return _bootstrap._file_path_hook(entry)
def test_success(self):
# Path hook should handle a directory where a known extension module
import imp
import os
import sys
+import tempfile
from test import support
from types import MethodType
import unittest
def test_implicit_hooks(self):
# Test that the implicit path hooks are used.
- existing_path = os.path.dirname(support.TESTFN)
bad_path = '<path>'
module = '<module>'
assert not os.path.exists(bad_path)
- with util.import_state():
- nothing = _bootstrap._DefaultPathFinder.find_module(module,
- path=[existing_path])
- self.assertTrue(nothing is None)
- self.assertTrue(existing_path in sys.path_importer_cache)
- self.assertTrue(not isinstance(sys.path_importer_cache[existing_path],
- imp.NullImporter))
- nothing = _bootstrap._DefaultPathFinder.find_module(module,
- path=[bad_path])
- self.assertTrue(nothing is None)
- self.assertTrue(bad_path in sys.path_importer_cache)
- self.assertTrue(isinstance(sys.path_importer_cache[bad_path],
- imp.NullImporter))
+ existing_path = tempfile.mkdtemp()
+ try:
+ with util.import_state():
+ nothing = _bootstrap._DefaultPathFinder.find_module(module,
+ path=[existing_path])
+ self.assertTrue(nothing is None)
+ self.assertTrue(existing_path in sys.path_importer_cache)
+ result = isinstance(sys.path_importer_cache[existing_path],
+ imp.NullImporter)
+ self.assertFalse(result)
+ nothing = _bootstrap._DefaultPathFinder.find_module(module,
+ path=[bad_path])
+ self.assertTrue(nothing is None)
+ self.assertTrue(bad_path in sys.path_importer_cache)
+ self.assertTrue(isinstance(sys.path_importer_cache[bad_path],
+ imp.NullImporter))
+ finally:
+ os.rmdir(existing_path)
+
def test_path_importer_cache_has_None(self):
# Test that the default hook is used when sys.path_importer_cache
this script.
XXX FAILING
- test_import
- execution bit
+ * test_import
+ - test_incorrect_code_name
file name differing between __file__ and co_filename (r68360 on trunk)
+ - test_import_by_filename
+ exception for trying to import by file name does not match
"""
import importlib
def test_Loader(self):
self.raises_NotImplementedError(self.Loader(), 'load_module')
+ # XXX misplaced; should be somewhere else
def test_Finder(self):
self.raises_NotImplementedError(self.Finder(), 'find_module')
assert name != name.lower()
def find(self, path):
- finder = _bootstrap._PyPycFileFinder(path)
+ finder = _bootstrap._FileFinder(path,
+ _bootstrap._SourceFinderDetails(),
+ _bootstrap._SourcelessFinderDetails())
return finder.find_module(self.name)
def sensitivity_test(self):
sensitive_pkg = 'sensitive.{0}'.format(self.name)
insensitive_pkg = 'insensitive.{0}'.format(self.name.lower())
context = source_util.create_modules(insensitive_pkg, sensitive_pkg)
- with context as mapping:
+ with context as mapping:
sensitive_path = os.path.join(mapping['.root'], 'sensitive')
insensitive_path = os.path.join(mapping['.root'], 'insensitive')
return self.find(sensitive_path), self.find(insensitive_path)
env.unset('PYTHONCASEOK')
sensitive, insensitive = self.sensitivity_test()
self.assertTrue(hasattr(sensitive, 'load_module'))
- self.assertIn(self.name, sensitive._base_path)
+ self.assertIn(self.name, sensitive.get_filename(self.name))
self.assertIsNone(insensitive)
def test_insensitive(self):
env.set('PYTHONCASEOK', '1')
sensitive, insensitive = self.sensitivity_test()
self.assertTrue(hasattr(sensitive, 'load_module'))
- self.assertIn(self.name, sensitive._base_path)
+ self.assertIn(self.name, sensitive.get_filename(self.name))
self.assertTrue(hasattr(insensitive, 'load_module'))
- self.assertIn(self.name, insensitive._base_path)
+ self.assertIn(self.name, insensitive.get_filename(self.name))
def test_main():
from . import util as source_util
import imp
+import marshal
import os
import py_compile
import stat
# [basic]
def test_module(self):
with source_util.create_modules('_temp') as mapping:
- loader = _bootstrap._PyPycFileLoader('_temp', mapping['_temp'],
- False)
+ loader = _bootstrap._SourceFileLoader('_temp', mapping['_temp'])
module = loader.load_module('_temp')
self.assertTrue('_temp' in sys.modules)
check = {'__name__': '_temp', '__file__': mapping['_temp'],
def test_package(self):
with source_util.create_modules('_pkg.__init__') as mapping:
- loader = _bootstrap._PyPycFileLoader('_pkg',
- mapping['_pkg.__init__'],
- True)
+ loader = _bootstrap._SourceFileLoader('_pkg',
+ mapping['_pkg.__init__'])
module = loader.load_module('_pkg')
self.assertTrue('_pkg' in sys.modules)
check = {'__name__': '_pkg', '__file__': mapping['_pkg.__init__'],
def test_lacking_parent(self):
with source_util.create_modules('_pkg.__init__', '_pkg.mod')as mapping:
- loader = _bootstrap._PyPycFileLoader('_pkg.mod',
- mapping['_pkg.mod'], False)
+ loader = _bootstrap._SourceFileLoader('_pkg.mod',
+ mapping['_pkg.mod'])
module = loader.load_module('_pkg.mod')
self.assertTrue('_pkg.mod' in sys.modules)
check = {'__name__': '_pkg.mod', '__file__': mapping['_pkg.mod'],
def test_module_reuse(self):
with source_util.create_modules('_temp') as mapping:
- loader = _bootstrap._PyPycFileLoader('_temp', mapping['_temp'],
- False)
+ loader = _bootstrap._SourceFileLoader('_temp', mapping['_temp'])
module = loader.load_module('_temp')
module_id = id(module)
module_dict_id = id(module.__dict__)
# everything that has happened above can be too fast;
# force an mtime on the source that is guaranteed to be different
# than the original mtime.
- loader.source_mtime = self.fake_mtime(loader.source_mtime)
+ loader.path_mtime = self.fake_mtime(loader.path_mtime)
module = loader.load_module('_temp')
self.assertTrue('testing_var' in module.__dict__,
"'testing_var' not in "
setattr(orig_module, attr, value)
with open(mapping[name], 'w') as file:
file.write('+++ bad syntax +++')
- loader = _bootstrap._PyPycFileLoader('_temp', mapping['_temp'],
- False)
+ loader = _bootstrap._SourceFileLoader('_temp', mapping['_temp'])
with self.assertRaises(SyntaxError):
loader.load_module(name)
for attr in attributes:
with source_util.create_modules('_temp') as mapping:
with open(mapping['_temp'], 'w') as file:
file.write('=')
- loader = _bootstrap._PyPycFileLoader('_temp', mapping['_temp'],
- False)
+ loader = _bootstrap._SourceFileLoader('_temp', mapping['_temp'])
with self.assertRaises(SyntaxError):
loader.load_module('_temp')
self.assertTrue('_temp' not in sys.modules)
class BadBytecodeTest(unittest.TestCase):
def import_(self, file, module_name):
- loader = _bootstrap._PyPycFileLoader(module_name, file, False)
+ loader = self.loader(module_name, file)
module = loader.load_module(module_name)
self.assertTrue(module_name in sys.modules)
except KeyError:
pass
py_compile.compile(mapping[name])
- bytecode_path = imp.cache_from_source(mapping[name])
- with open(bytecode_path, 'rb') as file:
- bc = file.read()
- new_bc = manipulator(bc)
- with open(bytecode_path, 'wb') as file:
- if new_bc:
- file.write(new_bc)
- if del_source:
+ if not del_source:
+ bytecode_path = imp.cache_from_source(mapping[name])
+ else:
os.unlink(mapping[name])
- make_legacy_pyc(mapping[name])
+ bytecode_path = make_legacy_pyc(mapping[name])
+ if manipulator:
+ with open(bytecode_path, 'rb') as file:
+ bc = file.read()
+ new_bc = manipulator(bc)
+ with open(bytecode_path, 'wb') as file:
+ if new_bc is not None:
+ file.write(new_bc)
return bytecode_path
- @source_util.writes_bytecode_files
- def test_empty_file(self):
- # When a .pyc is empty, regenerate it if possible, else raise
- # ImportError.
+ def _test_empty_file(self, test, *, del_source=False):
with source_util.create_modules('_temp') as mapping:
bc_path = self.manipulate_bytecode('_temp', mapping,
- lambda bc: None)
- self.import_(mapping['_temp'], '_temp')
- with open(bc_path, 'rb') as file:
- self.assertGreater(len(file.read()), 8)
- self.manipulate_bytecode('_temp', mapping, lambda bc: None,
- del_source=True)
- with self.assertRaises(ImportError):
- self.import_(mapping['_temp'], '_temp')
+ lambda bc: b'',
+ del_source=del_source)
+ test('_temp', mapping, bc_path)
@source_util.writes_bytecode_files
- def test_partial_magic(self):
+ def _test_partial_magic(self, test, *, del_source=False):
# When their are less than 4 bytes to a .pyc, regenerate it if
# possible, else raise ImportError.
with source_util.create_modules('_temp') as mapping:
bc_path = self.manipulate_bytecode('_temp', mapping,
- lambda bc: bc[:3])
- self.import_(mapping['_temp'], '_temp')
- with open(bc_path, 'rb') as file:
- self.assertGreater(len(file.read()), 8)
- self.manipulate_bytecode('_temp', mapping, lambda bc: bc[:3],
- del_source=True)
+ lambda bc: bc[:3],
+ del_source=del_source)
+ test('_temp', mapping, bc_path)
+
+ def _test_magic_only(self, test, *, del_source=False):
+ with source_util.create_modules('_temp') as mapping:
+ bc_path = self.manipulate_bytecode('_temp', mapping,
+ lambda bc: bc[:4],
+ del_source=del_source)
+ test('_temp', mapping, bc_path)
+
+ def _test_partial_timestamp(self, test, *, del_source=False):
+ with source_util.create_modules('_temp') as mapping:
+ bc_path = self.manipulate_bytecode('_temp', mapping,
+ lambda bc: bc[:7],
+ del_source=del_source)
+ test('_temp', mapping, bc_path)
+
+ def _test_no_marshal(self, *, del_source=False):
+ with source_util.create_modules('_temp') as mapping:
+ bc_path = self.manipulate_bytecode('_temp', mapping,
+ lambda bc: bc[:8],
+ del_source=del_source)
+ file_path = mapping['_temp'] if not del_source else bc_path
+ with self.assertRaises(EOFError):
+ self.import_(file_path, '_temp')
+
+ def _test_non_code_marshal(self, *, del_source=False):
+ with source_util.create_modules('_temp') as mapping:
+ bytecode_path = self.manipulate_bytecode('_temp', mapping,
+ lambda bc: bc[:8] + marshal.dumps(b'abcd'),
+ del_source=del_source)
+ file_path = mapping['_temp'] if not del_source else bytecode_path
with self.assertRaises(ImportError):
- self.import_(mapping['_temp'], '_temp')
+ self.import_(file_path, '_temp')
+
+ def _test_bad_marshal(self, *, del_source=False):
+ with source_util.create_modules('_temp') as mapping:
+ bytecode_path = self.manipulate_bytecode('_temp', mapping,
+ lambda bc: bc[:8] + b'<test>',
+ del_source=del_source)
+ file_path = mapping['_temp'] if not del_source else bytecode_path
+ with self.assertRaises(ValueError):
+ self.import_(file_path, '_temp')
+
+ def _test_bad_magic(self, test, *, del_source=False):
+ with source_util.create_modules('_temp') as mapping:
+ bc_path = self.manipulate_bytecode('_temp', mapping,
+ lambda bc: b'\x00\x00\x00\x00' + bc[4:])
+ test('_temp', mapping, bc_path)
+
+
+class SourceLoaderBadBytecodeTest(BadBytecodeTest):
+
+ loader = _bootstrap._SourceFileLoader
+
+ @source_util.writes_bytecode_files
+ def test_empty_file(self):
+ # When a .pyc is empty, regenerate it if possible, else raise
+ # ImportError.
+ def test(name, mapping, bytecode_path):
+ self.import_(mapping[name], name)
+ with open(bytecode_path, 'rb') as file:
+ self.assertGreater(len(file.read()), 8)
+
+ self._test_empty_file(test)
+
+ def test_partial_magic(self):
+ def test(name, mapping, bytecode_path):
+ self.import_(mapping[name], name)
+ with open(bytecode_path, 'rb') as file:
+ self.assertGreater(len(file.read()), 8)
+
+ self._test_partial_magic(test)
@source_util.writes_bytecode_files
def test_magic_only(self):
# When there is only the magic number, regenerate the .pyc if possible,
# else raise EOFError.
- with source_util.create_modules('_temp') as mapping:
- bc_path = self.manipulate_bytecode('_temp', mapping,
- lambda bc: bc[:4])
- self.import_(mapping['_temp'], '_temp')
- with open(bc_path, 'rb') as file:
+ def test(name, mapping, bytecode_path):
+ self.import_(mapping[name], name)
+ with open(bytecode_path, 'rb') as file:
self.assertGreater(len(file.read()), 8)
- self.manipulate_bytecode('_temp', mapping, lambda bc: bc[:4],
- del_source=True)
- with self.assertRaises(EOFError):
- self.import_(mapping['_temp'], '_temp')
+
+ @source_util.writes_bytecode_files
+ def test_bad_magic(self):
+ # When the magic number is different, the bytecode should be
+ # regenerated.
+ def test(name, mapping, bytecode_path):
+ self.import_(mapping[name], name)
+ with open(bytecode_path, 'rb') as bytecode_file:
+ self.assertEqual(bytecode_file.read(4), imp.get_magic())
+
+ self._test_bad_magic(test)
@source_util.writes_bytecode_files
def test_partial_timestamp(self):
# When the timestamp is partial, regenerate the .pyc, else
# raise EOFError.
- with source_util.create_modules('_temp') as mapping:
- bc_path = self.manipulate_bytecode('_temp', mapping,
- lambda bc: bc[:7])
- self.import_(mapping['_temp'], '_temp')
+ def test(name, mapping, bc_path):
+ self.import_(mapping[name], name)
with open(bc_path, 'rb') as file:
self.assertGreater(len(file.read()), 8)
- self.manipulate_bytecode('_temp', mapping, lambda bc: bc[:7],
- del_source=True)
- with self.assertRaises(EOFError):
- self.import_(mapping['_temp'], '_temp')
@source_util.writes_bytecode_files
def test_no_marshal(self):
# When there is only the magic number and timestamp, raise EOFError.
- with source_util.create_modules('_temp') as mapping:
- bc_path = self.manipulate_bytecode('_temp', mapping,
- lambda bc: bc[:8])
- with self.assertRaises(EOFError):
- self.import_(mapping['_temp'], '_temp')
+ self._test_no_marshal()
@source_util.writes_bytecode_files
- def test_bad_magic(self):
- # When the magic number is different, the bytecode should be
- # regenerated.
- with source_util.create_modules('_temp') as mapping:
- bc_path = self.manipulate_bytecode('_temp', mapping,
- lambda bc: b'\x00\x00\x00\x00' + bc[4:])
- self.import_(mapping['_temp'], '_temp')
- with open(bc_path, 'rb') as bytecode_file:
- self.assertEqual(bytecode_file.read(4), imp.get_magic())
+ def test_non_code_marshal(self):
+ self._test_non_code_marshal()
+ # XXX ImportError when sourceless
+
+ # [bad marshal]
+ @source_util.writes_bytecode_files
+ def test_bad_marshal(self):
+ # Bad marshal data should raise a ValueError.
+ self._test_bad_marshal()
# [bad timestamp]
@source_util.writes_bytecode_files
- def test_bad_bytecode(self):
+ def test_old_timestamp(self):
# When the timestamp is older than the source, bytecode should be
# regenerated.
zeros = b'\x00\x00\x00\x00'
bytecode_file.seek(4)
self.assertEqual(bytecode_file.read(4), source_timestamp)
- # [bad marshal]
- @source_util.writes_bytecode_files
- def test_bad_marshal(self):
- # Bad marshal data should raise a ValueError.
- with source_util.create_modules('_temp') as mapping:
- bytecode_path = imp.cache_from_source(mapping['_temp'])
- source_mtime = os.path.getmtime(mapping['_temp'])
- source_timestamp = importlib._w_long(source_mtime)
- source_util.ensure_bytecode_path(bytecode_path)
- with open(bytecode_path, 'wb') as bytecode_file:
- bytecode_file.write(imp.get_magic())
- bytecode_file.write(source_timestamp)
- bytecode_file.write(b'AAAA')
- with self.assertRaises(ValueError):
- self.import_(mapping['_temp'], '_temp')
- self.assertTrue('_temp' not in sys.modules)
-
# [bytecode read-only]
@source_util.writes_bytecode_files
def test_read_only_bytecode(self):
os.chmod(bytecode_path, stat.S_IWUSR)
+class SourcelessLoaderBadBytecodeTest(BadBytecodeTest):
+
+ loader = _bootstrap._SourcelessFileLoader
+
+ def test_empty_file(self):
+ def test(name, mapping, bytecode_path):
+ with self.assertRaises(ImportError):
+ self.import_(bytecode_path, name)
+
+ self._test_empty_file(test, del_source=True)
+
+ def test_partial_magic(self):
+ def test(name, mapping, bytecode_path):
+ with self.assertRaises(ImportError):
+ self.import_(bytecode_path, name)
+ self._test_partial_magic(test, del_source=True)
+
+ def test_magic_only(self):
+ def test(name, mapping, bytecode_path):
+ with self.assertRaises(EOFError):
+ self.import_(bytecode_path, name)
+
+ self._test_magic_only(test, del_source=True)
+
+ def test_bad_magic(self):
+ def test(name, mapping, bytecode_path):
+ with self.assertRaises(ImportError):
+ self.import_(bytecode_path, name)
+
+ self._test_bad_magic(test, del_source=True)
+
+ def test_partial_timestamp(self):
+ def test(name, mapping, bytecode_path):
+ with self.assertRaises(EOFError):
+ self.import_(bytecode_path, name)
+
+ self._test_partial_timestamp(test, del_source=True)
+
+ def test_no_marshal(self):
+ self._test_no_marshal(del_source=True)
+
+ def test_non_code_marshal(self):
+ self._test_non_code_marshal(del_source=True)
+
+
def test_main():
from test.support import run_unittest
- run_unittest(SimpleTest, BadBytecodeTest)
+ run_unittest(SimpleTest,
+ SourceLoaderBadBytecodeTest,
+ SourcelessLoaderBadBytecodeTest
+ )
if __name__ == '__main__':
"""
def import_(self, root, module):
- finder = _bootstrap._PyPycFileFinder(root)
+ finder = _bootstrap._FileFinder(root,
+ _bootstrap._SourceFinderDetails(),
+ _bootstrap._SourcelessFinderDetails())
return finder.find_module(module)
def run_test(self, test, create=None, *, compile_=None, unlink=None):
# XXX This is not a blackbox test!
name = '_temp'
loader = self.run_test(name, {'{0}.__init__'.format(name), name})
- self.assertTrue('__init__' in loader._base_path)
+ self.assertTrue('__init__' in loader.get_filename(name))
def test_failure(self):
"""Test the path hook for source."""
def test_success(self):
- # XXX Only work on existing directories?
with source_util.create_modules('dummy') as mapping:
- self.assertTrue(hasattr(_bootstrap._FileFinder(mapping['.root']),
+ self.assertTrue(hasattr(_bootstrap._file_path_hook(mapping['.root']),
'find_module'))
with source_util.create_modules(self.module_name) as mapping:
with open(mapping[self.module_name], 'wb') as file:
file.write(source)
- loader = _bootstrap._PyPycFileLoader(self.module_name,
- mapping[self.module_name], False)
+ loader = _bootstrap._SourceFileLoader(self.module_name,
+ mapping[self.module_name])
return loader.load_module(self.module_name)
def create_source(self, encoding):
with source_util.create_modules(module_name) as mapping:
with open(mapping[module_name], 'wb') as file:
file.write(source)
- loader = _bootstrap._PyPycFileLoader(module_name,
- mapping[module_name], False)
+ loader = _bootstrap._SourceFileLoader(module_name,
+ mapping[module_name])
return loader.load_module(module_name)
# [cr]
import re
import tempfile
import py_compile
-from test.support import forget, make_legacy_pyc, run_unittest, verbose
+from test.support import forget, make_legacy_pyc, run_unittest, unload, verbose
from test.script_helper import (
make_pkg, make_script, make_zip_pkg, make_zip_script, temp_dir)
__import__(mod_name)
os.remove(mod_fname)
make_legacy_pyc(mod_fname)
+ unload(mod_name) # In case loader caches paths
if verbose: print("Running from compiled:", mod_name)
d2 = run_module(mod_name) # Read from bytecode
self.assertIn("x", d2)
__import__(mod_name)
os.remove(mod_fname)
make_legacy_pyc(mod_fname)
+ unload(mod_name) # In case loader caches paths
if verbose: print("Running from compiled:", pkg_name)
d2 = run_module(pkg_name) # Read from bytecode
self.assertIn("x", d2)
__import__(mod_name)
os.remove(mod_fname)
make_legacy_pyc(mod_fname)
+ unload(mod_name) # In case the loader caches paths
if verbose: print("Running from compiled:", mod_name)
d2 = run_module(mod_name, run_name=run_name) # Read from bytecode
self.assertIn("__package__", d2)
def test_main():
- run_unittest(RunModuleCodeTest, RunModuleTest, RunPathTest)
+ run_unittest(
+ RunModuleCodeTest,
+ RunModuleTest,
+ RunPathTest
+ )
if __name__ == "__main__":
test_main()
projects / python / commitdiff