5 ImportManager Manage the import process
7 Importer Base class for replacing standard import functions
8 BuiltinImporter Emulate the import mechanism for builtin and frozen modules
13 # note: avoid importing non-builtin modules
14 import imp ### not available in JPython?
18 # for the DirectoryImporter
22 __all__ = ["ImportManager","Importer","BuiltinImporter"]
24 _StringType = type('')
25 _ModuleType = type(sys) ### doesn't work in JPython...
28 "Manage the import process."
30 def install(self, namespace=vars(__builtin__)):
31 "Install this ImportManager into the specified namespace."
33 if isinstance(namespace, _ModuleType):
34 namespace = vars(namespace)
36 # Note: we have no notion of "chaining"
38 # Record the previous import hook, then install our own.
39 self.previous_importer = namespace['__import__']
40 self.namespace = namespace
41 namespace['__import__'] = self._import_hook
44 #namespace['reload'] = self._reload_hook
47 "Restore the previous import mechanism."
48 self.namespace['__import__'] = self.previous_importer
50 def add_suffix(self, suffix, importFunc):
51 assert callable(importFunc)
52 self.fs_imp.add_suffix(suffix, importFunc)
54 ######################################################################
59 clsFilesystemImporter = None
61 def __init__(self, fs_imp=None):
62 # we're definitely going to be importing something in the future,
63 # so let's just load the OS-related facilities.
67 # This is the Importer that we use for grabbing stuff from the
68 # filesystem. It defines one more method (import_from_dir) for our use.
70 cls = self.clsFilesystemImporter or _FilesystemImporter
74 # Initialize the set of suffixes that we recognize and import.
75 # The default will import dynamic-load modules first, followed by
76 # .py files (or a .py file's cached bytecode)
77 for desc in imp.get_suffixes():
78 if desc[2] == imp.C_EXTENSION:
79 self.add_suffix(desc[0],
80 DynLoadSuffixImporter(desc).import_file)
81 self.add_suffix('.py', py_suffix_importer)
83 def _import_hook(self, fqname, globals=None, locals=None, fromlist=None):
84 """Python calls this hook to locate and import a module."""
86 parts = fqname.split('.')
88 # determine the context of this import
89 parent = self._determine_import_context(globals)
91 # if there is a parent, then its importer should manage this import
93 module = parent.__importer__._do_import(parent, parts, fromlist)
97 # has the top module already been imported?
99 top_module = sys.modules[parts[0]]
102 # look for the topmost module
103 top_module = self._import_top_module(parts[0])
105 # the topmost module wasn't found at all.
106 raise ImportError, 'No module named ' + fqname
108 # fast-path simple imports
113 if not top_module.__dict__.get('__ispkg__'):
114 # __ispkg__ isn't defined (the module was not imported by us),
117 # In the former case, there is no way that we could import
118 # sub-modules that occur in the fromlist (but we can't raise an
119 # error because it may just be names) because we don't know how
120 # to deal with packages that were imported by other systems.
122 # In the latter case (__ispkg__ == 0), there can't be any sub-
123 # modules present, so we can just return.
125 # In both cases, since len(parts) == 1, the top_module is also
126 # the "bottom" which is the defined return when a fromlist
130 importer = top_module.__dict__.get('__importer__')
132 return importer._finish_import(top_module, parts[1:], fromlist)
134 # Grrr, some people "import os.path" or do "from os.path import ..."
135 if len(parts) == 2 and hasattr(top_module, parts[1]):
137 return getattr(top_module, parts[1])
141 # If the importer does not exist, then we have to bail. A missing
142 # importer means that something else imported the module, and we have
143 # no knowledge of how to get sub-modules out of the thing.
144 raise ImportError, 'No module named ' + fqname
146 def _determine_import_context(self, globals):
147 """Returns the context in which a module should be imported.
149 The context could be a loaded (package) module and the imported module
150 will be looked for within that package. The context could also be None,
151 meaning there is no context -- the module should be looked for as a
155 if not globals or not globals.get('__importer__'):
156 # globals does not refer to one of our modules or packages. That
157 # implies there is no relative import context (as far as we are
158 # concerned), and it should just pick it off the standard path.
161 # The globals refer to a module or package of ours. It will define
162 # the context of the new import. Get the module/package fqname.
163 parent_fqname = globals['__name__']
165 # if a package is performing the import, then return itself (imports
166 # refer to pkg contents)
167 if globals['__ispkg__']:
168 parent = sys.modules[parent_fqname]
169 assert globals is parent.__dict__
172 i = parent_fqname.rfind('.')
174 # a module outside of a package has no particular import context
178 # if a module in a package is performing the import, then return the
179 # package (imports refer to siblings)
180 parent_fqname = parent_fqname[:i]
181 parent = sys.modules[parent_fqname]
182 assert parent.__name__ == parent_fqname
185 def _import_top_module(self, name):
186 # scan sys.path looking for a location in the filesystem that contains
187 # the module, or an Importer object that can import the module.
188 for item in sys.path:
189 if isinstance(item, _StringType):
190 module = self.fs_imp.import_from_dir(item, name)
192 module = item.import_top(name)
197 def _reload_hook(self, module):
198 "Python calls this hook to reload a module."
200 # reloading of a module may or may not be possible (depending on the
201 # importer), but at least we can validate that it's ours to reload
202 importer = module.__dict__.get('__importer__')
204 ### oops. now what...
207 # okay. it is using the imputil system, and we must delegate it, but
208 # we don't know what to do (yet)
209 ### we should blast the module dict and do another get_code(). need to
210 ### flesh this out and add proper docco...
211 raise SystemError, "reload not yet implemented"
215 "Base class for replacing standard import functions."
217 def import_top(self, name):
218 "Import a top-level module."
219 return self._import_one(None, name, name)
221 ######################################################################
225 def _finish_import(self, top, parts, fromlist):
226 # if "a.b.c" was provided, then load the ".b.c" portion down from
227 # below the top-level module.
228 bottom = self._load_tail(top, parts)
230 # if the form is "import a.b.c", then return "a"
232 # no fromlist: return the top of the import tree
235 # the top module was imported by self.
237 # this means that the bottom module was also imported by self (just
238 # now, or in the past and we fetched it from sys.modules).
240 # since we imported/handled the bottom module, this means that we can
241 # also handle its fromlist (and reliably use __ispkg__).
243 # if the bottom node is a package, then (potentially) import some
246 # note: if it is not a package, then "fromlist" refers to names in
247 # the bottom module rather than modules.
248 # note: for a mix of names and modules in the fromlist, we will
249 # import all modules and insert those into the namespace of
250 # the package module. Python will pick up all fromlist names
251 # from the bottom (package) module; some will be modules that
252 # we imported and stored in the namespace, others are expected
253 # to be present already.
255 self._import_fromlist(bottom, fromlist)
257 # if the form is "from a.b import c, d" then return "b"
260 def _import_one(self, parent, modname, fqname):
261 "Import a single module."
263 # has the module already been imported?
265 return sys.modules[fqname]
269 # load the module's code, or fetch the module itself
270 result = self.get_code(parent, modname, fqname)
274 module = self._process_result(result, fqname)
276 # insert the module into its parent
278 setattr(parent, modname, module)
281 def _process_result(self, (ispkg, code, values), fqname):
282 # did get_code() return an actual module? (rather than a code object)
283 is_module = isinstance(code, _ModuleType)
285 # use the returned module, or create a new one to exec code into
289 module = imp.new_module(fqname)
291 ### record packages a bit differently??
292 module.__importer__ = self
293 module.__ispkg__ = ispkg
295 # insert additional values into the module (before executing the code)
296 module.__dict__.update(values)
298 # the module is almost ready... make it visible
299 sys.modules[fqname] = module
301 # execute the code within the module's namespace
304 exec code in module.__dict__
306 if fqname in sys.modules:
307 del sys.modules[fqname]
310 # fetch from sys.modules instead of returning module directly.
311 # also make module's __name__ agree with fqname, in case
312 # the "exec code in module.__dict__" played games on us.
313 module = sys.modules[fqname]
314 module.__name__ = fqname
317 def _load_tail(self, m, parts):
318 """Import the rest of the modules, down from the top-level module.
320 Returns the last module in the dotted list of modules.
323 fqname = "%s.%s" % (m.__name__, part)
324 m = self._import_one(m, part, fqname)
326 raise ImportError, "No module named " + fqname
329 def _import_fromlist(self, package, fromlist):
330 'Import any sub-modules in the "from" list.'
332 # if '*' is present in the fromlist, then look for the '__all__'
333 # variable to find additional items (modules) to import.
335 fromlist = list(fromlist) + \
336 list(package.__dict__.get('__all__', []))
339 # if the name is already present, then don't try to import it (it
340 # might not be a module!).
341 if sub != '*' and not hasattr(package, sub):
342 subname = "%s.%s" % (package.__name__, sub)
343 submod = self._import_one(package, sub, subname)
345 raise ImportError, "cannot import name " + subname
347 def _do_import(self, parent, parts, fromlist):
348 """Attempt to import the module relative to parent.
350 This method is used when the import context specifies that <self>
351 imported the parent module.
354 top_fqname = parent.__name__ + '.' + top_name
355 top_module = self._import_one(parent, top_name, top_fqname)
357 # this importer and parent could not find the module (relatively)
360 return self._finish_import(top_module, parts[1:], fromlist)
362 ######################################################################
364 # METHODS TO OVERRIDE
366 def get_code(self, parent, modname, fqname):
367 """Find and retrieve the code for the given module.
369 parent specifies a parent module to define a context for importing. It
370 may be None, indicating no particular context for the search.
372 modname specifies a single module (not dotted) within the parent.
374 fqname specifies the fully-qualified module name. This is a
375 (potentially) dotted name from the "root" of the module namespace
377 If there is no parent, then modname==fqname.
379 This method should return None, or a 3-tuple.
381 * If the module was not found, then None should be returned.
383 * The first item of the 2- or 3-tuple should be the integer 0 or 1,
384 specifying whether the module that was found is a package or not.
386 * The second item is the code object for the module (it will be
387 executed within the new module's namespace). This item can also
388 be a fully-loaded module object (e.g. loaded from a shared lib).
390 * The third item is a dictionary of name/value pairs that will be
391 inserted into new module before the code object is executed. This
392 is provided in case the module's code expects certain values (such
393 as where the module was found). When the second item is a module
394 object, then these names/values will be inserted *after* the module
395 has been loaded/initialized.
397 raise RuntimeError, "get_code not implemented"
400 ######################################################################
402 # Some handy stuff for the Importers
405 # byte-compiled file suffix character
406 _suffix_char = __debug__ and 'c' or 'o'
408 # byte-compiled file suffix
409 _suffix = '.py' + _suffix_char
411 def _compile(pathname, timestamp):
412 """Compile (and cache) a Python source file.
414 The file specified by <pathname> is compiled to a code object and
417 Presuming the appropriate privileges exist, the bytecodes will be
418 saved back to the filesystem for future imports. The source file's
419 modification timestamp must be provided as a Long value.
421 codestring = open(pathname, 'rU').read()
422 if codestring and codestring[-1] != '\n':
423 codestring = codestring + '\n'
424 code = __builtin__.compile(codestring, pathname, 'exec')
426 # try to cache the compiled code
428 f = open(pathname + _suffix_char, 'wb')
433 f.write(struct.pack('<I', timestamp))
434 marshal.dump(code, f)
437 f.write(imp.get_magic())
442 _os_stat = _os_path_join = None
444 "Set up 'os' module replacement functions for use during import bootstrap."
446 names = sys.builtin_module_names
451 from posix import stat
472 raise ImportError, 'no os specific module found'
475 def join(a, b, sep=sep):
479 if lastchar == '/' or lastchar == sep:
489 def _os_path_isdir(pathname):
490 "Local replacement for os.path.isdir()."
492 s = _os_stat(pathname)
495 return (s.st_mode & 0170000) == 0040000
497 def _timestamp(pathname):
498 "Return the file modification time as a Long."
500 s = _os_stat(pathname)
503 return long(s.st_mtime)
506 ######################################################################
508 # Emulate the import mechanism for builtin and frozen modules
510 class BuiltinImporter(Importer):
511 def get_code(self, parent, modname, fqname):
513 # these modules definitely do not occur within a package context
516 # look for the module
517 if imp.is_builtin(modname):
519 elif imp.is_frozen(modname):
525 # got it. now load and return it.
526 module = imp.load_module(modname, None, modname, ('', '', type))
527 return 0, module, { }
530 ######################################################################
532 # Internal importer used for importing from the filesystem
534 class _FilesystemImporter(Importer):
538 def add_suffix(self, suffix, importFunc):
539 assert callable(importFunc)
540 self.suffixes.append((suffix, importFunc))
542 def import_from_dir(self, dir, fqname):
543 result = self._import_pathname(_os_path_join(dir, fqname), fqname)
545 return self._process_result(result, fqname)
548 def get_code(self, parent, modname, fqname):
549 # This importer is never used with an empty parent. Its existence is
550 # private to the ImportManager. The ImportManager uses the
551 # import_from_dir() method to import top-level modules/packages.
552 # This method is only used when we look for a module within a package.
555 for submodule_path in parent.__path__:
556 code = self._import_pathname(_os_path_join(submodule_path, modname), fqname)
559 return self._import_pathname(_os_path_join(parent.__pkgdir__, modname),
562 def _import_pathname(self, pathname, fqname):
563 if _os_path_isdir(pathname):
564 result = self._import_pathname(_os_path_join(pathname, '__init__'),
568 values['__pkgdir__'] = pathname
569 values['__path__'] = [ pathname ]
570 return 1, result[1], values
573 for suffix, importFunc in self.suffixes:
574 filename = pathname + suffix
576 finfo = _os_stat(filename)
580 return importFunc(filename, finfo, fqname)
583 ######################################################################
585 # SUFFIX-BASED IMPORTERS
588 def py_suffix_importer(filename, finfo, fqname):
589 file = filename[:-3] + _suffix
590 t_py = long(finfo[8])
591 t_pyc = _timestamp(file)
594 if t_pyc is not None and t_pyc >= t_py:
596 if f.read(4) == imp.get_magic():
597 t = struct.unpack('<I', f.read(4))[0]
599 code = marshal.load(f)
603 code = _compile(file, t_py)
605 return 0, code, { '__file__' : file }
607 class DynLoadSuffixImporter:
608 def __init__(self, desc):
611 def import_file(self, filename, finfo, fqname):
612 fp = open(filename, self.desc[1])
613 module = imp.load_module(fqname, fp, filename, self.desc)
614 module.__file__ = filename
615 return 0, module, { }
618 ######################################################################
620 def _print_importers():
621 items = sys.modules.items()
623 for name, module in items:
625 print name, module.__dict__.get('__importer__', '-- no importer')
627 print name, '-- non-existent module'
630 ImportManager().install()
631 sys.path.insert(0, BuiltinImporter())
633 ######################################################################
639 # type(sys) is not a module in JPython. what to use instead?
640 # imp.C_EXTENSION is not in JPython. same for get_suffixes and new_module
644 # sys.modules['foo'] = sys
646 # ---- standard import mechanism
649 # <module 'sys' (built-in)>
651 # ---- revamped import mechanism
653 # >>> imputil._test_revamp()
656 # <module 'foo' from 'foo.py'>
660 # should BuiltinImporter exist in sys.path or hard-wired in ImportManager?
661 # need __path__ processing
663 # move chaining to a subclass [gjs: it's been nuked]
664 # deinstall should be possible
665 # query mechanism needed: is a specific Importer installed?
666 # py/pyc/pyo piping hooks to filter/process these files
668 # distutils importer hooked to list of standard Internet repositories
669 # module->file location mapper to speed FS-based imports
671 # keep chaining so that it can play nice with other import hooks
674 # push MAL's mapper into sys.path[0] as a cache (hard-coded for apps)
677 # need to change sys.* references for rexec environs
678 # need hook for MAL's walk-me-up import strategy, or Tim's absolute strategy
679 # watch out for sys.modules[...] is None
680 # flag to force absolute imports? (speeds _determine_import_context and
681 # checking for a relative module)
682 # insert names of archives into sys.path (see quote below)
683 # note: reload does NOT blast module dict
684 # shift import mechanisms and policies around; provide for hooks, overrides
686 # add get_source stuff
687 # get_topcode and get_subcode
688 # CRLF handling in _compile
689 # race condition in _compile
690 # refactoring of os.py to deal with _os_bootstrap problem
691 # any special handling to do for importing a module with a SyntaxError?
692 # (e.g. clean up the traceback)
693 # implement "domain" for path-type functionality using pkg namespace
694 # (rather than FS-names like __path__)
695 # don't use the word "private"... maybe "internal"
698 # Guido's comments on sys.path caching:
700 # We could cache this in a dictionary: the ImportManager can have a
701 # cache dict mapping pathnames to importer objects, and a separate
702 # method for coming up with an importer given a pathname that's not yet
703 # in the cache. The method should do a stat and/or look at the
704 # extension to decide which importer class to use; you can register new
705 # importer classes by registering a suffix or a Boolean function, plus a
706 # class. If you register a new importer class, the cache is zapped.
707 # The cache is independent from sys.path (but maintained per
708 # ImportManager instance) so that rearrangements of sys.path do the
709 # right thing. If a path is dropped from sys.path the corresponding
710 # cache entry is simply no longer used.
712 # My/Guido's comments on factoring ImportManager and Importer:
714 # > However, we still have a tension occurring here:
716 # > 1) implementing policy in ImportManager assists in single-point policy
717 # > changes for app/rexec situations
718 # > 2) implementing policy in Importer assists in package-private policy
719 # > changes for normal, operating conditions
721 # > I'll see if I can sort out a way to do this. Maybe the Importer class will
722 # > implement the methods (which can be overridden to change policy) by
723 # > delegating to ImportManager.
725 # Maybe also think about what kind of policies an Importer would be
726 # likely to want to change. I have a feeling that a lot of the code
727 # there is actually not so much policy but a *necessity* to get things
728 # working given the calling conventions for the __import__ hook: whether
729 # to return the head or tail of a dotted name, or when to do the "finish