python - 使用imp.load_source动态加载python模块和软件包

我正在尝试从python 2.7中的任意文件夹位置动态加载模块和软件包。它适用于裸露的单个文件模块。但是尝试装入程序包有点困难。

我能确定的最好办法是将init.py文件加载到包（文件夹）中。但是例如说我有这个：

root:
  mod.py
  package:
    __init__.py
    sub.py

如果mod.py包含：

from package import sub

使用我当前的加载代码（如下），除非我将以下内容添加到package/__init__.py，否则它将失败，表明没有名为“ sub”的软件包。

import sub

我必须想象这是因为导入包时，它通常还会扫描其中的所有其他子文件。我是否还需要手动执行此操作，或者是否有类似于imp.load_source的方法也可以处理软件包文件夹？

加载代码：

import md5
import sys
import os.path
import imp
import traceback
import glob

def load_package(path, base):
    try:
        try:
            sys.path.append(path + "/" + base)
            init = path + "/" + base + "/__init__.py"
            if not os.path.exists(init):
                return None

            fin = open(init, 'rb')

            return  (base, imp.load_source(base, init, fin))
        finally:
            try: fin.close()
            except: pass
    except ImportError, x:
        traceback.print_exc(file = sys.stderr)
        raise
    except:
        traceback.print_exc(file = sys.stderr)
        raise

def load_module(path):
    try:
        try:
            code_dir = os.path.dirname(path)
            code_file = os.path.basename(path)
            base = code_file.replace(".py", "")

            fin = open(path, 'rb')

            hash = md5.new(path).hexdigest() + "_" + code_file
            return  (base, imp.load_source(base, path, fin))
        finally:
            try: fin.close()
            except: pass
    except ImportError, x:
        traceback.print_exc(file = sys.stderr)
        raise
    except:
        traceback.print_exc(file = sys.stderr)
        raise

def load_folder(dir):
    sys.path.append(dir)
    mods = {}

    for p in glob.glob(dir + "/*/"):
        base = p.replace("\\", "").replace("/", "")
        base = base.replace(dir.replace("\\", "").replace("/", ""), "")
        package = load_package(dir, base)
        if package:
            hash, pack = package
            mods[hash] = pack

    for m in glob.glob(dir + "/*.py"):
        hash, mod = load_module(m)
        mods[hash] = mod

    return mods

最佳答案

以下代码在功能上等同于您的代码，以traceback.print_exc为模（您应让客户端处理该代码-如果未处理，则无论如何该异常最终都会打印出来）：

def _load_package(path, base):
    sys.path.append(path + "/" + base)
    init = path + "/" + base + "/__init__.py"
    if not os.path.exists(init):
        return None, None
    with open(init, 'rb') as fin:
        return base, imp.load_source(base, init, fin)

def _load_module(path):
    code_file = os.path.basename(path)
    base = code_file.replace(".py", "")
    with open(path, 'rb') as fin:
        return base, imp.load_source(base, path, fin)

def load_folder(dir):
    sys.path.append(dir)
    mods = {}
    for p in glob.glob(dir + "/*/"):
        base = p.replace("\\", "").replace("/", "")
        base = base.replace(dir.replace("\\", "").replace("/", ""), "")
        hash, pack = _load_package(dir, base)
        if hash: mods[hash] = pack
    for m in glob.glob(dir + "/*.py"): ##: /*/*.py
        hash, mod = _load_module(m)
        mods[hash] = mod
    return mods

## My added code
print('Python %s on %s' % (sys.version, sys.platform))

root_ = r'C:\Dropbox\eclipse_workspaces\python\sandbox\root'

def depyc(root, _indent=''): # deletes .pyc which will end up being imported
    if not _indent: print '\nListing', root
    for p in os.listdir(root):
        name = _indent + p
        abspath = os.path.join(root, p)
        if os.path.isdir(abspath):
            print name + ':'
            depyc(abspath, _indent=_indent + '  ')
        else:
            name_ = name[-4:]
            if name_ == '.pyc':
                os.remove(abspath)
                continue
            print name
    if not _indent: print

depyc(root_)
load_folder(root_)

印刷品：

Python 2.7.10 (default, May 23 2015, 09:40:32) [MSC v.1500 32 bit (Intel)] on win32

Listing C:\Dropbox\eclipse_workspaces\python\sandbox\root
mod.py
package:
  sub.py
  __init__.py

C:\Dropbox\eclipse_workspaces\python\sandbox\root/package/__init__.py imported!
C:\Dropbox\eclipse_workspaces\python\sandbox\root\mod.py imported!

mod.py，sub.py和__init__.py仅包含

print(__file__ + u' imported!')

现在将mod.py修改为：

from package import sub
print(__file__ + u' imported!')

我们确实得到：

Listing....

C:\Dropbox\eclipse_workspaces\python\sandbox\root/package/__init__.py imported! <### this may move around ###>
Traceback (most recent call last):
  File "C:/Users/MrD/.PyCharm40/config/scratches/load_folder.py", line 57, in <module>
    load_folder(root_)
  File "C:/Users/MrD/.PyCharm40/config/scratches/load_folder.py", line 31, in load_folder
    hash, mod = _load_module(m)
  File "C:/Users/MrD/.PyCharm40/config/scratches/load_folder.py", line 20, in _load_module
    return base, imp.load_source(base, path, fin)
  File "C:\Dropbox\eclipse_workspaces\python\sandbox\root\mod.py", line 1, in <module>
    from package import sub
ImportError: cannot import name sub

请注意错误是“无法导入名称子”，而不是“没有名为”子“的程序包”。那为什么不能呢？

修改__init__.py：

# package/__init__.py
print(__file__ + u' imported!')

print '__name__', '->', __name__
print '__package__', '->', __package__
print '__path__', '->', __path__

印刷品：

Listing...

C:\Dropbox\eclipse_workspaces\python\sandbox\root/package/__init__.py imported! <### not really ###>
__name__ -> package
__package__ -> None
__path__ ->
Traceback (most recent call last):
  File "C:/Users/MrD/.PyCharm40/config/scratches/load_folder.py", line 59, in <module>
    load_folder(root_)
  File "C:/Users/MrD/.PyCharm40/config/scratches/load_folder.py", line 30, in load_folder
    hash, pack = _load_package(dir, base)
  File "C:/Users/MrD/.PyCharm40/config/scratches/load_folder.py", line 14, in _load_package
    init = imp.load_source(base, init, fin)
  File "C:\Dropbox\eclipse_workspaces\python\sandbox\root/package/__init__.py", line 5, in <module>
    print '__path__', '->', __path__
NameError: name '__path__' is not defined

直接导入时将打印：

>>> sys.path.extend([r'C:\Dropbox\eclipse_workspaces\python\sandbox\root'])
>>> import package
C:\Dropbox\eclipse_workspaces\python\sandbox\root\package\__init__.py imported!
__name__ -> package
__package__ -> None
__path__ -> ['C:\\Dropbox\\eclipse_workspaces\\python\\sandbox\\root\\package']

因此，将_load_package修改为：

def _load_package(path, base):
    pkgDir = os.path.abspath(os.path.join(path, base))
    init = os.path.join(pkgDir, "__init__.py")
    if not os.path.exists(init):
        return None, None
    file, pathname, description = imp.find_module(base, [path])
    print file, pathname, description # None, pkgDir, ('', '', 5)
    pack = sys.modules.get(base, None) # load_module will reload - yak!
    if pack is None:
        sys.modules[base] = pack = imp.load_module(base, file, pathname, description)
    return base, pack

按照以下方式解决：

...
    if pack is None:
        sys.modules[base] = pack = imp.load_module(base, None, '', description)
        pack.__path__ = [pkgDir]

或使用原始代码：

with open(init, 'rb') as fin:
    source = imp.load_source(base, init, fin)
    source.__path__ = path + "/" + base
    return base, source

因此，正在发生的事情是，程序包依靠其__path __属性来正常运行。



继续对此进行黑客攻击，并提出：

import sys
import os.path
import imp

def _load_(root, name):
    file_object, pathname, description = imp.find_module(name, [root])
    pack = sys.modules.get(name, None)
    try:
        if pack is None:
            pack = imp.load_module(name, file_object, pathname, description)
        else:
            print 'In cache', pack
    finally:
        if file_object is not None: file_object.close()
    return name, pack

def load_folder(root):
    # sys.path.append(root)
    mods = {}
    paths = [(item, os.path.join(root, item)) for item in os.listdir(root)]
    packages = filter(lambda path_tuple: os.path.exists(
        os.path.join((path_tuple[1]), "__init__.py")), paths)
    py_files = filter(lambda path_tuple: path_tuple[0][-3:] == '.py', paths)
    del paths
    # first import packages as in original - modules may import from them
    for path, _abspath in packages:
        print 'Importing', _abspath
        key, mod = _load_(root, name=path) # will use pyc if available!
        mods[key] = mod
    # then modules
    for path, _abspath in py_files:
        print 'Importing', _abspath
        key, mod = _load_(root, name=path[:-3])
        mods[key] = mod
    return mods

我合并了程序包和模块，加载了删除imp.load_source的代码（少了一个棘手的功能），而是依赖imp.load_module。我不会直接与sys.path混淆，因为imp.load_module will reload [!]我会检查sys.modules缓存。返回的mods字典完全未经测试-您必须以某种方式实现哈希（_abspath应该足够）。

运行方式：

def depyc(root, rmpyc, _indent=''):
    if not _indent: print '\nListing', root
    for p in os.listdir(root):
        name = _indent + p
        abspath = os.path.join(root, p)
        if os.path.isdir(abspath):
            print name + ':'
            depyc(abspath, rmpyc, _indent=_indent + '  ')
        else:
            if rmpyc and name[-4:] == '.pyc':
                os.remove(abspath)
                continue
            print name
    if not _indent: print

## Run ##
print('Python %s on %s' % (sys.version, sys.platform))
root_ = os.path.join(os.getcwdu(), u'root')
depyc(root_, False) # False will end up importing the pyc files !
load_folder(root_)

测试各种场景-

带有示例root/目录的代码为here

关于python - 使用imp.load_source动态加载python模块和软件包，我们在Stack Overflow上找到一个类似的问题：https://stackoverflow.com/questions/30761518/