Import Deep Dive

Presented at EuroPython 2015
https://ep2015.europython.eu/conference/talks/import-deep-dive

1. Import Deep Dive Petr Viktorin @encukou encukou@gmail.com encukou.cz EuroPython 2015
2. import random
3. Prior art David Beazley — Modules and Packages: Live and Let Die! http://pyvideo.org/video/3387
4. Photo © Paul Hermans, https://en.wikipedia.org/wiki/File:Tunnelaquarium_14-05-2009_15-54-09.JPG
5. import random
6. import random random = __import__('random')
7. import random random = __import__('random') from ..spam import foo
8. import random random = __import__('random') from ..spam import foo foo = __import__('spam', globals(), locals(), ['foo'], 2).foo https://docs.python.org/3/library/functions.html#__import__
9. import random random = __import__('random')
10. import random random = __import__('random') The Import Machinery
11. import random random = __import__('random') The Import Machinery importlib.import_module('random')
12. import random The Import Machinery importlib.import_module('random')
13. The Import Machinery Photo © Les Chatfield, https://www.flickr.com/photos/elsie/8229790
14. The Import Machinery Photo © Les Chatfield, https://www.flickr.com/photos/elsie/8229790 sans: • Locking • Caching • Error handling • Pythons older than 3.4 • Backwards compatibility shims
15. importlib.import_module('random')
16. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module = sys.modules[name] return module >>> import random >>> old = random >>> import random >>> random is old True
17. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module = sys.modules[name] return module >>> import random >>> old = random >>> import random >>> random is old True
18. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module = sys.modules[name] return module >>> import random >>> old = random >>> import random >>> random is old True
19. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module = sys.modules[name] return module >>> import random >>> old = random >>> del sys.modules['random'] >>> import random >>> random is old False >>> (random.sample is ... old.sample) False >>> sys.modules['impostor'] = 'Not a module, is it?' >>> import impostor >>> impostor[:12] 'Not a module'
20. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module = sys.modules[name] return module >>> import random >>> old = random >>> del sys.modules['random'] >>> import random >>> random is old False >>> (random.sample is ... old.sample) False >>> sys.modules['impostor'] = 'Not a module, is it?' >>> import impostor >>> impostor[:12] 'Not a module'
21. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module = sys.modules[name] return module >>> sys.path ['', '/usr/lib64/python34.zip', '/usr/lib64/python3.4', ...] >>> importlib.util.find_spec('antigravity') ModuleSpec(name='antigravity', loader=<_frozen_importlib.SourceFileLoader ...>, origin='/usr/lib64/python3.4/antigravity.py') >>> import io >>> io.__spec__ ModuleSpec(name='io', loader=<_frozen_importlib.SourceFileLoader ...>, origin='/usr/lib64/python3.4/io.py')
22. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module = sys.modules[name] return module >>> sys.path ['', '/usr/lib64/python34.zip', '/usr/lib64/python3.4', ...] >>> importlib.util.find_spec('antigravity') ModuleSpec(name='antigravity', loader=<_frozen_importlib.SourceFileLoader ...>, origin='/usr/lib64/python3.4/antigravity.py') >>> import io >>> io.__spec__ ModuleSpec(name='io', loader=<_frozen_importlib.SourceFileLoader ...>, origin='/usr/lib64/python3.4/io.py')
23. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module = sys.modules[name] return module >>> sys.path ['', '/usr/lib64/python34.zip', '/usr/lib64/python3.4', ...] >>> importlib.util.find_spec('antigravity') ModuleSpec(name='antigravity', loader=<_frozen_importlib.SourceFileLoader ...>, origin='/usr/lib64/python3.4/antigravity.py') >>> import io >>> io.__spec__ ModuleSpec(name='io', loader=<_frozen_importlib.SourceFileLoader ...>, origin='/usr/lib64/python3.4/io.py')
24. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module = sys.modules[name] return module >>> sys.path ['', '/usr/lib64/python34.zip', '/usr/lib64/python3.4', ...] >>> importlib.util.find_spec('antigravity') ModuleSpec(name='antigravity', loader=<_frozen_importlib.SourceFileLoader ...>, origin='/usr/lib64/python3.4/antigravity.py') >>> import io >>> io.__spec__ ModuleSpec(name='io', loader=<_frozen_importlib.SourceFileLoader ...>, origin='/usr/lib64/python3.4/io.py')
25. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module = sys.modules[name] return module • Put module in sys.modules • Initialize the module
26. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module = sys.modules[name] return module foo.py: import bar bar.do_bar() def do_foo(): print('Fooing!') bar.py: import foo foo.do_foo() def do_bar(): print('Barring!') • Put module in sys.modules • Initialize the module
27. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module = sys.modules[name] return module foo.py: import bar bar.do_bar() def do_foo(): print('Fooing!') bar.py: import foo foo.do_foo() def do_bar(): print('Barring!') • Put module in sys.modules • Initialize the module
28. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module = sys.modules[name] return module foo.py: import bar bar.do_bar() def do_foo(): print('Fooing!') bar.py: import foo foo.do_foo() def do_bar(): print('Barring!') • Put module in sys.modules • Initialize the module
29. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module = sys.modules[name] return module foo.py: import bar bar.do_bar() def do_foo(): print('Fooing!') bar.py: import foo foo.do_foo() def do_bar(): print('Barring!') • Put module in sys.modules • Initialize the module
30. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module = sys.modules[name] return module foo.py: import bar bar.do_bar() def do_foo(): print('Fooing!') bar.py: import foo foo.do_foo() def do_bar(): print('Barring!') • Put module in sys.modules • Initialize the module
31. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module = sys.modules[name] return module
32. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module = sys.modules[name] return module Filename Module name random.py random urllib/ __init__.py urllib parse.py urllib.parse request.py urllib.request response.py urllib.response Top-level module Package Parent of the below Submodule
33. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module = sys.modules[name] return module Filename Module name random.py random urllib/ __init__.py urllib parse.py urllib.parse request.py urllib.request response.py urllib.response Top-level module Package Parent of the below Submodule
34. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module = sys.modules[name] return module import urllib.parse >>> urllib.__path__ ['/usr/lib64/python3.4/urllib'] Filename Module name random.py random urllib/ __init__.py urllib parse.py urllib.parse request.py urllib.request response.py urllib.response Top-level module Package Parent of the below Submodule
35. import_module(name): sys.modules[name]? return it submodules: load parent module sys.modules[name]? return it spec = find_spec(name, path) load(spec) module = sys.modules[name] submodules: set module as attribute of parent return module import urllib.parse >>> urllib.__path__ ['/usr/lib64/python3.4/urllib'] Filename Module name random.py random urllib/ __init__.py urllib parse.py urllib.parse request.py urllib.request response.py urllib.response Top-level module Package Parent of the below Submodule
36. import_module(name): sys.modules[name]? return it submodules: load parent module sys.modules[name]? return it spec = find_spec(name, path) load(spec) module = sys.modules[name] submodules: set module as attribute of parent return module • Put module in sys.modules • Initialize the module
37. import_module(name): sys.modules[name]? return it submodules: load parent module sys.modules[name]? return it spec = find_spec(name, path) load(spec) module = sys.modules[name] submodules: set module as attribute of parent return module foo/__init__.py: from foo import main from foo import consts foo/main.py: import foo use(foo.consts.value) foo/consts.py: value = 42 • Put module in sys.modules • Initialize the module
38. import_module(name): sys.modules[name]? return it submodules: load parent module sys.modules[name]? return it spec = find_spec(name, path) load(spec) module = sys.modules[name] submodules: set module as attribute of parent return module foo/__init__.py: from foo import main from foo import consts foo/main.py: import foo use(foo.consts.value) foo/consts.py: value = 42 • Put module in sys.modules • Initialize the module
39. import_module(name): sys.modules[name]? return it submodules: load parent module sys.modules[name]? return it spec = find_spec(name, path) load(spec) module = sys.modules[name] submodules: set module as attribute of parent return module foo/__init__.py: from foo import main from foo import consts foo/main.py: import foo use(foo.consts.value) foo/consts.py: value = 42 • Put module in sys.modules • Initialize the module
40. import_module(name): sys.modules[name]? return it submodules: load parent module sys.modules[name]? return it spec = find_spec(name, path) load(spec) module = sys.modules[name] submodules: set module as attribute of parent return module foo/__init__.py: from foo import main from foo import consts foo/main.py: import foo use(foo.consts.value) foo/consts.py: value = 42 • Put module in sys.modules • Initialize the module
41. import_module(name): sys.modules[name]? return it submodules: load parent module sys.modules[name]? return it spec = find_spec(name, path) load(spec) module = sys.modules[name] submodules: set module as attribute of parent return module foo/__init__.py: from foo import main from foo import consts foo/main.py: import foo use(foo.consts.value) foo/consts.py: value = 42 • Put module in sys.modules • Initialize the module
42. import_module(name): sys.modules[name]? return it submodules: load parent module sys.modules[name]? return it spec = find_spec(name, path) load(spec) module = sys.modules[name] submodules: set module as attribute of parent return module foo/__init__.py: from foo import main from foo import consts foo/main.py: import foo use(foo.consts.value) foo/consts.py: value = 42 • Put module in sys.modules • Initialize the module
43. import_module(name): sys.modules[name]? return it submodules: load parent module sys.modules[name]? return it spec = find_spec(name, path) load(spec) module = sys.modules[name] submodules: set module as attribute of parent return module foo/__init__.py: from foo import main from foo import consts foo/main.py: import foo use(foo.consts.value) foo/consts.py: value = 42 • Put module in sys.modules • Initialize the module
44. __init__ should: import from submodules set __all__ nothing else submodules should: not import directly from __init__ not have internal import cycles
45. import_module(name): sys.modules[name]? return it submodules: load parent module sys.modules[name]? return it spec = find_spec(name, path) load(spec) module = sys.modules[name] submodules: set module as attribute of parent return module ?
46. Where's the module? >>> import random >>> random <module 'random' from '/usr/lib64/python3.4/random.py'> >>> random.__file__ '/usr/lib64/python3.4/random.py' >>> import sys >>> sys <module 'sys' (built-in)> >>> sys.__file__ Traceback (most recent call last): File "<stdin>", line 1, in <module> AttributeError: 'module' object has no attribute '__file__' /usr/bin/python3
47. Where's the module? >>> import random >>> random <module 'random' from '/usr/lib64/python3.4/random.py'> >>> random.__file__ '/usr/lib64/python3.4/random.py' >>> import sys >>> sys <module 'sys' (built-in)> >>> sys.__file__ Traceback (most recent call last): File "<stdin>", line 1, in <module> AttributeError: 'module' object has no attribute '__file__' /usr/bin/python3
48. Where's the module? >>> import random >>> random <module 'random' from '/usr/lib64/python3.4/random.py'> >>> random.__file__ '/usr/lib64/python3.4/random.py' >>> import sys >>> sys <module 'sys' (built-in)> >>> sys.__file__ Traceback (most recent call last): File "<stdin>", line 1, in <module> AttributeError: 'module' object has no attribute '__file__' /usr/bin/python3
49. Where's the module? >>> import random >>> random <module 'random' from '/usr/lib64/python3.4/random.py'> >>> random.__file__ '/usr/lib64/python3.4/random.py' >>> import sys >>> sys <module 'sys' (built-in)> >>> sys.__file__ Traceback (most recent call last): File "<stdin>", line 1, in <module> AttributeError: 'module' object has no attribute '__file__' /usr/bin/python3
50. Where's the module? >>> import random >>> random <module 'random' from '/usr/lib64/python3.4/random.py'> >>> random.__file__ '/usr/lib64/python3.4/random.py' >>> import sys >>> sys <module 'sys' (built-in)> >>> sys.__file__ Traceback (most recent call last): File "<stdin>", line 1, in <module> AttributeError: 'module' object has no attribute '__file__' /usr/bin/python3
51. Where's the module? >>> import random >>> random <module 'random' from '/usr/lib64/python3.4/random.py'> >>> random.__file__ '/usr/lib64/python3.4/random.py' >>> import sys >>> sys <module 'sys' (built-in)> >>> sys.__file__ Traceback (most recent call last): File "<stdin>", line 1, in <module> AttributeError: 'module' object has no attribute '__file__' /usr/bin/python3
52. Where's the module? >>> import random >>> random <module 'random' from '/usr/lib64/python3.4/random.py'> >>> random.__file__ '/usr/lib64/python3.4/random.py' >>> import sys >>> sys <module 'sys' (built-in)> >>> sys.__file__ Traceback (most recent call last): File "<stdin>", line 1, in <module> AttributeError: 'module' object has no attribute '__file__' /usr/bin/python3
53. Photo © Allentchang, https://commons.wikimedia.org/wiki/File:Fish_in_Okinawa_Churaumi_Aquarium.JPG C Python Compiled in Built-in Frozen From File Extension Source sys _frozen_importlib math random
54. Photo © Allentchang, https://commons.wikimedia.org/wiki/File:Fish_in_Okinawa_Churaumi_Aquarium.JPG C Python Compiled in Built-in Frozen From File Extension Source sys _frozen_importlib math random
55. Photo © Allentchang, https://commons.wikimedia.org/wiki/File:Fish_in_Okinawa_Churaumi_Aquarium.JPG C Python Compiled in Built-in Frozen From File Extension Source sys _frozen_importlib math random
56. >>> sys.meta_path [<BuiltinImporter>, <FrozenImporter>, <PathFinder>]
57. find_spec(name, path): for finder in sys.meta_path: call its find_spec(name, path) return spec if successful >>> sys.meta_path [<BuiltinImporter>, <FrozenImporter>, <PathFinder>]
58. find_spec(name, path): for finder in sys.meta_path: call its find_spec(name, path) return spec if successful >>> sys.meta_path [<BuiltinImporter>, <FrozenImporter>, <PathFinder>] >>> sys.path ['', '/usr/lib64/python34.zip', '/usr/lib/python3.4', ...]
59. find_spec(name, path): for finder in sys.meta_path: call its find_spec(name, path) return spec if successful >>> sys.meta_path [<BuiltinImporter>, <FrozenImporter>, <PathFinder>] >>> sys.path_hooks [<zipimporter>, <path_hook_for_FileFinder>] >>> sys.path ['', '/usr/lib64/python34.zip', '/usr/lib/python3.4', ...]
60. find_spec(name, path): for finder in sys.meta_path: call its find_spec(name, path) return spec if successful PathFinder.find_spec(name, path): for directory in path: get sys.path_hooks entry call its find_spec(name) return spec if successful >>> sys.meta_path [<BuiltinImporter>, <FrozenImporter>, <PathFinder>] >>> sys.path_hooks [<zipimporter>, <path_hook_for_FileFinder>] >>> sys.path ['', '/usr/lib64/python34.zip', '/usr/lib/python3.4', ...]
61. find_spec(name, path): for finder in sys.meta_path: call its find_spec(name, path) return spec if successful PathFinder.find_spec(name, path): for directory in path: get sys.path_hooks entry call its find_spec(name) return spec if successful >>> sys.meta_path [<BuiltinImporter>, <FrozenImporter>, <PathFinder>] >>> sys.path_hooks [<zipimporter>, <path_hook_for_FileFinder>] >>> sys.path ['', '/usr/lib64/python34.zip', '/usr/lib/python3.4', ...] random.cpython-34m.so (Extension module) random.abi3.so (Extension module) random.so (Extension module) random.py (Source module) random.pyc (Sourceless module)
62. find_spec(name, path): for finder in sys.meta_path: call its find_spec(name, path) return spec if successful PathFinder.find_spec(name, path): for directory in path: get sys.path_hooks entry call its find_spec(name) return spec if successful ModuleSpec: name random origin /usr/lib64/python3.4/random.py cached /usr/lib64/python3.4/__pycache__/random.cpython-34.pyc loader importlib.machinery.SourceFileLoader loader_state, parent, submodule_search_locations
63. import_module(name): sys.modules[name]? return it submodules: load parent module sys.modules[name]? return it spec = find_spec(name, path) load(spec) module = sys.modules[name] submodules: set module as attribute of parent return module ?
64. load(spec): module = spec.loader.create_module(spec) if module is None: module = types.ModuleType(spec.name) set initial module attributes sys.modules[spec.name] = module spec.loader.exec_module(module, spec)
65. load(spec): module = spec.loader.create_module(spec) if module is None: module = types.ModuleType(spec.name) set initial module attributes sys.modules[spec.name] = module spec.loader.exec_module(module, spec)
66. load(spec): module = spec.loader.create_module(spec) if module is None: module = types.ModuleType(spec.name) set initial module attributes sys.modules[spec.name] = module spec.loader.exec_module(module, spec) spec → __spec__ spec.name → __name__ spec.loader → __loader__ spec.parent → __package__ spec.origin → __file__ spec.cached → __cached__ spec.submodule_search_locations → __path__
67. load(spec): module = spec.loader.create_module(spec) if module is None: module = types.ModuleType(spec.name) set initial module attributes sys.modules[spec.name] = module spec.loader.exec_module(module, spec) spec → __spec__ spec.name → __name__ spec.loader → __loader__ spec.parent → __package__ spec.origin → __file__ spec.cached → __cached__ spec.submodule_search_locations → __path__ >>> import __main__ >>> a = 'hello' >>> __main__.a 'hello' >>> __main__.a = 'world' >>> a 'world'
68. load(spec): module = spec.loader.create_module(spec) if module is None: module = types.ModuleType(spec.name) set initial module attributes sys.modules[spec.name] = module spec.loader.exec_module(module, spec) spec → __spec__ spec.name → __name__ spec.loader → __loader__ spec.parent → __package__ spec.origin → __file__ spec.cached → __cached__ spec.submodule_search_locations → __path__ >>> import __main__ >>> a = 'hello' >>> __main__.a 'hello' >>> __main__.a = 'world' >>> a 'world' if __name__ == '__main__'
69. load(spec): module = spec.loader.create_module(spec) if module is None: module = types.ModuleType(spec.name) set initial module attributes sys.modules[spec.name] = module spec.loader.exec_module(module, spec) spec → __spec__ spec.name → __name__ spec.loader → __loader__ spec.parent → __package__ spec.origin → __file__ spec.cached → __cached__ spec.submodule_search_locations → __path__ >>> import __main__ >>> a = 'hello' >>> __main__.a 'hello' >>> __main__.a = 'world' >>> a 'world' if __name__ == '__main__'
70. load(spec): module = spec.loader.create_module(spec) if module is None: module = types.ModuleType(spec.name) set initial module attributes sys.modules[spec.name] = module spec.loader.exec_module(module, spec) • Put module in sys.modules • Initialize the module
71. get_code(module, spec): if spec.cached exists, and matches origin stats, return it! load source from origin and compile it write bytecode to spec.cached ModuleSpec: name random origin /usr/lib64/python3.4/random.py cached /usr/lib64/python3.4/__pycache__/random.cpython-34.pyc ...
72. get_code(module, spec): if spec.cached exists, and matches origin stats, return it! load source from origin and compile it write bytecode to spec.cached Python 2 somemodule.py somemodule.pyc
73. get_code(module, spec): if spec.cached exists, and matches origin stats, return it! load source from origin and compile it write bytecode to spec.cached Python 2 somemodule.py somemodule.pyc
74. get_code(module, spec): if spec.cached exists, and matches origin stats, return it! load source from origin and compile it write bytecode to spec.cached Python 2 somemodule.py somemodule.pyc Python 3 somemodule.py __pycache__/ somemodule.cpython-34.pyc
75. get_code(module, spec): if spec.cached exists, and matches origin stats, return it! load source from origin and compile it write bytecode to spec.cached Python 2 somemodule.py somemodule.pyc Python 3 somemodule.py __pycache__/ somemodule.cpython-34.pyc Or: somemodule.py somemodule.pyc
76. import_module(name): sys.modules[name]? return it submodules: load parent module sys.modules[name]? return it spec = find_spec(name, path) load(spec) module = sys.modules[name] submodules: set module as attribute of parent return module find_spec(name, path): for finder in sys.meta_path: call its find_spec(name, path) return spec if successful PathFinder.find_spec(name, path): for directory in path: get sys.path_hooks entry call its find_spec(name) return spec if successful load(spec): module = spec.loader.create_module(spec) if module is None: module = types.ModuleType(spec.name) set initial module attributes sys.modules[spec.name] = module spec.loader.exec_module(module, spec) get_code(module, spec): if spec.cached exists, and matches origin stats, return it! load source from origin and compile it write bytecode to spec.cached
77. C Python Bytecode Compiled-in Built-in Frozen From File Extension Sourceless Zip file zip source zip sourceless Source