Import Deep Dive

Import Deep Dive

870d613430249e453343efc9667ef636?s=128

Petr Viktorin

July 24, 2015
Tweet

Transcript

  1. Import Deep Dive Petr Viktorin @encukou encukou@gmail.com encukou.cz EuroPython 2015

  2. import random

  3. Prior art David Beazley — Modules and Packages: Live and

    Let Die! http://pyvideo.org/video/3387
  4. Photo © Paul Hermans, https://en.wikipedia.org/wiki/File:Tunnelaquarium_14-05-2009_15-54-09.JPG

  5. import random

  6. import random random = __import__('random')

  7. import random random = __import__('random') from ..spam import foo

  8. import random random = __import__('random') from ..spam import foo foo

    = __import__('spam', globals(), locals(), ['foo'], 2).foo https://docs.python.org/3/library/functions.html#__import__
  9. import random random = __import__('random')

  10. import random random = __import__('random') The Import Machinery

  11. import random random = __import__('random') The Import Machinery importlib.import_module('random')

  12. import random The Import Machinery importlib.import_module('random')

  13. The Import Machinery Photo © Les Chatfield, https://www.flickr.com/photos/elsie/8229790

  14. The Import Machinery Photo © Les Chatfield, https://www.flickr.com/photos/elsie/8229790 sans: •

    Locking • Caching • Error handling • Pythons older than 3.4 • Backwards compatibility shims
  15. importlib.import_module('random')

  16. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module

    = sys.modules[name] return module >>> import random >>> old = random >>> import random >>> random is old True
  17. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module

    = sys.modules[name] return module >>> import random >>> old = random >>> import random >>> random is old True
  18. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module

    = sys.modules[name] return module >>> import random >>> old = random >>> import random >>> random is old True
  19. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module

    = sys.modules[name] return module >>> import random >>> old = random >>> del sys.modules['random'] >>> import random >>> random is old False >>> (random.sample is ... old.sample) False >>> sys.modules['impostor'] = 'Not a module, is it?' >>> import impostor >>> impostor[:12] 'Not a module'
  20. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module

    = sys.modules[name] return module >>> import random >>> old = random >>> del sys.modules['random'] >>> import random >>> random is old False >>> (random.sample is ... old.sample) False >>> sys.modules['impostor'] = 'Not a module, is it?' >>> import impostor >>> impostor[:12] 'Not a module'
  21. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module

    = sys.modules[name] return module >>> sys.path ['', '/usr/lib64/python34.zip', '/usr/lib64/python3.4', ...] >>> importlib.util.find_spec('antigravity') ModuleSpec(name='antigravity', loader=<_frozen_importlib.SourceFileLoader ...>, origin='/usr/lib64/python3.4/antigravity.py') >>> import io >>> io.__spec__ ModuleSpec(name='io', loader=<_frozen_importlib.SourceFileLoader ...>, origin='/usr/lib64/python3.4/io.py')
  22. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module

    = sys.modules[name] return module >>> sys.path ['', '/usr/lib64/python34.zip', '/usr/lib64/python3.4', ...] >>> importlib.util.find_spec('antigravity') ModuleSpec(name='antigravity', loader=<_frozen_importlib.SourceFileLoader ...>, origin='/usr/lib64/python3.4/antigravity.py') >>> import io >>> io.__spec__ ModuleSpec(name='io', loader=<_frozen_importlib.SourceFileLoader ...>, origin='/usr/lib64/python3.4/io.py')
  23. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module

    = sys.modules[name] return module >>> sys.path ['', '/usr/lib64/python34.zip', '/usr/lib64/python3.4', ...] >>> importlib.util.find_spec('antigravity') ModuleSpec(name='antigravity', loader=<_frozen_importlib.SourceFileLoader ...>, origin='/usr/lib64/python3.4/antigravity.py') >>> import io >>> io.__spec__ ModuleSpec(name='io', loader=<_frozen_importlib.SourceFileLoader ...>, origin='/usr/lib64/python3.4/io.py')
  24. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module

    = sys.modules[name] return module >>> sys.path ['', '/usr/lib64/python34.zip', '/usr/lib64/python3.4', ...] >>> importlib.util.find_spec('antigravity') ModuleSpec(name='antigravity', loader=<_frozen_importlib.SourceFileLoader ...>, origin='/usr/lib64/python3.4/antigravity.py') >>> import io >>> io.__spec__ ModuleSpec(name='io', loader=<_frozen_importlib.SourceFileLoader ...>, origin='/usr/lib64/python3.4/io.py')
  25. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module

    = sys.modules[name] return module • Put module in sys.modules • Initialize the module
  26. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module

    = sys.modules[name] return module foo.py: import bar bar.do_bar() def do_foo(): print('Fooing!') bar.py: import foo foo.do_foo() def do_bar(): print('Barring!') • Put module in sys.modules • Initialize the module
  27. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module

    = sys.modules[name] return module foo.py: import bar bar.do_bar() def do_foo(): print('Fooing!') bar.py: import foo foo.do_foo() def do_bar(): print('Barring!') • Put module in sys.modules • Initialize the module
  28. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module

    = sys.modules[name] return module foo.py: import bar bar.do_bar() def do_foo(): print('Fooing!') bar.py: import foo foo.do_foo() def do_bar(): print('Barring!') • Put module in sys.modules • Initialize the module
  29. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module

    = sys.modules[name] return module foo.py: import bar bar.do_bar() def do_foo(): print('Fooing!') bar.py: import foo foo.do_foo() def do_bar(): print('Barring!') • Put module in sys.modules • Initialize the module
  30. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module

    = sys.modules[name] return module foo.py: import bar bar.do_bar() def do_foo(): print('Fooing!') bar.py: import foo foo.do_foo() def do_bar(): print('Barring!') • Put module in sys.modules • Initialize the module
  31. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module

    = sys.modules[name] return module
  32. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module

    = sys.modules[name] return module Filename Module name random.py random urllib/ __init__.py urllib parse.py urllib.parse request.py urllib.request response.py urllib.response Top-level module Package Parent of the below Submodule
  33. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module

    = sys.modules[name] return module Filename Module name random.py random urllib/ __init__.py urllib parse.py urllib.parse request.py urllib.request response.py urllib.response Top-level module Package Parent of the below Submodule
  34. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module

    = sys.modules[name] return module import urllib.parse >>> urllib.__path__ ['/usr/lib64/python3.4/urllib'] Filename Module name random.py random urllib/ __init__.py urllib parse.py urllib.parse request.py urllib.request response.py urllib.response Top-level module Package Parent of the below Submodule
  35. import_module(name): sys.modules[name]? return it submodules: load parent module sys.modules[name]? return

    it spec = find_spec(name, path) load(spec) module = sys.modules[name] submodules: set module as attribute of parent return module import urllib.parse >>> urllib.__path__ ['/usr/lib64/python3.4/urllib'] Filename Module name random.py random urllib/ __init__.py urllib parse.py urllib.parse request.py urllib.request response.py urllib.response Top-level module Package Parent of the below Submodule
  36. import_module(name): sys.modules[name]? return it submodules: load parent module sys.modules[name]? return

    it spec = find_spec(name, path) load(spec) module = sys.modules[name] submodules: set module as attribute of parent return module • Put module in sys.modules • Initialize the module
  37. import_module(name): sys.modules[name]? return it submodules: load parent module sys.modules[name]? return

    it spec = find_spec(name, path) load(spec) module = sys.modules[name] submodules: set module as attribute of parent return module foo/__init__.py: from foo import main from foo import consts foo/main.py: import foo use(foo.consts.value) foo/consts.py: value = 42 • Put module in sys.modules • Initialize the module
  38. import_module(name): sys.modules[name]? return it submodules: load parent module sys.modules[name]? return

    it spec = find_spec(name, path) load(spec) module = sys.modules[name] submodules: set module as attribute of parent return module foo/__init__.py: from foo import main from foo import consts foo/main.py: import foo use(foo.consts.value) foo/consts.py: value = 42 • Put module in sys.modules • Initialize the module
  39. import_module(name): sys.modules[name]? return it submodules: load parent module sys.modules[name]? return

    it spec = find_spec(name, path) load(spec) module = sys.modules[name] submodules: set module as attribute of parent return module foo/__init__.py: from foo import main from foo import consts foo/main.py: import foo use(foo.consts.value) foo/consts.py: value = 42 • Put module in sys.modules • Initialize the module
  40. import_module(name): sys.modules[name]? return it submodules: load parent module sys.modules[name]? return

    it spec = find_spec(name, path) load(spec) module = sys.modules[name] submodules: set module as attribute of parent return module foo/__init__.py: from foo import main from foo import consts foo/main.py: import foo use(foo.consts.value) foo/consts.py: value = 42 • Put module in sys.modules • Initialize the module
  41. import_module(name): sys.modules[name]? return it submodules: load parent module sys.modules[name]? return

    it spec = find_spec(name, path) load(spec) module = sys.modules[name] submodules: set module as attribute of parent return module foo/__init__.py: from foo import main from foo import consts foo/main.py: import foo use(foo.consts.value) foo/consts.py: value = 42 • Put module in sys.modules • Initialize the module
  42. import_module(name): sys.modules[name]? return it submodules: load parent module sys.modules[name]? return

    it spec = find_spec(name, path) load(spec) module = sys.modules[name] submodules: set module as attribute of parent return module foo/__init__.py: from foo import main from foo import consts foo/main.py: import foo use(foo.consts.value) foo/consts.py: value = 42 • Put module in sys.modules • Initialize the module
  43. import_module(name): sys.modules[name]? return it submodules: load parent module sys.modules[name]? return

    it spec = find_spec(name, path) load(spec) module = sys.modules[name] submodules: set module as attribute of parent return module foo/__init__.py: from foo import main from foo import consts foo/main.py: import foo use(foo.consts.value) foo/consts.py: value = 42 • Put module in sys.modules • Initialize the module
  44. __init__ should: import from submodules set __all__ nothing else submodules

    should: not import directly from __init__ not have internal import cycles
  45. import_module(name): sys.modules[name]? return it submodules: load parent module sys.modules[name]? return

    it spec = find_spec(name, path) load(spec) module = sys.modules[name] submodules: set module as attribute of parent return module ?
  46. Where's the module? >>> import random >>> random <module 'random'

    from '/usr/lib64/python3.4/random.py'> >>> random.__file__ '/usr/lib64/python3.4/random.py' >>> import sys >>> sys <module 'sys' (built-in)> >>> sys.__file__ Traceback (most recent call last): File "<stdin>", line 1, in <module> AttributeError: 'module' object has no attribute '__file__' /usr/bin/python3
  47. Where's the module? >>> import random >>> random <module 'random'

    from '/usr/lib64/python3.4/random.py'> >>> random.__file__ '/usr/lib64/python3.4/random.py' >>> import sys >>> sys <module 'sys' (built-in)> >>> sys.__file__ Traceback (most recent call last): File "<stdin>", line 1, in <module> AttributeError: 'module' object has no attribute '__file__' /usr/bin/python3
  48. Where's the module? >>> import random >>> random <module 'random'

    from '/usr/lib64/python3.4/random.py'> >>> random.__file__ '/usr/lib64/python3.4/random.py' >>> import sys >>> sys <module 'sys' (built-in)> >>> sys.__file__ Traceback (most recent call last): File "<stdin>", line 1, in <module> AttributeError: 'module' object has no attribute '__file__' /usr/bin/python3
  49. Where's the module? >>> import random >>> random <module 'random'

    from '/usr/lib64/python3.4/random.py'> >>> random.__file__ '/usr/lib64/python3.4/random.py' >>> import sys >>> sys <module 'sys' (built-in)> >>> sys.__file__ Traceback (most recent call last): File "<stdin>", line 1, in <module> AttributeError: 'module' object has no attribute '__file__' /usr/bin/python3
  50. Where's the module? >>> import random >>> random <module 'random'

    from '/usr/lib64/python3.4/random.py'> >>> random.__file__ '/usr/lib64/python3.4/random.py' >>> import sys >>> sys <module 'sys' (built-in)> >>> sys.__file__ Traceback (most recent call last): File "<stdin>", line 1, in <module> AttributeError: 'module' object has no attribute '__file__' /usr/bin/python3
  51. Where's the module? >>> import random >>> random <module 'random'

    from '/usr/lib64/python3.4/random.py'> >>> random.__file__ '/usr/lib64/python3.4/random.py' >>> import sys >>> sys <module 'sys' (built-in)> >>> sys.__file__ Traceback (most recent call last): File "<stdin>", line 1, in <module> AttributeError: 'module' object has no attribute '__file__' /usr/bin/python3
  52. Where's the module? >>> import random >>> random <module 'random'

    from '/usr/lib64/python3.4/random.py'> >>> random.__file__ '/usr/lib64/python3.4/random.py' >>> import sys >>> sys <module 'sys' (built-in)> >>> sys.__file__ Traceback (most recent call last): File "<stdin>", line 1, in <module> AttributeError: 'module' object has no attribute '__file__' /usr/bin/python3
  53. Photo © Allentchang, https://commons.wikimedia.org/wiki/File:Fish_in_Okinawa_Churaumi_Aquarium.JPG C Python Compiled in Built-in Frozen

    From File Extension Source sys _frozen_importlib math random
  54. Photo © Allentchang, https://commons.wikimedia.org/wiki/File:Fish_in_Okinawa_Churaumi_Aquarium.JPG C Python Compiled in Built-in Frozen

    From File Extension Source sys _frozen_importlib math random
  55. Photo © Allentchang, https://commons.wikimedia.org/wiki/File:Fish_in_Okinawa_Churaumi_Aquarium.JPG C Python Compiled in Built-in Frozen

    From File Extension Source sys _frozen_importlib math random
  56. >>> sys.meta_path [<BuiltinImporter>, <FrozenImporter>, <PathFinder>]

  57. find_spec(name, path): for finder in sys.meta_path: call its find_spec(name, path)

    return spec if successful >>> sys.meta_path [<BuiltinImporter>, <FrozenImporter>, <PathFinder>]
  58. find_spec(name, path): for finder in sys.meta_path: call its find_spec(name, path)

    return spec if successful >>> sys.meta_path [<BuiltinImporter>, <FrozenImporter>, <PathFinder>] >>> sys.path ['', '/usr/lib64/python34.zip', '/usr/lib/python3.4', ...]
  59. find_spec(name, path): for finder in sys.meta_path: call its find_spec(name, path)

    return spec if successful >>> sys.meta_path [<BuiltinImporter>, <FrozenImporter>, <PathFinder>] >>> sys.path_hooks [<zipimporter>, <path_hook_for_FileFinder>] >>> sys.path ['', '/usr/lib64/python34.zip', '/usr/lib/python3.4', ...]
  60. find_spec(name, path): for finder in sys.meta_path: call its find_spec(name, path)

    return spec if successful PathFinder.find_spec(name, path): for directory in path: get sys.path_hooks entry call its find_spec(name) return spec if successful >>> sys.meta_path [<BuiltinImporter>, <FrozenImporter>, <PathFinder>] >>> sys.path_hooks [<zipimporter>, <path_hook_for_FileFinder>] >>> sys.path ['', '/usr/lib64/python34.zip', '/usr/lib/python3.4', ...]
  61. find_spec(name, path): for finder in sys.meta_path: call its find_spec(name, path)

    return spec if successful PathFinder.find_spec(name, path): for directory in path: get sys.path_hooks entry call its find_spec(name) return spec if successful >>> sys.meta_path [<BuiltinImporter>, <FrozenImporter>, <PathFinder>] >>> sys.path_hooks [<zipimporter>, <path_hook_for_FileFinder>] >>> sys.path ['', '/usr/lib64/python34.zip', '/usr/lib/python3.4', ...] random.cpython-34m.so (Extension module) random.abi3.so (Extension module) random.so (Extension module) random.py (Source module) random.pyc (Sourceless module)
  62. find_spec(name, path): for finder in sys.meta_path: call its find_spec(name, path)

    return spec if successful PathFinder.find_spec(name, path): for directory in path: get sys.path_hooks entry call its find_spec(name) return spec if successful ModuleSpec: name random origin /usr/lib64/python3.4/random.py cached /usr/lib64/python3.4/__pycache__/random.cpython-34.pyc loader importlib.machinery.SourceFileLoader loader_state, parent, submodule_search_locations
  63. import_module(name): sys.modules[name]? return it submodules: load parent module sys.modules[name]? return

    it spec = find_spec(name, path) load(spec) module = sys.modules[name] submodules: set module as attribute of parent return module ?
  64. load(spec): module = spec.loader.create_module(spec) if module is None: module =

    types.ModuleType(spec.name) set initial module attributes sys.modules[spec.name] = module spec.loader.exec_module(module, spec)
  65. load(spec): module = spec.loader.create_module(spec) if module is None: module =

    types.ModuleType(spec.name) set initial module attributes sys.modules[spec.name] = module spec.loader.exec_module(module, spec)
  66. load(spec): module = spec.loader.create_module(spec) if module is None: module =

    types.ModuleType(spec.name) set initial module attributes sys.modules[spec.name] = module spec.loader.exec_module(module, spec) spec → __spec__ spec.name → __name__ spec.loader → __loader__ spec.parent → __package__ spec.origin → __file__ spec.cached → __cached__ spec.submodule_search_locations → __path__
  67. load(spec): module = spec.loader.create_module(spec) if module is None: module =

    types.ModuleType(spec.name) set initial module attributes sys.modules[spec.name] = module spec.loader.exec_module(module, spec) spec → __spec__ spec.name → __name__ spec.loader → __loader__ spec.parent → __package__ spec.origin → __file__ spec.cached → __cached__ spec.submodule_search_locations → __path__ >>> import __main__ >>> a = 'hello' >>> __main__.a 'hello' >>> __main__.a = 'world' >>> a 'world'
  68. load(spec): module = spec.loader.create_module(spec) if module is None: module =

    types.ModuleType(spec.name) set initial module attributes sys.modules[spec.name] = module spec.loader.exec_module(module, spec) spec → __spec__ spec.name → __name__ spec.loader → __loader__ spec.parent → __package__ spec.origin → __file__ spec.cached → __cached__ spec.submodule_search_locations → __path__ >>> import __main__ >>> a = 'hello' >>> __main__.a 'hello' >>> __main__.a = 'world' >>> a 'world' if __name__ == '__main__'
  69. load(spec): module = spec.loader.create_module(spec) if module is None: module =

    types.ModuleType(spec.name) set initial module attributes sys.modules[spec.name] = module spec.loader.exec_module(module, spec) spec → __spec__ spec.name → __name__ spec.loader → __loader__ spec.parent → __package__ spec.origin → __file__ spec.cached → __cached__ spec.submodule_search_locations → __path__ >>> import __main__ >>> a = 'hello' >>> __main__.a 'hello' >>> __main__.a = 'world' >>> a 'world' if __name__ == '__main__'
  70. load(spec): module = spec.loader.create_module(spec) if module is None: module =

    types.ModuleType(spec.name) set initial module attributes sys.modules[spec.name] = module spec.loader.exec_module(module, spec) • Put module in sys.modules • Initialize the module
  71. get_code(module, spec): if spec.cached exists, and matches origin stats, return

    it! load source from origin and compile it write bytecode to spec.cached ModuleSpec: name random origin /usr/lib64/python3.4/random.py cached /usr/lib64/python3.4/__pycache__/random.cpython-34.pyc ...
  72. get_code(module, spec): if spec.cached exists, and matches origin stats, return

    it! load source from origin and compile it write bytecode to spec.cached Python 2 somemodule.py somemodule.pyc
  73. get_code(module, spec): if spec.cached exists, and matches origin stats, return

    it! load source from origin and compile it write bytecode to spec.cached Python 2 somemodule.py somemodule.pyc
  74. get_code(module, spec): if spec.cached exists, and matches origin stats, return

    it! load source from origin and compile it write bytecode to spec.cached Python 2 somemodule.py somemodule.pyc Python 3 somemodule.py __pycache__/ somemodule.cpython-34.pyc
  75. get_code(module, spec): if spec.cached exists, and matches origin stats, return

    it! load source from origin and compile it write bytecode to spec.cached Python 2 somemodule.py somemodule.pyc Python 3 somemodule.py __pycache__/ somemodule.cpython-34.pyc Or: somemodule.py somemodule.pyc
  76. import_module(name): sys.modules[name]? return it submodules: load parent module sys.modules[name]? return

    it spec = find_spec(name, path) load(spec) module = sys.modules[name] submodules: set module as attribute of parent return module find_spec(name, path): for finder in sys.meta_path: call its find_spec(name, path) return spec if successful PathFinder.find_spec(name, path): for directory in path: get sys.path_hooks entry call its find_spec(name) return spec if successful load(spec): module = spec.loader.create_module(spec) if module is None: module = types.ModuleType(spec.name) set initial module attributes sys.modules[spec.name] = module spec.loader.exec_module(module, spec) get_code(module, spec): if spec.cached exists, and matches origin stats, return it! load source from origin and compile it write bytecode to spec.cached
  77. C Python Bytecode Compiled-in Built-in Frozen From File Extension Sourceless

    Zip file zip source zip sourceless Source