Playing with Python's internals by Alex Hall

This talk will look at two of my libraries which stretch the limits of what's possible with Python:

1. [birdseye](https://github.com/alexmojaki/birdseye), a debugger that records the value of every expression for easy viewing, and
2. [sorcery](https://github.com/alexmojaki/sorcery), a framework for writing magical functions which know the context in which they are called.

They work by inspecting and manipulating Python's inner workings: execution frames, code objects, and most importantly the Abstract Syntax Tree (AST). I will give an overview of these concepts and explain how some parts of the libraries work.

This is for people interested in peeking under the hood of Python from within Python, i.e. no C and no messing with the interpreter.

Transcript

1. Playing with Python’s Internals Alex Hall github.com/alexmojaki
3. $> pip install birdseye CPython 2.7, 3.5+ github.com/alexmojaki/birdseye (Google: python birdseye)
4. from birdseye import eye @eye def factorial(n): $> birdseye
5. import ast
7. node = ast.parse('x = 2; print(x + 3)') ast.dump(node) => Module(body=[ Assign(targets=[Name(id='x')], value=Num(n=2)), Expr(value=Call(func=Name(id='print'), args=[BinOp(left=Name(id='x'), op=Add(), right=Num(n=3))])) ])
8. node = ast.parse('x = 2; print(x + 3)') ast.dump(node) => Module(body=[ Assign(targets=[Name(id='x')], value=Num(n=2)), Expr(value=Call(func=Name(id='print'), args=[BinOp(left=Name(id='x'), op=Add(), right=Num(n=3))])) ])
9. node = ast.parse('x = 2; print(x + 3)') ast.dump(node) => Module(body=[ Assign(targets=[Name(id='x')], value=Num(n=2)), Expr(value=Call(func=Name(id='print'), args=[BinOp(left=Name(id='x'), op=Add(), right=Num(n=3))])) ])
10. node = ast.parse('x = 2; print(x + 3)') ast.dump(node) => Module(body=[ Assign(targets=[Name(id='x')], value=Num(n=2)), Expr(value=Call(func=Name(id='print'), args=[BinOp(left=Name(id='x'), op=Add(), right=Num(n=3))])) ]) node.body[0].value.n => 2
11. node = ast.parse('x = 2; print(x + 3)') node.body[0].value.n => 2 code = compile(node, filename='<string>', mode='exec') => <code object <module> at 0x104c990c0, file "<string>", line 1>
12. node = ast.parse('x = 2; print(x + 3)') node.body[0].value.n => 2 code = compile(node, filename='<string>', mode='exec') => <code object <module> at 0x104c990c0, file "<string>", line 1>
13. node = ast.parse('x = 2; print(x + 3)') node.body[0].value.n => 2 code = compile(node, filename='<string>', mode='exec') => <code object <module> at 0x104c990c0, file "<string>", line 1> exec(code) => 5
14. node = ast.parse('x = 2; print(x + 3)') node.body[0].value.n = 10 code = compile(node, filename='<string>', mode='exec') => <code object <module> at 0x104c990c0, file "<string>", line 1> exec(code) => 5 13
15. node = ast.parse('x = 2; print(x + 3)') class MyVisitor(ast.NodeTransformer): def visit_Num(self, _node): return ast.Num(n=100) MyVisitor().visit(node) ast.fix_missing_locations(node) code = compile(node, filename='<string>', mode='exec') exec(code) => 200
16. node = ast.parse('x = 2; print(x + 3)') class MyVisitor(ast.NodeTransformer): def visit_Num(self, _node): return ast.Num(n=100) MyVisitor().visit(node) ast.fix_missing_locations(node) code = compile(node, filename='<string>', mode='exec') exec(code) => 200
17. expr → after(before(...), expr) stmt → with context(...): stmt
18. expr → after(before(...), expr) foo(1/(i*2)) 1/(i*2) i*2 before after
19. expr → after(before(...), expr) foo(1/(i*2)) 1/(i*2) i*2 before after
20. import inspect class A: def __init__(self): super().__init__() print('success!') source = inspect.getsource(A.__init__).strip() node = ast.parse(source) code = compile(node, filename=__file__, mode='exec') exec(source) A.__init__ = __init__ A() RuntimeError: super(): __class__ cell not found
21. import inspect class A: def __init__(self): super().__init__() print('success!') source = inspect.getsource(A.__init__).strip() node = ast.parse(source) code = compile(node, filename=__file__, mode='exec') exec(source) A.__init__ = __init__ A() RuntimeError: super(): __class__ cell not found
22. import inspect class A: def __init__(self): super().__init__() print('success!') source = inspect.getsource(A.__init__).strip() node = ast.parse(source) # modify node somehow... code = compile(node, filename=__file__, mode='exec') exec(source) A.__init__ = __init__ A() RuntimeError: super(): __class__ cell not found
23. import inspect class A: def __init__(self): super().__init__() print('success!') source = inspect.getsource(A.__init__).strip() node = ast.parse(source) # modify node somehow... code = compile(node, filename=__file__, mode='exec') exec(source) A.__init__ = __init__ A() RuntimeError: super(): __class__ cell not found
24. import inspect class A: def __init__(self): super().__init__() print('success!') source = inspect.getsource(A.__init__).strip() node = ast.parse(source) # modify node somehow... code = compile(node, filename=__file__, mode='exec') exec(source) A.__init__ = __init__ A() RuntimeError: super(): __class__ cell not found
25. help(code) => class code(object) | code(argcount, kwonlyargcount, nlocals, stacksize, | flags, codestring, constants, names, varnames, | filename, name, firstlineno, lnotab, | [freevars, [cellvars]]) | | Create a code object. Not for the faint of heart. ...
26. y = 2 class A: z = 4 def __init__(self): ... filename = inspect.getsourcefile(A.__init__) source = open(filename).read() node = ast.parse(source) code = compile(node, filename=filename, mode='exec') => <code object <module> ...> code.co_consts => (2, <code object A ...>, 'A', None) code.co_consts[1].co_consts => ('A', 4, <code object __init__ ...>, 'A.__init__')
27. y = 2 class A: z = 4 def __init__(self): ... A.__init__.__code__.co_name: '__init__' A.__init__.__code__.co_firstlineno: 4ode = ast.parse(source) code = compile(node, filename=filename, mode='exec') => <code object <module> ...> code.co_consts => (2, <code object A ...>, 'A', None) code.co_consts[1].co_consts => ('A', 4, <code object __init__ ...>, 'A.__init__')
28. y = 2 class A: z = 4 def __init__(self): ... filename = inspect.getsourcefile(A.__init__) source = open(filename).read() node = ast.parse(source) # modify node somehow... code = compile(node, filename=filename, mode='exec') => <code object <module> ... line 1> code.co_consts => (2, <code object A ...>, 'A', None) code.co_consts[1].co_consts => ('A', 4, <code object __init__ ...>, 'A.__init__')
29. y = 2 class A: z = 4 def __init__(self): ... filename = inspect.getsourcefile(A.__init__) source = open(filename).read() node = ast.parse(source) # modify node somehow... code = compile(node, filename=filename, mode='exec') => <code object <module> ... line 1> code.co_consts => (2, <code object A ... line 2>, 'A', None) code.co_consts[1].co_consts => ('A', 4, <code object __init__ ...>, 'A.__init__')
30. y = 2 class A: z = 4 def __init__(self): ... filename = inspect.getsourcefile(A.__init__) source = open(filename).read() node = ast.parse(source) # modify node somehow... code = compile(node, filename=filename, mode='exec') => <code object <module> ... line 1> code.co_consts => (2, <code object A ... line 2>, 'A', None) code.co_consts[1].co_consts => ('A', 4, <code object __init__ ... line 4>, 'A.__init__')
31. from types import FunctionType new_func = FunctionType(new_func_code, func.__globals__, func.__name__, func.__defaults__, func.__closure__)
32. Questions?
33. $> pip install sorcery CPython ≥ 3.5 github.com/alexmojaki/sorcery (Google: python sorcery)
34. from sorcery import *
35. foo = func('foo') bar = func('bar') ↓ foo, bar = [ func(name) for name in assigned_names() ]
36. foo = func('foo') bar = func('bar') ↓ foo, bar = [ func(name) for name in assigned_names() ]
37. foo = func('foo') bar = func('bar') ↓ foo, bar = [ func(name) for name in assigned_names() ] ('foo', 'bar')
38. class Thing(Enum): foo = 'foo' bar = 'bar' ↓ class Thing(Enum): foo, bar = assigned_names()
39. class Thing(Enum): foo = 'foo' bar = 'bar' ↓ class Thing(Enum): foo, bar = assigned_names()
40. foo = d['foo'] bar = d['bar'] ↓ foo, bar = unpack_keys(d) for foo, bar in unpack_keys([{‘foo’: 1, ‘bar’: 2}, …]):
41. thing[‘foo’] = x.foo bar = x.bar ↓ thing[‘foo’], bar = unpack_attrs(x)
42. dict(foo=foo, bar=bar, spam=thing()) ↓ dict_of(thing.foo, thing[‘bar’](1), spam=thing())
43. None if foo is None else foo.bar() ↓ maybe(foo).bar()
44. def foo(): bar() def bar(): 1/0 foo()
45. def foo(): bar() def bar(): 1/0 foo() Traceback (most recent call last): File "/my/script.py", line 7, in <module> foo() File "/my/script.py", line 2, in foo bar() File "/my/script.py", line 5, in bar 1/0 ZeroDivisionError: division by zero
46. def foo(): bar() def bar(): 1/0 foo() Traceback (most recent call last): File "/my/script.py", line 7, in <module> foo() File "/my/script.py", line 2, in foo bar() File "/my/script.py", line 5, in bar 1/0 ZeroDivisionError: division by zero
47. def foo(): bar() def bar(): 1/0 foo() Traceback (most recent call last): File "/my/script.py", line 7, in <module> foo() File "/my/script.py", line 2, in foo bar() File "/my/script.py", line 5, in bar 1/0 ZeroDivisionError: division by zero code.co_filename frame.f_lineno code.co_name
48. def foo(): x = 1 y = 2 return 3 * bar(x) def bar(): previous = inspect.currentframe().f_back previous.f_locals: {'x': 1, 'y': 2} previous.f_code: <code object foo ...> previous.f_lineno: 4
49. def foo(): x = 1 y = 2 return 3 * bar(x) def bar(): previous = inspect.currentframe().f_back previous.f_locals: {'x': 1, 'y': 2} previous.f_code: <code object foo ...> previous.f_lineno: 4
50. def foo(): x = 1 y = 2 return 3 * bar(x) def bar(): previous = inspect.currentframe().f_back f = open(previous.f_code.co_filename) lines = f.readlines() line = lines[previous.f_lineno - 1].strip() node = ast.parse(line) # find Call node... => Call(func=Name(id='bar'), ...)
51. def foo(): x = 1 y = bar return 3 * y(x) def bar(): previous = inspect.currentframe().f_back f = open(previous.f_code.co_filename) lines = f.readlines() line = lines[previous.f_lineno - 1].strip() node = ast.parse(line) # find Call node... previous.f_locals: {'y': <function bar ...>, ...} => Call(func=Name(id='y'), ...)
52. def foo(): x = 1 y = 2 bar(x, y) @spell def bar(frame_info, ...): ... args: (1, 2) frame_info.call: Call(func=Name(id='bar'), args=[Name(id='x'), Name(id='y')])
53. def foo(): x = 1 y = 2 bar(x, y) @spell def bar(frame_info, ...): ... args: (1, 2) frame_info.call: Call(func=Name(id='bar'), args=[Name(id='x'), Name(id='y')])
54. dict_of(foo, bar) == dict(foo=foo, bar=bar) @spell def dict_of(frame_info, *values): return { arg.id: value for arg, value in zip(frame_info.call.args, values) }
55. import sys from inspect import stack def trace(frame, event, arg): ... sys.settrace(trace)
56. import sys from inspect import stack def trace(frame, event, arg): if event == 'call': print(' ' * len(stack()), frame.f_code.co_name) sys.settrace(trace)
57. def main(): for x in range(3): foo() def foo(): bar() def bar(): pass main()
58. def main(): for x in range(3): foo() def foo(): bar() def bar(): pass main() main foo bar foo bar foo bar
59. Open Space: Creative ideas specific to Python 13:45 - 15:15