Playing with Python's internals by Alex Hall

Slide 1

Slide 1 text

Playing with Python’s Internals Alex Hall github.com/alexmojaki

Slide 2

Slide 2 text

No content

Slide 3

Slide 3 text

$> pip install birdseye CPython 2.7, 3.5+ github.com/alexmojaki/birdseye (Google: python birdseye)

Slide 4

Slide 4 text

from birdseye import eye @eye def factorial(n): $> birdseye

Slide 5

Slide 5 text

import ast

Slide 6

Slide 6 text

No content

Slide 7

Slide 7 text

node = ast.parse('x = 2; print(x + 3)') ast.dump(node) => Module(body=[ Assign(targets=[Name(id='x')], value=Num(n=2)), Expr(value=Call(func=Name(id='print'), args=[BinOp(left=Name(id='x'), op=Add(), right=Num(n=3))])) ])

Slide 8

Slide 8 text

Slide 9

Slide 9 text

Slide 10

Slide 10 text

Slide 11

Slide 11 text

node = ast.parse('x = 2; print(x + 3)') node.body[0].value.n => 2 code = compile(node, filename='', mode='exec') =>

 at 0x104c990c0, file "", line 1>

Slide 12

Slide 12 text

node = ast.parse('x = 2; print(x + 3)') node.body[0].value.n => 2 code = compile(node, filename='', mode='exec') =>

 at 0x104c990c0, file "", line 1>

Slide 13

Slide 13 text

node = ast.parse('x = 2; print(x + 3)') node.body[0].value.n => 2 code = compile(node, filename='', mode='exec') =>

 at 0x104c990c0, file "", line 1>
exec(code)
=> 5

Slide 14

Slide 14 text

node = ast.parse('x = 2; print(x + 3)') node.body[0].value.n = 10 code = compile(node, filename='', mode='exec') =>

 at 0x104c990c0, file "", line 1>
exec(code)
=> 5 13

Slide 15

Slide 15 text

node = ast.parse('x = 2; print(x + 3)') class MyVisitor(ast.NodeTransformer): def visit_Num(self, _node): return ast.Num(n=100) MyVisitor().visit(node) ast.fix_missing_locations(node) code = compile(node, filename='', mode='exec') exec(code) => 200

Slide 16

Slide 16 text

Slide 17

Slide 17 text

expr → after(before(...), expr) stmt → with context(...): stmt

Slide 18

Slide 18 text

expr → after(before(...), expr) foo(1/(i*2)) 1/(i*2) i*2 before after

Slide 19

Slide 19 text

expr → after(before(...), expr) foo(1/(i*2)) 1/(i*2) i*2 before after

Slide 20

Slide 20 text

import inspect class A: def __init__(self): super().__init__() print('success!') source = inspect.getsource(A.__init__).strip() node = ast.parse(source) code = compile(node, filename=__file__, mode='exec') exec(source) A.__init__ = __init__ A() RuntimeError: super(): __class__ cell not found

Slide 21

Slide 21 text

Slide 22

Slide 22 text

import inspect class A: def __init__(self): super().__init__() print('success!') source = inspect.getsource(A.__init__).strip() node = ast.parse(source) # modify node somehow... code = compile(node, filename=__file__, mode='exec') exec(source) A.__init__ = __init__ A() RuntimeError: super(): __class__ cell not found

Slide 23

Slide 23 text

Slide 24

Slide 24 text

Slide 25

Slide 25 text

Slide 26

Slide 26 text

y = 2 class A: z = 4 def __init__(self): ... filename = inspect.getsourcefile(A.__init__) source = open(filename).read() node = ast.parse(source) code = compile(node, filename=filename, mode='exec') =>

 ...>
code.co_consts
=> (2, , 'A', None)
code.co_consts[1].co_consts
=> ('A', 4, , 'A.__init__')

Slide 27

Slide 27 text

y = 2 class A: z = 4 def __init__(self): ... A.__init__.__code__.co_name: '__init__' A.__init__.__code__.co_firstlineno: 4ode = ast.parse(source) code = compile(node, filename=filename, mode='exec') =>

 ...>
code.co_consts
=> (2, , 'A', None)
code.co_consts[1].co_consts
=> ('A', 4, , 'A.__init__')

Slide 28

Slide 28 text

y = 2 class A: z = 4 def __init__(self): ... filename = inspect.getsourcefile(A.__init__) source = open(filename).read() node = ast.parse(source) # modify node somehow... code = compile(node, filename=filename, mode='exec') =>

 ... line 1>
code.co_consts
=> (2, , 'A', None)
code.co_consts[1].co_consts
=> ('A', 4, , 'A.__init__')

Slide 29

Slide 29 text

 ... line 1>
code.co_consts
=> (2, , 'A', None)
code.co_consts[1].co_consts
=> ('A', 4, , 'A.__init__')

Slide 30

Slide 30 text

 ... line 1>
code.co_consts
=> (2, , 'A', None)
code.co_consts[1].co_consts
=> ('A', 4, , 'A.__init__')

Slide 31

Slide 31 text

from types import FunctionType new_func = FunctionType(new_func_code, func.__globals__, func.__name__, func.__defaults__, func.__closure__)

Slide 32

Slide 32 text

Questions?

Slide 33

Slide 33 text

$> pip install sorcery CPython ≥ 3.5 github.com/alexmojaki/sorcery (Google: python sorcery)

Slide 34

Slide 34 text

from sorcery import *

Slide 35

Slide 35 text

foo = func('foo') bar = func('bar') ↓ foo, bar = [ func(name) for name in assigned_names() ]

Slide 36

Slide 36 text

foo = func('foo') bar = func('bar') ↓ foo, bar = [ func(name) for name in assigned_names() ]

Slide 37

Slide 37 text

foo = func('foo') bar = func('bar') ↓ foo, bar = [ func(name) for name in assigned_names() ] ('foo', 'bar')

Slide 38

Slide 38 text

class Thing(Enum): foo = 'foo' bar = 'bar' ↓ class Thing(Enum): foo, bar = assigned_names()

Slide 39

Slide 39 text

class Thing(Enum): foo = 'foo' bar = 'bar' ↓ class Thing(Enum): foo, bar = assigned_names()

Slide 40

Slide 40 text

foo = d['foo'] bar = d['bar'] ↓ foo, bar = unpack_keys(d) for foo, bar in unpack_keys([{‘foo’: 1, ‘bar’: 2}, …]):

Slide 41

Slide 41 text

thing[‘foo’] = x.foo bar = x.bar ↓ thing[‘foo’], bar = unpack_attrs(x)

Slide 42

Slide 42 text

dict(foo=foo, bar=bar, spam=thing()) ↓ dict_of(thing.foo, thing[‘bar’](1), spam=thing())

Slide 43

Slide 43 text

None if foo is None else foo.bar() ↓ maybe(foo).bar()

Slide 44

Slide 44 text

def foo(): bar() def bar(): 1/0 foo()

Slide 45

Slide 45 text

def foo(): bar() def bar(): 1/0 foo() Traceback (most recent call last): File "/my/script.py", line 7, in foo() File "/my/script.py", line 2, in foo bar() File "/my/script.py", line 5, in bar 1/0 ZeroDivisionError: division by zero

Slide 46

Slide 46 text

Slide 47

Slide 47 text

Slide 48

Slide 48 text

def foo(): x = 1 y = 2 return 3 * bar(x) def bar(): previous = inspect.currentframe().f_back previous.f_locals: {'x': 1, 'y': 2} previous.f_code:


previous.f_lineno: 4

Slide 49

Slide 49 text

def foo(): x = 1 y = 2 return 3 * bar(x) def bar(): previous = inspect.currentframe().f_back previous.f_locals: {'x': 1, 'y': 2} previous.f_code:


previous.f_lineno: 4

Slide 50

Slide 50 text

def foo(): x = 1 y = 2 return 3 * bar(x) def bar(): previous = inspect.currentframe().f_back f = open(previous.f_code.co_filename) lines = f.readlines() line = lines[previous.f_lineno - 1].strip() node = ast.parse(line) # find Call node... => Call(func=Name(id='bar'), ...)

Slide 51

Slide 51 text

def foo(): x = 1 y = bar return 3 * y(x) def bar(): previous = inspect.currentframe().f_back f = open(previous.f_code.co_filename) lines = f.readlines() line = lines[previous.f_lineno - 1].strip() node = ast.parse(line) # find Call node... previous.f_locals: {'y': , ...} => Call(func=Name(id='y'), ...)

Slide 52

Slide 52 text

def foo(): x = 1 y = 2 bar(x, y) @spell def bar(frame_info, ...): ... args: (1, 2) frame_info.call: Call(func=Name(id='bar'), args=[Name(id='x'), Name(id='y')])

Slide 53

Slide 53 text

def foo(): x = 1 y = 2 bar(x, y) @spell def bar(frame_info, ...): ... args: (1, 2) frame_info.call: Call(func=Name(id='bar'), args=[Name(id='x'), Name(id='y')])

Slide 54

Slide 54 text

dict_of(foo, bar) == dict(foo=foo, bar=bar) @spell def dict_of(frame_info, *values): return { arg.id: value for arg, value in zip(frame_info.call.args, values) }

Slide 55

Slide 55 text

import sys from inspect import stack def trace(frame, event, arg): ... sys.settrace(trace)

Slide 56

Slide 56 text

import sys from inspect import stack def trace(frame, event, arg): if event == 'call': print(' ' * len(stack()), frame.f_code.co_name) sys.settrace(trace)

Slide 57

Slide 57 text

def main(): for x in range(3): foo() def foo(): bar() def bar(): pass main()

Slide 58

Slide 58 text

def main(): for x in range(3): foo() def foo(): bar() def bar(): pass main() main foo bar foo bar foo bar

Slide 59

Slide 59 text

Open Space: Creative ideas specific to Python 13:45 - 15:15