PyConZA 2012: "Stepping through CPython" by Larry Hastings

Ever wondered how CPython actually works internally? This talk will show you. We start with a simple Python program, then slowly step through CPython, showing in exhaustive detail what happens when it runs that program. Along the way we'll examine the design and implementation of various major CPython subsystems and see how they fit together. The audience should be conversant in C and Python.

The goal of the talk is to sufficiently familiarize the audience with CPython's internal structure such that a programmer versed in C and Python but having never dealt with an interpreter would be able to comfortably dive in and start hacking on CPython.

The program examined will be simple but deliberately designed to exercise most of CPython's runtime behavior. This will include loading modules implemented in C and in Python, loading bytecode cached on disk, and a cross-section of bytecodes. (For example, I only need to examine one of the BINARY_* math operands; I don't need to walk through every single one.)

Areas I expect to examine:

* built-in modules, including ones that are automatically loaded before your program starts
* bytecode, including the various implementations of the inner loop (switch statement, labels-as-values)
* the peephole optimizer
* on-disk format
* marshal
* the magic version number
* mention lnotab but probably skip the gory details
* the stack machine
* unwinding the stack after an exception (and producing tracebacks)
* contrast CPython's approach with Stackless
* All the possible fields of PyObject, an overview of fields in PyType
* built-in types
* the implementations of a few key internal types
* list, dict, tuple, str, byte, int, bool, None
* though not to the level of detail that Hettinger or Rhodes did in past talks
* interned values
* the GIL and reference counting
* weakrefs
* garbage collection
* Py_TRASHCAN
* CPython's small-block and arena allocators
* The parser, though I don't want to spend a lot of time on it (runtime is where the fun is ;)
* Internal utility functions like PyArg_Parse

I'll be giving the talk based on CPython 3.3.

Transcript

1. Stepping Through CPython Larry Hastings larry@hastings.org PyCon ZA 2012
2. Lack Of Introduction “Extreme” Architectural sightseeing tour You = Core Developer about Python 3.3.0 in C
3. Stepping? Stepping Running
4. Directory Organization documentation Doc parser Grammar platform Mac PCbuild miscellany Misc Tools Include Lib Modules Objects Python
5. The Program import sys import colorsys def foo(a, b): return a * b print(foo(6, 7)) class C: pass c = C()
6. Down The Rabbit Hole % gdb python3 (gdb) break main (gdb) run the_program.py Breakpoint 1, main (argc=2, argv=0x7fffffffe108) at ./Modules/python.c:25 25 wchar_t **argv_copy = (wchar_t **)PyMem_Malloc(sizeof(wchar_t*)*argc);
7. main Modules/python.c Copies argv sets locale calls Py_Main
8. Memory Management Two “families” PyMem_ PyObject_ memory malloc, realloc, free don't call malloc()!
9. PyMem_ Include/pymem.h Objects/object.c PyMem_Malloc() malloc() #define PyMem_MALLOC malloc() PYMALLOC_DEBUG → _PyMem_DebugMalloc()
10. Breakpoint 2 (gdb) break Py_Main (gdb) c Breakpoint 2, Py_Main (argc=2, argv=0x828010) at Modules/main.c:319 319 wchar_t *command = NULL;
11. Py_Main Modules/main.c _PyOS_Getopt Py_GETENV stdio Py_Initialize run script / -m module / -c command
12. Breakpoint 3 (gdb) break _PyObject_New (gdb) c Breakpoint 3, _PyObject_New (tp=0x7c0ec0) at Objects/object.c:241 241 op = (PyObject *) PyObject_MALLOC(_PyObject_SIZE(tp)); (gdb) clear _PyObject_New Deleted breakpoint 3
13. PyObject Include/object.h Objects/object.c typedef struct _object { _PyObject_HEAD_EXTRA Py_ssize_t ob_refcnt; struct _typeobject *ob_type; } PyObject;
14. PyObject Include/object.h Objects/object.c typedef struct _object { _PyObject_HEAD_EXTRA Py_ssize_t ob_refcnt; struct _typeobject *ob_type; } PyObject;
15. PyObject Include/object.h Objects/object.c typedef struct _object { _PyObject_HEAD_EXTRA Py_ssize_t ob_refcnt; struct _typeobject *ob_type; } PyObject;
16. PyObject Include/object.h Objects/object.c typedef struct _object { _PyObject_HEAD_EXTRA Py_ssize_t ob_refcnt; struct _typeobject *ob_type; } PyObject;
17. PyTypeObject typedef struct _typeobject { PyObject_VAR_HEAD const char *tp_name; Py_ssize_t tp_basicsize, tp_itemsize; destructor tp_dealloc; printfunc tp_print; getattrfunc tp_getattr; setattrfunc tp_setattr; void *tp_reserved; reprfunc tp_repr; PyNumberMethods *tp_as_number; PySequenceMethods *tp_as_sequence; PyMappingMethods *tp_as_mapping; hashfunc tp_hash; ternaryfunc tp_call; reprfunc tp_str; getattrofunc tp_getattro; setattrofunc tp_setattro; inquiry tp_clear; richcmpfunc tp_richcompare; Py_ssize_t tp_weaklistoffset; getiterfunc tp_iter; iternextfunc tp_iternext; struct PyMethodDef *tp_methods; struct PyMemberDef *tp_members; struct PyGetSetDef *tp_getset; struct _typeobject *tp_base; PyObject *tp_dict; descrgetfunc tp_descr_get; descrsetfunc tp_descr_set; Py_ssize_t tp_dictoffset; initproc tp_init; allocfunc tp_alloc; newfunc tp_new; freefunc tp_free; inquiry tp_is_gc; PyObject *tp_bases; PyObject *tp_mro; PyObject *tp_weaklist; destructor tp_del; unsigned int tp_version_tag; #ifdef COUNT_ALLOCS Py_ssize_t tp_allocs; Py_ssize_t tp_frees; Py_ssize_t tp_maxalloc; struct _typeobject *tp_prev; struct _typeobject *tp_next; #endif } PyTypeObject;
18. Protocols In C Include/abstract.h Objects/abstract.c Object Buffer tp_as_buffer Number tp_as_number Mappingtp_as_mapping Sequence tp_as_sequence “fast” sequence
19. Reference Counting Py_INCREF(o) Py_XINCREF(o) Py_DECREF(o) Py_XDECREF(o) Py_CLEAR(o)
20. PyObject_ memory Include/objimpl.h Objects/obmalloc.c #define WITH_PYMALLOC PyObject_Malloc() custom small block allocator PYMALLOC_DEBUG → _PyObject_DebugMalloc
21. PyObject_ memory #define PyObject_MALLOC PyObject_Malloc PYMALLOC_DEBUG → _PyObject_DebugMalloc ! WITH_PYMALLOC → PyMem_MALLOC
22. PyObject_ small block allocator address 256k struct arena_object s t
23. PyObject_ small block allocator address 256k
24. PyObject_ small block allocator 4k 4k 4k 4k 4k 4k 4k 4k 4k 4k 4k 4k 4k 4k 4k 4k 4k 4k 4k 4k 4k 4k 4k 4k
25. PyObject_ small block allocator 4k 4k 4k 4k 4k 4k 4k 4k 4k 4k 4k 4k 4k 4k 4k 4k 4k 4k 4k 4k 4k 4k 4k 4k
26. PyObject_ small block allocator 4k
27. PyObject_ small block allocator 4k
28. PyObject_ small block allocator struct pool_header
29. PyObject_ small block allocator struct pool_header freelist
30. PyObject_ small block allocator struct pool_header struct pool_hea struct pool_hea ol_header ol_header struct pool_header
31. PyObject_ small block allocator struct po struct po struct pool_header struct pool_header struct pool_header struct pool_header struct pool_header struct pool_header struct pool_header struct pool_header static poolp usedpools[]
32. Breakpoint 4 (gdb) b PyEval_EvalFrameEx (gdb) c Breakpoint 4, PyEval_EvalFrameEx (f=0x85e360, throwflag=0) at Python/ceval.c:808 808 PyObject *retval = NULL; /* Return value */ (gdb) clear PyEval_EvalFrameEx Deleted breakpoint 4
33. PyEval_EvalFrameEx Python/ceval.c “The big eval loop” 2300 lines Stack-based virtual machine for bytecodes
34. Registers register PyObject **stack_pointer; /* value stack free slot */ register unsigned char *next_instr; register int opcode; /* Current opcode */ register int oparg; /* Current opcode argument, if any */ register enum why_code why; /* Reason for block stack unwind */ register int err; /* Error status -- nonzero if error */ register PyObject *x; /* Result object -- NULL if error */ register PyObject *v; /* Temporary objects popped off stack */ register PyObject *w; register PyObject *u; register PyObject *t; register PyObject **fastlocals, **freevars; PyObject *retval = NULL; /* Return value */
35. Opcodes Include/opcode. h STOP_CODE 0 POP_TOP 1 ROT_TWO 2 ROT_THREE 3 DUP_TOP 4 DUP_TOP_TWO 5 NOP 9 UNARY_POSITIVE 10 UNARY_NEGATIVE 11 UNARY_NOT 12 UNARY_INVERT 15 BINARY_POWER 19 BINARY_MULTIPLY 20 BINARY_MODULO 22 BINARY_ADD 23 BINARY_SUBTRACT 24 BINARY_SUBSCR 25 INPLACE_FLOOR_DIVIDE 28 INPLACE_TRUE_DIVIDE 29 STORE_MAP 54 INPLACE_ADD 55 INPLACE_SUBTRACT 56 INPLACE_MULTIPLY 57 INPLACE_MODULO 59 STORE_SUBSCR 60 DELETE_SUBSCR 61 BINARY_LSHIFT 62 BINARY_RSHIFT 63 BINARY_AND 64 BINARY_XOR 65 BINARY_OR 66 INPLACE_POWER 67 GET_ITER 68 STORE_LOCALS 69 PRINT_EXPR 70 LOAD_BUILD_CLASS 71 INPLACE_LSHIFT 75 INPLACE_RSHIFT 76 INPLACE_AND 77 STORE_ATTR 95 DELETE_ATTR 96 STORE_GLOBAL 97 DELETE_GLOBAL 98 LOAD_CONST 100 LOAD_NAME 101 BUILD_TUPLE 102 BUILD_LIST 103 BUILD_SET 104 BUILD_MAP 105 LOAD_ATTR 106 COMPARE_OP 107 IMPORT_NAME 108 IMPORT_FROM 109 JUMP_FORWARD 110 JUMP_IF_FALSE_OR_POP 111 JUMP_IF_TRUE_OR_POP 112
36. Traditional switch loop for (;;) { switch (*bytecode_pointer++) { case BINARY_ADD: /* code for binary add */ break; case BINARY_SUBTRACT: /* etc */ break; }
37. “Faster opcode dispatch on gcc” Issue 4753 gcc labels as values, computed gotos void *pointer = &&label; goto *pointer; exit(0); switch (value) { case label: printf(“foo\n”);
38. “Faster opcode dispatch on gcc” Issue 4753 gcc labels as values, computed gotos void *pointer = &&label; goto *pointer; exit(0); switch (value) { case label: printf(“foo\n”);
39. Bytecode import sys import colorsys def foo(a, b): return a * b print(foo(6, 7)) class C: pass c = C()
40. Bytecode import sys import colorsys def foo(a, b): return a * b print(foo(6, 7)) class C: pass c = C() 1 0 LOAD_CONST 0 (0) 3 LOAD_CONST 1 (None) 6 IMPORT_NAME 0 (sys) 9 STORE_NAME 0 (sys)
41. Preloaded modules % python3 >>> import sys >>> sorted(sys.modules.keys()) ['__main__', '_bisect', '_codecs', '_collections', '_frozen_importlib', '_functools', '_heapq', '_imp', '_io', '_locale', '_sre', '_sysconfigdata', '_thread', '_warnings', '_weakref', '_weakrefset', 'abc', 'bisect', 'builtins', 'codecs', 'collections', 'collections.abc', 'copyreg', 'encodings', 'encodings.aliases', 'encodings.latin_1', 'encodings.utf_8', 'errno', 'functools', 'genericpath', 'heapq', 'io', 'itertools', 'keyword', 'locale', 'marshal', 'operator', 'os', 'os.path', 'posix', 'posixpath', 're', 'readline', 'reprlib',
42. Modules in C Modules/*module.c Py_Initialize (Python/pythonrun.c) Extension modules “Extending and Embedding the Python Interpreter” Ned Batchelder “A Whirlwind Excursion through Python C Extensions”
43. Bytecode import sys import colorsys def foo(a, b): return a * b print(foo(6, 7)) class C: pass c = C() 1 0 LOAD_CONST 0 (0) 3 LOAD_CONST 1 (None) 6 IMPORT_NAME 0 (sys) 9 STORE_NAME 0 (sys)
44. Bytecode import sys import colorsys def foo(a, b): return a * b print(foo(6, 7)) class C: pass c = C() 2 12 LOAD_CONST 0 (0) 15 LOAD_CONST 1 (None) 18 IMPORT_NAME 1 (colorsys) 21 STORE_NAME 1 (colorsys)
45. Lib/colorsys.py __all__ = ["rgb_to_yiq","yiq_to_rgb","rgb_to_hls","hls_to_rgb", "rgb_to_hsv","hsv_to_rgb"] # Some floating point constants ONE_THIRD = 1.0/3.0 ONE_SIXTH = 1.0/6.0 TWO_THIRD = 2.0/3.0 # YIQ: used by composite video signals (linear combinations of RGB) # Y: perceived grey level (0.0 == black, 1.0 == white) # I, Q: color components def rgb_to_yiq(r, g, b): y = 0.30*r + 0.59*g + 0.11*b i = 0.60*r - 0.28*g - 0.32*b q = 0.21*r - 0.52*g + 0.31*b return (y, i, q) def yiq_to_rgb(y, i, q): r = y + 0.948262*i + 0.624013*q
46. Lib/__pycache__/colorsys.pyc 0000000 0c9e 0a0d 6b93 5067 0e6b 0000 0063 0000 0000010 0000 0000 0000 0000 0600 0000 4000 0000 0000020 7300 0088 0000 0064 5a00 0000 0164 6400 0000030 0002 0364 6400 0004 0564 6400 0006 0667 0000040 5a00 0001 1464 5a00 0002 1564 5a00 0003 0000050 1664 5a00 0004 0b64 6400 0001 0084 5a00 0000060 0005 0c64 6400 0002 0084 5a00 0006 0d64 0000070 6400 0003 0084 5a00 0007 0e64 6400 0004 0000080 0084 5a00 0008 0f64 6400 0010 0084 5a00 0000090 0009 1164 6400 0005 0084 5a00 000a 1264 00000a0 6400 0006 0084 5a00 000b 1364 5300 1728 00000b0 0000 7500 024a 0000 6f43 766e 7265 6973 00000c0 6e6f 6620 6e75 7463 6f69 736e 6220 7465 00000d0 6577 6e65 5220 4247 6120 646e 6f20 6874 00000e0 7265 6320 6c6f 726f 7320 7379 6574 736d 00000f0 0a2e 540a 6968 2073 6f6d 7564 656c 2073 0000100 7270 766f 6469 7365 7420 6f77 6620 6e75 0000110 7463 6f69 736e 6620 726f 6520 6361 2068 0000120 6f63 6f6c 2072 7973 7473 6d65 4120 4342 0000130 0a3a 200a 7220 6267 745f 5f6f 6261 2863 0000140 2c72 6720 202c 2962 2d20 3e2d 6120 202c 0000150 2c62 6320 200a 6120 6362 745f 5f6f 6772
47. .pyc marshal module Include/marshal.h Python/marshal.c 0000000 0c9e 0a0d 6b93 5067 “magic tag” Lib/importlib/_bootstrap.py 3230 | 0x0a0d0000 0C 9e 10 13 st_mtime of .py
48. Bytecode import sys import colorsys def foo(a, b): return a * b print(foo(6, 7)) class C: pass c = C() 3 24 LOAD_CONST 2 (<code object foo>) 27 MAKE_FUNCTION 0 30 STORE_NAME 2 (foo)
49. Bytecode import sys import colorsys def foo(a, b): return a * b print(foo(6, 7)) class C: pass c = C() 5 33 LOAD_NAME 3 (print) 36 LOAD_NAME 2 (foo) 39 LOAD_CONST 3 (6) 42 LOAD_CONST 4 (7) 45 CALL_FUNCTION 2 48 CALL_FUNCTION 1 51 POP_TOP
50. Bytecode import sys import colorsys def foo(a, b): return a * b print(foo(6, 7)) class C: pass c = C() 6 52 LOAD_BUILD_CLASS 53 LOAD_CONST 5 (<code object C>) 56 MAKE_FUNCTION 0 59 LOAD_CONST 6 ('C') 62 CALL_FUNCTION 2 65 STORE_NAME 4 (C)
51. Bytecode import sys import colorsys def foo(a, b): return a * b print(foo(6, 7)) class C: pass c = C() 8 68 LOAD_NAME 4 (C) 71 CALL_FUNCTION 0 74 STORE_NAME 5 (c) 77 LOAD_CONST 1 (None) 80 RETURN_VALUE
52. Bytecode for foo() import sys import colorsys def foo(a, b): return a * b print(foo(6, 7)) class C: pass c = C() 4 0 LOAD_FAST 0 (a) 3 LOAD_FAST 1 (b) 6 BINARY_MULTIPLY 7 RETURN_VALUE
53. BINARY_MULTIPLY TARGET(BINARY_MULTIPLY) w = POP(); v = TOP(); x = PyNumber_Multiply(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); if (x != NULL) DISPATCH(); break;
54. BINARY_MULTIPLY TARGET(BINARY_MULTIPLY) w = POP(); v = TOP(); x = PyNumber_Multiply(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); if (x != NULL) DISPATCH(); break;
55. BINARY_MULTIPLY TARGET(BINARY_MULTIPLY) w = POP(); v = TOP(); x = PyNumber_Multiply(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); if (x != NULL) DISPATCH(); break;
56. BINARY_MULTIPLY TARGET(BINARY_MULTIPLY) w = POP(); v = TOP(); x = PyNumber_Multiply(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); if (x != NULL) DISPATCH(); break;
57. PyNumber_Multiply PyObject * PyNumber_Multiply(PyObject *v, PyObject *w) { PyObject *result = binary_op1(v, w, NB_SLOT(nb_multiply)); if (result == Py_NotImplemented) { PySequenceMethods *mv = v->ob_type- >tp_as_sequence; PySequenceMethods *mw = w->ob_type- >tp_as_sequence; Py_DECREF(result); if (mv && mv->sq_repeat) { return sequence_repeat(mv->sq_repeat, v, w); } else if (mw && mw->sq_repeat) { return sequence_repeat(mw->sq_repeat, w, v); } result = binop_type_error(v, w, "*");
58. BINARY_MULTIPLY TARGET(BINARY_MULTIPLY) w = POP(); v = TOP(); x = PyNumber_Multiply(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); if (x != NULL) DISPATCH(); break;
59. BINARY_MULTIPLY TARGET(BINARY_MULTIPLY) w = POP(); v = TOP(); x = PyNumber_Multiply(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); if (x != NULL) DISPATCH(); break;
60. BINARY_MULTIPLY TARGET(BINARY_MULTIPLY) w = POP(); v = TOP(); x = PyNumber_Multiply(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); if (x != NULL) DISPATCH(); break;
61. Inferior 1 exited normally (gdb) c Continuing. 42 [Inferior 1 (process 18377) exited normally] (gdb)
62. The End Larry Hastings larry@hastings.org radiofreepython.co