Python потребляет много памяти, или как уменьшить размер объектов.

Заур Шибзухов (МПГУ, профессор) @ Moscow Python Meetup 73 (Online)

"Объекты в CPython занимают в памяти больше места, чем могли бы. Можно ли это исправить, в каких случаях и как".

Видео: http://www.moscowpython.ru/meetup/73/python-memory2/

Transcript

1. Python: Python: , ,
2. Python , ?
3. Dict Dict In [2]: ob = {'x':1, 'y':2, 'z':3} a = ob['x'] 3.6: Compact Dict ( PyPy )
4. dict dict In [3]: print('sizeof:', sys.getsizeof(ob)) 1 000 000 232 10 000 000 2320 sizeof: 232
5. ! ! In [4]: class Point: # def __init__(self, x, y, z): self.x = x self.y = y self.z = z ob = Point(1,2,3) a = ob.x In [5]: print(' ✂ ✄ ☎ ✆ ✝ ✞ :', sys.getsizeof(ob), sys.getsizeof(ob.__dict__)) (__dict__) . PEP 412: Key-Sharing Dic ✁ onary, Python 3.3+ ✂ ✄ ☎ ✆ ✝ ✞ : 48 104
6. " # $ " # $ ------------------------ PyGC_Head 16 ✟ ✁ ✂ ------------------------ PyObject_HEAD 16 ✟ ✁ ✂ __weakref__ 8 ✟ ✁ ✂ __dict__ 8 ✟ ✁ ✂ ------------------------ ✄ ☎ ✆ ✝ ✞ : 48 ✠ ✡ ☛ ☞ % # & % # & 1 000 000 152 10 000 000 1520
7. ! + ! + __slots__ In [6]: class Point: __slots__ = 'x', 'y', 'z' def __init__(self, x, y, z): self.x = x self.y = y self.z = z
8. In [7]: ob = Point(1,2,3) print(' ✂ ✄ ☎ ✆ ✝ ✞ :', sys.getsizeof(ob)) print(' ☛ ☞ ✡ ✌ __dict__?', hasattr(ob, '__dict__')) print(' ☛ ☞ ✡ ✌ __weakref__?', hasattr(ob, '__weakref__')) ✂ ✄ ☎ ✆ ✝ ✞ : 56 ☛ ☞ ✡ ✌ __dict__? False ☛ ☞ ✡ ✌ __weakref__? False
9. " $ " $ ------------------------ PyGC_Head 16 ✟ ✁ ✂ ------------------------ PyObject_HEAD 16 ✟ ✁ ✂ x 8 ✟ ✁ ✂ y 8 ✟ ✁ ✂ z 8 ✟ ✁ ✂ ------------------------ ✄ ☎ ✆ ✝ ✞ : 56 ✠ ✡ ☛ ☞ ' # ' # 1 000 000 56 10 000 000 560
10. In [8]: pprint(Point.__dict__) x , y , z Point.__dict__ ! " " mappingproxy({'__doc__': None, '__init__': <function Point.__init__ at 0x7f452d08dca0>, '__module__': '__main__', '__slots__': ('x', 'y', 'z'), 'x': <member 'x' of 'Point' objects>, 'y': <member 'y' of 'Point' objects>, 'z': <member 'z' of 'Point' objects>})
11. Tuple Tuple , , . # : . . In [9]: ob = (1,2,3) x, y, z = ob[0], ob[1], ob[2] In [10]: print(" ✂ ✄ ☎ ✆ ✝ ✞ :", sys.getsizeof(ob)) ✂ ✄ ☎ ✆ ✝ ✞ : 64
12. " $ " $ ------------------------ PyGC_Head 16 ✟ ✁ ✂ ------------------------ PyObject_HEAD 16 ✟ ✁ ✂ ob_size 8 ✟ ✁ ✂ [0] 8 ✟ ✁ ✂ [1] 8 ✟ ✁ ✂ [2] 8 ✟ ✁ ✂ ------------------------ ✄ ☎ ✆ ✝ ✞ : 64 ✠ ✡ ☛ ☞ ' # ' # ( , # ! ! __slots__ ( , # ! ! __slots__ 1 000 000 64 10 000 000 640
13. Named Tuples Named Tuples tuple In [11]: Point = collections.namedtuple("Point", "x y z") ob = Point(1,2,3) x = ob.x y = ob[1]
14. " $ " $ ------------------------ PyGC_Head 16 ✟ ✁ ✂ ------------------------ PyObject_HEAD 16 ✟ ✁ ✂ ob_size 8 ✟ ✁ ✂ x 8 ✟ ✁ ✂ y 8 ✟ ✁ ✂ z 8 ✟ ✁ ✂ ------------------------ ✄ ☎ ✆ ✝ ✞ : 64 ✠ ✡ ☛ ☞
15. In [12]: pprint(Point.__dict__) mappingproxy({'__doc__': 'Point(x, y, z)', '__getnewargs__': <function Point.__getnewargs__ at 0x7f452c8701 f0>, '__module__': '__main__', '__new__': <staticmethod object at 0x7f452c873040>, '__repr__': <function Point.__repr__ at 0x7f452c8700d0>, '__slots__': (), '_asdict': <function Point._asdict at 0x7f452c870160>, '_field_defaults': {}, '_fields': ('x', 'y', 'z'), '_fields_defaults': {}, '_make': <classmethod object at 0x7f452c8730d0>, '_replace': <function Point._replace at 0x7f452c85cc10>, 'x': <_collections._tuplegetter object at 0x7f452c8730a0>, 'y': <_collections._tuplegetter object at 0x7f452c873070>, 'z': <_collections._tuplegetter object at 0x7f452c873130>})
16. Mutable Tuple Mutable Tuple namedtuple $ "% namedtuple tuple. Python tuple. & recordclass mutabletuple – "% tuple. mutabletuple tuple PyGC_Head.
17. In [13]: mt = recordclass.mutabletuple(1,2,3) print(mt) print(mt[0], mt[1:]) print(' ✂ ✄ ☎ ✆ ✝ ✞ ✍ :', 'mutabletuple:', sys.getsizeof(mt), 'tuple:', sys.getsizeof ((1,2,3))) mutabletuple(1, 2, 3) 1 mutabletuple(2, 3) ✂ ✄ ☎ ✆ ✝ ✞ ✍ : mutabletuple: 48 tuple: 64
18. " # $ " # $ ------------------------ PyObject_HEAD 16 ✟ ✁ ✂ ob_size 8 ✟ ✁ ✂ [0] 8 ✟ ✁ ✂ [1] 8 ✟ ✁ ✂ [2] 8 ✟ ✁ ✂ ------------------------ ✄ ☎ ✆ ✝ ✞ : 40 ✠ ✡ ☛ ☞
19. Recordclass: ! namedtuple Recordclass: ! namedtuple ' ( ! ✥ ec ♦ ✥ dclass mutabletuple ! " . recordclass API namedtuple API. ) " , " ( ! recordclass * , % " namedtuple " __slots__ . + # )
20. In [14]: Point = recordclass.recordclass("Point", "x y z") ob = Point(1,2,3) print(ob) print(" ✂ ✄ ☎ ✆ ✝ ✞ :", sys.getsizeof(ob)) Point(x=1, y=2, z=3) ✂ ✄ ☎ ✆ ✝ ✞ : 48
21. " # $ " # $ ------------------------ PyObject_HEAD 16 ✟ ✁ ✂ ob_size 8 ✟ ✁ ✂ x 8 ✟ ✁ ✂ y 8 ✟ ✁ ✂ z 8 ✟ ✁ ✂ ------------------------ ✄ ☎ ✆ ✝ ✞ : 48 ✠ ✡ ☛ ☞ PyGC_Head .
22. In [15]: pprint(Point.__dict__) mappingproxy({'__dict__': <property object at 0x7f452c872d60>, '__doc__': 'Point(x, y, z)', '__fields__': ('x', 'y', 'z'), '__getnewargs__': <function Point.__getnewargs__ at 0x7f452c8702 80>, '__getstate__': <function Point.__getstate__ at 0x7f452c87b040>, '__module__': '__main__', '__new__': <staticmethod object at 0x7f452c873910>, '__reduce__': <function Point.__reduce__ at 0x7f452c87b0d0>, '__repr__': <function Point.__repr__ at 0x7f452c8708b0>, '__slots__': (), '_asdict': <function Point._asdict at 0x7f452c870ca0>, '_make': <classmethod object at 0x7f452c873100>, '_replace': <function Point._replace at 0x7f452c870820>, 'x': <recordclass.mutabletuple.mutabletuple_itemgetset object at 0x7f452d08ae30>, 'y': <recordclass.mutabletuple.mutabletuple_itemgetset object at 0x7f452d08ae70>, 'z': <recordclass.mutabletuple.mutabletuple_itemgetset object at 0x7f452d08aef0>})
23. * + * + In [16]: class Point(recordclass.RecordClass): x:int y:int z:int ob = Point(1,2,3) print(Point.__annotations__) print(ob) print(" ✂ ✄ ☎ ✆ ✝ ✞ :", sys.getsizeof(ob)) {'x': <class 'int'>, 'y': <class 'int'>, 'z': <class 'int'>} Point(x=1, y=2, z=3) ✂ ✄ ☎ ✆ ✝ ✞ : 48
24. Dataobject Dataobject In [17]: class Point(recordclass.dataobject): x:int y:int z:int ob = Point(1,2,3) print(ob) print(" ✂ ✄ ☎ ✆ ✝ ✞ :", sys.getsizeof(ob)) Point(x=1, y=2, z=3) ✂ ✄ ☎ ✆ ✝ ✞ : 40
25. " # $ " # $ ------------------------ PyObject_HEAD 16 ✟ ✁ ✂ x 8 ✟ ✁ ✂ y 8 ✟ ✁ ✂ z 8 ✟ ✁ ✂ ------------------------ ✄ ☎ ✆ ✝ ✞ : 40 ✠ ✡ ☛ ☞
26. * + * + In [18]: Point = recordclass.make_dataclass("Point", {"x":int, "y":int, "z":int}) print(Point.__annotations__) ob = Point(1,2,3) print(ob) print(" ✂ ✄ ☎ ✆ ✝ ✞ :", sys.getsizeof(ob)) {'x': <class 'int'>, 'y': <class 'int'>, 'z': <class 'int'>} Point(x=1, y=2, z=3) ✂ ✄ ☎ ✆ ✝ ✞ : 40
27. Dataarray Dataarray In [19]: Point = recordclass.make_arrayclass("Point", 3) ob = Point(1,2,3) print(ob) print(" ✂ ✄ ☎ ✆ ✝ ✞ :", sys.getsizeof(ob)) Point(1, 2, 3) ✂ ✄ ☎ ✆ ✝ ✞ : 40
28. ' ' Reference Counting Cyclic Garbage Collection Python ! " Reference Counting Cyclic Garbage Collection gc ! , ! . CGC , class
29. CGC " , # : In [20]: lst = [1,2,3] lst.append(lst) print(lst) 16 % Python 3.8 32-64 % Python < 3.8 CGC ! $ PyGC_Head : [1, 2, 3, [...]]
30. CGC? CGC? % $ , # & # , % $ , # & # , # , # , In [21]: class Address: town: str street: str class Person: name: str address: Address # " " " –
31. " . " . ? ?