Python Magic Methods

823230e10f3221b462ca3a13c3edbb9f?s=47 Dreampuf
April 17, 2013

Python Magic Methods

http://docs.python.org/2/reference/datamodel.html
• A Guide to Python's Magic Methods - Rafe Kettler 
http://www.rafekettler.com/magicmethods.html
• Pickle: An interesting stack language - Alexandre
http://peadrop.com/blog/2007/06/18/pickle-an-interesting-stack-language/
• Flask, Django, Tornado, sh, Pipe, Gunicorn, Sqlalchemy

823230e10f3221b462ca3a13c3edbb9f?s=128

Dreampuf

April 17, 2013
Tweet

Transcript

  1. 4.

    class Parser(object): def __init__(self, mesg_class, cfg, source): self.mesg_class = mesg_class

    self.cfg = cfg if hasattr(source, "recv"): self.unreader = SocketUnreader(source) else: self.unreader = IterUnreader(source) self.mesg = None # request counter (for keepalive connetions) self.req_count = 0 def __iter__(self): return self def __next__(self): # Stop if HTTP dictates a stop. if self.mesg and self.mesg.should_close(): raise StopIteration() # Discard any unread body of the previous message if self.mesg: data = self.mesg.body.read(8192) while data: data = self.mesg.body.read(8192) # Parse the next request self.req_count += 1 self.mesg = self.mesg_class(self.cfg, self.unreader, self.req_count) if not self.mesg: raise StopIteration() return self.mesg next = __next__
  2. 5.

    class WSGIApplication(web.Application): def __call__(self, environ, start_response): handler = web.Application.__call__(self, HTTPRequest(environ))

    assert handler._finished reason = handler._reason status = str(handler._status_code) + " " + reason headers = list(handler._headers.get_all()) if hasattr(handler, "_new_cookie"): for cookie in handler._new_cookie.values(): headers.append(("Set-Cookie", cookie.OutputString(None))) start_response(status, [(native_str(k), native_str(v)) for (k, v) in headers]) return handler._write_buffer
  3. 6.

    class WSGIHandler(base.BaseHandler): initLock = Lock() request_class = WSGIRequest def __call__(self,

    environ, start_response): # Set up middleware if needed. We couldn't do this earlier, because # settings weren't available. if self._request_middleware is None: with self.initLock: try: # Check that middleware is still uninitialised. if self._request_middleware is None: self.load_middleware() except: # Unload whatever middleware we got self._request_middleware = None raise set_script_prefix(base.get_script_name(environ)) signals.request_started.send(sender=self.__class__) try: request = self.request_class(environ) except UnicodeDecodeError: logger.warning('Bad Request (UnicodeDecodeError)', exc_info=sys.exc_info(), extra={ 'status_code': 400, } ) response = http.HttpResponseBadRequest() else: response = self.get_response(request)
  4. 7.

    class WSGIHandler(base.BaseHandler): initLock = Lock() request_class = WSGIRequest def __call__(self,

    environ, start_response): # Set up middleware if needed. We couldn't do this earlier, because # settings weren't available. if self._request_middleware is None: with self.initLock: try: # Check that middleware is still uninitialised. if self._request_middleware is None: self.load_middleware() except: # Unload whatever middleware we got self._request_middleware = None raise set_script_prefix(base.get_script_name(environ)) signals.request_started.send(sender=self.__class__) try: request = self.request_class(environ) except UnicodeDecodeError: logger.warning('Bad Request (UnicodeDecodeError)', exc_info=sys.exc_info(), extra={ 'status_code': 400, } ) response = http.HttpResponseBadRequest() else: response = self.get_response(request)
  5. 8.

    def handle_request(self, listener, req, client, addr): environ = {} resp

    = None self.cfg.pre_request(self, req) resp, environ = wsgi.create(req, client, addr, listener.getsockname(), self.cfg) respiter = self.wsgi(environ, resp.start_response) try: if isinstance(respiter, environ['wsgi.file_wrapper']): resp.write_file(respiter) else: for item in respiter: resp.write(item) resp.close() finally: if hasattr(respiter, "close"): respiter.close()
  6. 9.

    class StreamReader(Codec): #... def __iter__(self): return self def __getattr__(self, name,

    getattr=getattr): return getattr(self.stream, name) def __enter__(self): return self def __exit__(self, type, value, tb): self.stream.close() stdlib: codecs.py
  7. 11.
  8. 13.

    from os.path import join class FileObject(object): '''Wrapper for file objects

    to make sure the file gets closed on deletion.''' def __init__(self, filepath='~', filename='sample.txt'): # open a file filename in filepath in read and write mode self.file = open(join(filepath, filename), 'r+') # super(FileObject, self).__init__(*) def __del__(self): self.file.close() del self.file
  9. 14.

    class MinervaOpenAPI(API): HOST = OPENAPI_HOST @GET("tags/%(tag)s") def get_tag(tag): pass @GET("tags/%(tag)s/feeds/filter:%(filter)s")

    def get_tag_feeds(tag, filter=str): pass @GET("tags/%(tag)s/feeds/count/filter:%(filter)s") def get_tag_feeds_count(tag, filter=str): pass @POST("tags/synonym_list") def get_synonym_list(taglist=str): pass api = MinervaOpenAPI() api.get_tag("NBA")
  10. 15.

    class APIMeta(type): def __new__(cls, clsname, bases, dct): for name, method

    in dct.iteritems(): if not isinstance(method, types.FunctionType) or not hasattr(method, "url"): continue args, varags, varkw, defaults = inspect.getargspec(method) kw_len = 0 if defaults is None else len(defaults) required = args[:len(args)-kw_len] optional_type = dict(zip(args[kw_len:], defaults)) if kw_len > 0 else {} dct[name] = APIMethod(name=method.func_name, url=HOST + method.url, method=method.method, dsc=method.__doc__, required=required, optional_type=optional_type) return super(APIMeta, cls).__new__(cls, clsname, bases, dct) class API(object): HOST = "http://localhost:8080/" __metaclass__ = APIMeta
  11. 18.

    __cmp__(self, other) cmp(a, b) __eq__(self, other) a == b __ne__(self,

    other) a != b __lt__(self, other) a < b __gt__(self, other) a > b __le__(self, other) a <= b __ge__(self, other) a >= b Comparison operators
  12. 19.

    class Word(str): '''Class for words, defining comparison based on word

    length.''' def __new__(cls, word): # Note that we have to use __new__. This is because str is an immutable # type, so we have to initialize it early (at creation) if ' ' in word: print "Value contains spaces. Truncating to first space." word = word[:word.index(' ')] # Word is now all chars before first space return str.__new__(cls, word) def __gt__(self, other): return len(self) > len(other) def __lt__(self, other): return len(self) < len(other) def __ge__(self, other): return len(self) >= len(other) def __le__(self, other): return len(self) <= len(other) def __eq__(self, other): return super(str, self).__eq__(other)
  13. 20.

    @total_ordering class Word(str): '''Class for words, defining comparison based on

    word length.''' def __new__(cls, word): # Note that we have to use __new__. This is because str is an immutable # type, so we have to initialize it early (at creation) if ' ' in word: print "Value contains spaces. Truncating to first space." word = word[:word.index(' ')] # Word is now all chars before first space return str.__new__(cls, word) def __le__(self, other): return len(self) <= len(other) def __eq__(self, other): return super(str, self).__eq__(other) http://docs.python.org/2/library/functools.html#functools.total_ordering
  14. 21.

    __pos__(self) +v __neg__(self) -v __abs__(self) abs(v) __invert__(self) ~v __round__(self, n)

    round(v) __floor__(self) math.floor(v) __ceil__(self) math.ceil(v) __trunc__(self) math.trunc(v) Unary operators
  15. 22.

    __add__(self, other) a+b __sub__(self, other) a-b __mul__(self, other) a*b __floordiv__(self,

    other) a // b __div__(self, other) a / b __truediv__(self, other) a / b (from __furture__ import division) __mod__(self, other) a % b __divmod__(self, other) divmod(a, b) __pow__ a ** b __lshift__(self, other) a << b __rshift__(self, other) a >> b __and__(self, other) a & b __or__(self, other) a | b __xor__(self, other) a ^ b Arithmetic operators
  16. 23.

    __radd__(self, other) b+a __rsub__(self, other) b-a __rmul__(self, other) b*a __rfloordiv__(self,

    other) b // a __rdiv__(self, other) b / a __rtruediv__(self, other) b / a (from __furture__ import division) __rmod__(self, other) b % a __rdivmod__(self, other) divmod(b, a) __rpow__ b ** a __rlshift__(self, other) b << a __rrshift__(self, other) b << a __rand__(self, other) b & a __ror__(self, other) b | a __rxor__(self, other) b ^ a Reflected arithmetic operators
  17. 24.

    __iadd__(self, other) a+=b __isub__(self, other) a-=b __imul__(self, other) a*=b __ifloordiv__(self,

    other) a //= b __idiv__(self, other) a /= b __itruediv__(self, other) a/=b (from __furture__ import division) __imod_(self, other) a %= b divmod(a, b) __ipow__ a **= b __ilshift__(self, other) a <<= b __irshift__(self, other) a >>= b __iand__(self, other) a &= b __ior__(self, other) a |= b __ixor__(self, other) a ^= b Assignment operators
  18. 25.

    sum(select(where(take_while(fib(), lambda x: x < 1000000) lambda x: x %

    2), lambda x: x * x)) fib() | take_while(lambda x: x < 1000000) \ | where(lambda x: x % 2) \ | select(lambda x: x * x) \ | sum() Pipe Module https://github.com/JulienPalard/Pipe
  19. 26.

    __int__(self) int(v) __long__(self) long(v) __float__(self) float(v) __complex__(self) complex(v) __oct__(self) oct(v)

    __hex__(self) hex(v) __index__(self) slice(v) __trunc__(self) math.trunc(v) __coerce__(self, other) coerce(3, 3.0) Type conversion
  20. 27.

    In [55]: slice(5) Out[55]: slice(None, 5, None) In [56]: range(10)[slice(5)]

    Out[56]: [0, 1, 2, 3, 4] In [57]: (5).__index__ Out[57]: <method-wrapper '__index__' of int object at 0x7fcd2ac0ffd8> http://www.python.org/dev/peps/pep-0357/
  21. 28.

    __str__(self) str(v) __repr__(self) repr(v) __unicode__(self) unicode(v) __format__(self, formatstr) v.format(“123”) __hash__(self)

    hash(v) __nonzero__(self) bool(v) __dir__(self) div(v) __sizeof__(self) sys.getsizeof(v) Representing the Classes
  22. 29.

    __getattr__(self, name) v.name __setattr__(self, name, value) v.name = value __delattr__(self,

    name) del v.name __getattribute__(self, name) v.name Controlling Attribute Access http://docs.python.org/2/reference/datamodel.html#object.__getattribute__
  23. 30.

    Controlling Attribute Access def __setattr__(self, name, value): self.name = value

    # since every time an attribute is assigned, __setattr__() is called, this # is recursion. # so this really means self.__setattr__('name', value). Since the method # keeps calling itself, the recursion goes on forever causing a crash def __setattr__(self, name, value): self.__dict__[name] = value # assigning to the dict of names in the class # define custom behavior here http://docs.python.org/2/reference/datamodel.html#object.__getattribute__
  24. 31.

    Controlling Attribute Access class AccessCounter(object): '''A class that contains a

    value and implements an access counter. The counter increments each time the value is changed.''' def __init__(self, val): super(AccessCounter, self).__setattr__('counter', 0) super(AccessCounter, self).__setattr__('value', val) def __setattr__(self, name, value): if name == 'value': super(AccessCounter, self).__setattr__('counter', self.counter + 1) # Make this unconditional. # If you want to prevent other attributes to be set, raise AttributeError(name) super(AccessCounter, self).__setattr__(name, value) def __delattr__(self, name): if name == 'value': super(AccessCounter, self).__setattr__('counter', self.counter + 1) super(AccessCounter, self).__delattr__(name)]
  25. 32.

    sh Module from sh import ifconfig print(ifconfig("wlan0")) """ wlan0 Link

    encap:Ethernet HWaddr 00:00:00:00:00:00 inet addr:192.168.1.100 Bcast:192.168.1.255 Mask: 255.255.255.0 inet6 addr: ffff::ffff:ffff:ffff:fff/64 Scope:Link UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1 RX packets:0 errors:0 dropped:0 overruns:0 frame:0 TX packets:0 errors:0 dropped:0 overruns:0 carrier:0 collisions:0 txqueuelen:1000 RX bytes:0 (0 GB) TX bytes:0 (0 GB) """ from sh import git, ls, wc git.checkout("master") print(ls("-l")) longest_line = wc(__file__, "-L") from sh import tail # runs forever for line in tail("-f", "/var/log/some_log_file.log", _iter=True): print(line) http://amoffat.github.io/sh/
  26. 33.

    class SelfWrapper(ModuleType): def __init__(self, self_module, baked_args={}): self.self_module = self_module def

    __getattr__(self, name): return Command._create(name, **self.baked_args) def __call__(self, **kwargs): return SelfWrapper(self.self_module, kwargs) self = sys.modules[__name__] sys.modules[__name__] = SelfWrapper(self) sh Module
  27. 34.

    Sequences __len__(self) len(v) __getitem__(self, key) v[key], v[start:stop:step] __setitem__(self, key, value)

    v[key] = value __delitem__(self, key) del v[key] __iter__(self) iter(v) __reversed__(self) reversed(v) __contains__(self, item) item in v , item not in v __missing__(self, key)
  28. 36.

    Sequences class FunctionalList(object): '''A class wrapping a list with some

    extra functional magic, like head, tail, init, last, drop, and take.''' def __init__(self, values=None): if values is None: self.values = [] else: self.values = values def __len__(self): return len(self.values) def __getitem__(self, key): # if key is of invalid type or value, the list values will raise the error return self.values[key] def __setitem__(self, key, value): self.values[key] = value def __delitem__(self, key): del self.values[key] def __iter__(self): return iter(self.values) def __reversed__(self): return FunctionalList(reversed(self.values)) def append(self, value): self.values.append(value) # get last element return self.values[-1]
  29. 37.

    class Query(object): def slice(self, start, stop): """apply LIMIT/OFFSET to the

    ``Query`` based on a " "range and return the newly resulting ``Query``.""" pass def __getitem__(self, item): if isinstance(item, slice): start, stop, step = util.decode_slice(item) if (isinstance(start, int) and start < 0) \ or (isinstance(stop, int) and stop < 0): return list(self)[item] res = self.slice(start, stop) return list(res)[None:None:item.step] else: if item == -1: return list(self)[-1] else: return list(self[item:item + 1])[0] def __iter__(self): context = self._compile_context() context.statement.use_labels = True if self._autoflush and not self._populate_existing: self.session._autoflush() return self._execute_and_instances(context)
  30. 38.

    Collections Abstract Base Classes ABC Inherits from Abstract Methods Mixin

    Methods Container __contains__ Hashable __hash__ Iterable __iter__ Iterator Iterable next __iter__ Sized __len__ Callable __call__ Sequence Sized, Iterable, Container __getitem__, __len__ __contains__, __iter__, __reversed__, index, and count MutableSequence Sequence __getitem__, __setitem__,__delitem__, __len__,insert Inherited Sequence methods and append , reverse , extend , pop ,remove , and __iadd__ Set Sized, Iterable, Container __contains__, __iter__,__len__ __le__, __lt__, __eq__, __ne__, __gt__, __ge__, __and__, __or__,__sub__, __xor__, and isdisjoint MutableSet Set __contains__, __iter__,__len__, add, discard Inherited Set methods and clear , pop , remove , __ior__ , __iand__ ,__ixor__ , and __isub__ Mapping Sized, Iterable, Container __getitem__, __iter__,__len__ __contains__, keys, items, values, get, __eq__, and __ne__ MutableMapping Mapping __getitem__, __setitem__,__delitem__, __iter__,__len__ Inherited Mapping methods and pop , popitem , clear , update , andsetdefault MappingView Sized __len__ ItemsView MappingView, Set __contains__, __iter__ KeysView MappingView, Set __contains__, __iter__ ValuesView MappingView __contains__, __iter__
  31. 39.

    class ListBasedSet(collections.Set): def __init__(self, iterable): self.elements = lst = []

    for value in iterable: if value not in lst: lst.append(value) def __iter__(self): return iter(self.elements) def __contains__(self, value): return value in self.elements def __len__(self): return len(self.elements) s1 = ListBasedSet('abcdef') s2 = ListBasedSet('defghi') overlap = s1 & s2
  32. 41.

    Callable objects • __call__(self, [args...]) class Entity(object): '''Class to represent

    an entity. Callable to update the entity's position.''' def __init__(self, size, x, y): self.x, self.y = x, y self.size = size def __call__(self, x, y): '''Change the position of the entity.''' self.x, self.y = x, y # snip...
  33. 42.

    Callable objects • __call__(self, [args...]) In [8]: entity = Entity(5,

    0, 0) In [9]: entity(1, 1) In [10]: entity.x, entity.y Out[10]: (1, 1)
  34. 44.

    Context Managers class Closer(object): '''A context manager to automatically close

    an object with a close method in a with statement.''' def __init__(self, obj): self.obj = obj def __enter__(self): return self.obj # bound to target def __exit__(self, exception_type, exception_val, trace): try: self.obj.close() except AttributeError: # obj isn't closable print 'Not closable.' return True # exception handled successfully
  35. 45.

    from flask import g, request, session @app.route("/verify/<email>/<verify_code>", methods=["GET"]) def verify(email,

    verify_code=None): users = g.db.get("user", {}) user = users.get(email) today = date.today() for i in xrange(config.VERIFY_AVAILABILITY_DAY): t = today - timedelta(days=i) ciphertext = sha1("{0}{1}{2}".format(config.SALT, email, t.toordinal())).hexdigest() if ciphertext == verify_code: user["verified"] = True users[email] = user g.db["user"] = users g.db.sync() session["user"] = email break return render_template("success.html", user=user)
  36. 46.

    # "globlas" module from functools import partial from werkzeug.local import

    LocalStack, LocalProxy def _lookup_object(name): top = _request_ctx_stack.top if top is None: raise RuntimeError('working outside of request context') return getattr(top, name) _request_ctx_stack = LocalStack() _app_ctx_stack = LocalStack() request = LocalProxy(partial(_lookup_object, 'request')) session = LocalProxy(partial(_lookup_object, 'session')) g = LocalProxy(partial(_lookup_object, 'g')) # "app" module def wsgi_app(self, environ, start_response): with self.request_context(environ): try: response = self.full_dispatch_request() except Exception, e: response = self.make_response(self.handle_exception(e)) return response(environ, start_response)
  37. 47.

    class RequestContext(object): def __enter__(self): top = _request_ctx_stack.top app_ctx = _app_ctx_stack.top

    if app_ctx is None or app_ctx.app != self.app: app_ctx = self.app.app_context() app_ctx.push() self._implicit_app_ctx_stack.append(app_ctx) else: self._implicit_app_ctx_stack.append(None) _request_ctx_stack.push(self) self.session = self.app.open_session(self.request) if self.session is None: self.session = self.app.make_null_session() def __exit__(self, exc_type=None, exc=None, tb=None): app_ctx = self._implicit_app_ctx_stack.pop() if not self._implicit_app_ctx_stack: if exc is None: exc = sys.exc_info()[1] self.app.do_teardown_request(exc)
  38. 48.

    def test_basic_url_generation(self): app = flask.Flask(__name__) app.config['SERVER_NAME'] = 'localhost' app.config['PREFERRED_URL_SCHEME'] =

    'https' @app.route('/') def index(): pass with app.app_context(): rv = flask.url_for('index') self.assert_equal(rv, 'https://localhost/')
  39. 49.
  40. 51.

    Descriptor Objects In [19]: a = Distance() In [20]: a.foot

    = 5 In [21]: a.meter Out[21]: 1.5240185320653499
  41. 52.

    class Meter(object): '''Descriptor for a meter.''' def __init__(self, value=0.0): self.value

    = float(value) def __get__(self, instance, owner): return self.value def __set__(self, instance, value): self.value = float(value) class Foot(object): '''Descriptor for a foot.''' def __get__(self, instance, owner): return instance.meter * 3.2808 def __set__(self, instance, value): instance.meter = float(value) / 3.2808 class Distance(object): '''Class to represent distance holding two descriptors for feet and meters.''' meter = Meter() foot = Foot()
  42. 54.

    >>> b = copy.deepcopy(a) Traceback (most recent call last): File

    "<stdin>", line 1, in <module> File "/usr/local/Cellar/python/2.7.2/lib/python2.7/copy.py", line 172, in deepcopy copier = getattr(x, "__deepcopy__", None) File "<stdin>", line 3, in __getattr__ KeyError: '__deepcopy__'
  43. 55.

    try: from collections import defaultdict except ImportError: class defaultdict(dict): def

    __init__(self, default_factory=None, *a, **kw): if (default_factory is not None and not hasattr(default_factory, '__call__')): raise TypeError('first argument must be callable') dict.__init__(self, *a, **kw) self.default_factory = default_factory def __getitem__(self, key): try: return dict.__getitem__(self, key) except KeyError: return self.__missing__(key) def __copy__(self): return type(self)(self.default_factory, self) def __deepcopy__(self, memo): import copy return type(self)(self.default_factory, copy.deepcopy(self.items())) http://www.peterbe.com/plog/must__deepcopy__
  44. 57.

    Pickling import time class Slate: '''Class to store a string

    and a changelog, and forget its value when pickled.''' def __init__(self, value): self.value = value self.last_change = time.asctime() self.history = {} def change(self, new_value): # Change the value. Commit last value to history self.history[self.last_change] = self.value self.value = new_value self.last_change = time.asctime() def print_changes(self): print 'Changelog for Slate object:' for k, v in self.history.items(): print '%s\t %s' % (k, v) def __getstate__(self): # Deliberately do not return self.value or self.last_change. # We want to have a "blank slate" when we unpickle. return self.history def __setstate__(self, state): # Make self.history = state and last_change and value undefined self.history = state self.value, self.last_change = None, None
  45. 58.

    Pickling In [25]: s = Slate(123) In [26]: s.change(4) In

    [27]: s.change(5) In [28]: s.change(6) In [29]: s.print_changes() Changelog for Slate object: Tue Apr 16 22:03:02 2013 5 Tue Apr 16 22:03:01 2013 4 Tue Apr 16 22:02:50 2013 123 In [30]: pickled_s = pickle.dumps(s) In [31]: ns = pickle.loads(pickled_s) In [32]: ns.print_changes() Changelog for Slate object: Tue Apr 16 22:03:02 2013 5 Tue Apr 16 22:03:01 2013 4 Tue Apr 16 22:02:50 2013 123 In [33]: s.value, ns.value Out[33]: (6, None)
  46. 59.

    Why Pickling need reduce pik> (I1 ...> S'abc' ...> F2.0

    ...> t. (1, 'abc', 2.0) pik> (I1 ...> (I2 ...> I3 ...> tt. (1, (2, 3)) pik> (I0 ...> I1 ...> I2 ...> l. [0, 1, 2] pik> (S'red' ...> I00 ...> S'blue' ...> I01 ...> d. {'blue': True, 'red': False} pik> c__builtin__ ...> set ...> ((S'a' ...> S'a' ...> S'b' ...> ltR. set(['a', 'b']) http://aisk.me/blog/2013/02/01/170/
  47. 61.

    Why to use? Beautiful is better than ugly. Explicit is

    better than implicit. Simple is better than complex. Complex is better than complicated. Flat is better than nested. Sparse is better than dense. Readability counts. Special cases aren't special enough to break the rules. Although practicality beats purity. Errors should never pass silently. Unless explicitly silenced. In the face of ambiguity, refuse the temptation to guess.
  48. 62.
  49. 63.

    • http://docs.python.org/2/reference/datamodel.html • A Guide to Python's Magic Methods -

    Rafe Kettler http://www.rafekettler.com/magicmethods.html • Pickle: An interesting stack language - Alexandre http://peadrop.com/blog/2007/06/18/pickle-an-interesting- stack-language/ • Flask, Django, Tornado, sh, Pipe, Gunicorn, Sqlalchemy Reference & Thanks
  50. 64.