Upgrade to Pro — share decks privately, control downloads, hide ads and more …

Eventlet, ZeroMQ, and You

Eventlet, ZeroMQ, and You

A talk I gave at PyCon Ukraine 2011.

Andrew Godwin

October 22, 2011
Tweet

More Decks by Andrew Godwin

Other Decks in Programming

Transcript

  1. import eventlet from eventlet.green import urllib2 urls = ['http://ep.io', 'http://t.co']

    results = [] def fetch(url): results.append(urllib2.urlopen(url).read()) for url in urls: eventlet.spawn(fetch, url)
  2. import eventlet import time def thread1(): time.sleep(5) # Does not

    yield print 'five seconds a' eventlet.spawn(thread1) time.sleep(5) # Does not yield print 'five seconds b'
  3. import eventlet import time def thread1(): eventlet.sleep(5) print 'five seconds

    a' eventlet.spawn(thread1) eventlet.sleep(5) print 'five seconds b'
  4. import urllib2 from eventlet.green import urllib2 import zmq from eventlet.green

    import zmq import socket from eventlet.green import socket
  5. from eventlet.queue import Queue q = Queue() def worker(): while

    True: print q.get() def producer(): while True: eventlet.sleep(1) q.put('lol') eventlet.spawn(worker) eventlet.spawn(producer)
  6. from eventlet import GreenPool from eventlet.pools import Pool conns =

    Pool(create=make_db_connection) threads = GreenPool(10) def querier(id): with pool.item() as conn: conn.execute('... WHERE id = %s', [id]) pool.imap(querier, [1, 2, 3, 4, 5])
  7. Experiences from production There's still race conditions and bad libraries.

    http://www.flickr.com/photos/iamdabe/4931554963/
  8. address = ('0.0.0.0', 80) family = socket.AF_INET sock = eventlet.listen(address,

    family) eventlet.serve( sock, self.handle, concurrency = 10000, )
  9. address = ('127.0.0.1', 8042) family = socket.AF_INET sock = eventlet.listen(address,

    family) management_app = ManagementApp(self) wsgi.server( sock, management_app.handle, )
  10. class Spin(Action): def handle(self, sock, read_data, path, headers): while True:

    # Sleep first eventlet.sleep(self.check_interval) # Check for another action action = self.balancer.resolve_host(self.host) if not isinstance(action, Spin): return action.handle( sock, read_data, path, headers )
  11. from eventlet.green import zmq ctx = zmq.Context() sock = ctx.socket(zmq.REQ)

    sock.connect('tcp://127.0.0.1:1234') sock.connect('tcp://127.0.0.1:4321') sock.send('hi there') print sock.recv()
  12. from eventlet.green import zmq ctx = zmq.Context() sock = ctx.socket(zmq.REP)

    sock.bind('tcp://0.0.0.0:1234') while True: message = sock.recv() sock.send('you said: %s' % message)
  13. from eventlet.green import zmq ctx = zmq.Context() sock = ctx.socket(zmq.PUB)

    sock.connect('tcp://127.0.0.1:1234') sock.send('Hello, world!')
  14. from eventlet.green import zmq ctx = zmq.Context() sock = ctx.socket(zmq.SUB)

    sock.bind('tcp://0.0.0.0:1234') while True: print sock.recv()
  15. The restrictions It's not quite too good to be true.

    http://www.flickr.com/photos/maistora/3237164755/
  16. Advanced ZeroMQ & Eventlet When one greenthread is not enough

    http://www.flickr.com/photos/maistora/3237164755/
  17. sock = ctx.socket(zmq.REQ) sock.send('help me! you're my only hope!') try:

    with Timeout(30): response = sock.recv() deal_with(response) except Timeout: cleanup()
  18. try: parts = sock.recv_multipart(flags=zmq.NOBLOCK) except zmq.ZMQError, e: if e.errno ==

    zmq.EAGAIN: eventlet.sleep(0.1) continue else: raise identities = parts[:parts.index("")] def call(identities, data): result = func(self, data) sock.send_multipart(identities + ["", result]) eventlet.spawn_n(call, identities, parts[-1])
  19. # Servers zmq.PULL -> Recieves logs zmq.REP -> Lets website

    query for log data # Clients zmq.PUSH -> Sends logs, connected to all servers