Real time webapps using python

Transcript

1. Real time webapps using Python @HURRYCANE
2. IDEA ??? PROFIT
3. IDEA realtime ??? > PROFIT
4. THE PROBLEM Queue API Workers
5. Queue API Workers THE PROBLEM x 2M
6. Queue API Workers THE PROBLEM
7. Queue API Workers Under high load only Without logs THE PROBLEM
8. Queue API Workers THE IDEA On the fly inspection
9. Queue API Workers WebApp THE IDEA Aggregator
10. REALTIME
11. THE IDEA API Workers Real time aggregator WebApp
12. THE REALTIME WEB
13. Techcrunch says the way you read your rss
14. it’s about pushing
15. RATHER THAN POLLING
16. CLIENT SERVER
17. CLIENT SERVER
18. AJAX COMET web-socks
19. API Workers Real time aggregator WebApp ZeroMQ Flask / WebSockets Gevent
20. GEVENT Co-rutine based networking library uses greenlets to provide high-level sync API uses libEV (1.0rc2)
21. application code gevent greenlet libev kernel GEVENT
22. import gevent.monkey gevent.monkey.patch_socket() # OR gevent.monkey.patch_all() import gevent import urllib2 import simplejson as json def fetch(pid): response = urllib2.urlopen('http://json-time.appspot.com/time.json') result = response.read() json_result = json.loads(result) datetime = json_result['datetime'] print 'Process ', pid, datetime return json_result['datetime'] def asynchronous(): threads = [] for i in range(1,10): threads.append(gevent.spawn(fetch, i)) gevent.joinall(threads) print 'Asynchronous:' asynchronous()
23. zeromq intelligent socket library many kinds of connection patterns multiple platforms and languages fast and small long term goal: replace linux socket apis
24. import zmq context = zmq.Context() # Socket to talk to server print "Connecting to hello world server…" socket = context.socket(zmq.REQ) socket.connect ("tcp://localhost:5555") # Do 10 requests, waiting each time for a response for request in range (10): print "Sending request ", request,"…" socket.send ("Hello") # Get the reply. message = socket.recv() print "Received reply ", request, "[", message, "]"
25. import zmq import time context = zmq.Context() socket = context.socket(zmq.REP) socket.bind("tcp://*:5555") while True: # Wait for next request from client message = socket.recv() print "Received request: ", message # Do some 'work' time.sleep (1) # Do some 'work' # Send reply back to client socket.send("World")
28. and many more
29. flask from flask import Flask app = Flask(__name__) @@app.route("/") def hello(): return "Hello World!" if __name__ == "__main__": app.run()
30. trackviz AGGREGATOR FLASK + WSGI GEVENTWEBSOCKET GEVENT REDIS
31. trackviz AGGREGATOR import logging import zmq.green as zmq from gevent import Greenlet from trackviz.aggregator import Parser logger = logging.getLogger('simplysocial.trackviz') class Aggregator(Greenlet): def __init__(self, context, redis, parser_klass=Parser): super(Aggregator, self).__init__() [...] def _run(self): logger.debug("Starting!") while True: message = self.socket.recv() self.parser.parse(message) self.socket.send("OK")
32. trackviz FLASK import os [...] app = Flask(__name__) app.secret_key = os.urandom(24) app.debug = True class VisualiseApp(object): def __init__(self, context, redis): [...] def __call__(self, environ, start_response): path = environ["PATH_INFO"] if path == "/index": WebsocketHandler(self.context, self.redis, environ["wsgi.websocket"]).handle() else: return app(environ, start_response) import views
33. trackviz websockets import json import msgpack import zmq.green as zmq class WebsocketHandler(object): def __init__(self, context, redis, ws): [...] self.socket = self.context.socket(zmq.SUB) self.socket.setsockopt(zmq.SUBSCRIBE, "index") self.socket.connect('inproc://queue') def handle(self): while True: message = self.socket.recv() # remove topic name pos = message.find(" ") message = msgpack.unpackb(message[pos+1:]) self.ws.send(json.dumps(message))
34. Q/A
35. THANKS