Share
Embed
Yield
Asynchronised operations are good. Callbacks are not. Named callbacks clutter scopes and are disruptive. Anonymous callbacks create deeply nested code and become totally unreadable—Python does not even support that because it just lacks style! We deserve better. Let’s take a look on how Python 3.4’s new “asyncio” (aka Tulip) brings elegance to async tasks.
Tzu-ping Chung
May 17, 2014
Transcript
82 slides
-
There Are Things • asyncio fundementals • asyncio.Future • Generators
(yield) • PEP 380 (yield from) • Coroutines • asyncio examples -
There’s No Time • asyncio fundementals • asyncio.Future • Generators
(yield) • PEP 380 (yield from) • Coroutines • asyncio examples -
• “Reactor” • In charge of dispatching events for tasks
• Tasks give up control when they don’t need it Event Loop -
import asyncio! ! def print_and_repeat(loop):! print('Hello World')! loop.call_later(! 2,! print_and_repeat,!
loop! )! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_repeat(loop)! loop.run_forever() https://code.google.com/p/tulip/source/browse/examples/hello_callback.py -
import asyncio! ! def print_and_repeat(loop):! print('Hello World')! loop.call_later(! 2,! print_and_repeat,!
loop! )! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_repeat(loop)! loop.run_forever() -
import asyncio! ! def print_and_repeat(loop):! print('Hello World')! loop.call_later(! 2,! print_and_repeat,!
loop! )! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_repeat(loop)! loop.run_forever() -
import asyncio! ! def print_and_repeat(loop):! print('Hello World')! loop.call_later(! 2,! print_and_repeat,!
loop! )! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_repeat(loop)! loop.run_forever() -
• Takes registration for callbacks • Calls them when it
should • Hard to persist states Event Loop -
Future • Deferred? Promises/A+? • “Run this for me and
tell me the result.” • Like concurrent.futures.Future -
def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None! )!
return f! ! def print_and_schedule(future):! print('Hello World')! f = sleep(2)! f.add_done_callback(print_and_schedule)! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_schedule()! loop.run_forever() -
def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None! )!
return f! ! def print_and_schedule(future):! print('Hello World')! f = sleep(2)! f.add_done_callback(print_and_schedule)! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_schedule()! loop.run_forever() -
def sleep(secs):! f = asyncio.Future(loop=None)! f._loop.call_later(! secs, f.set_result, None! )!
return f! ! def print_and_schedule(future):! print('Hello World')! f = sleep(2)! f.add_done_callback(print_and_schedule)! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_schedule()! loop.run_forever() -
def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None! )!
return f! ! def print_and_schedule(future):! print('Hello World')! f = sleep(2)! f.add_done_callback(print_and_schedule)! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_schedule()! loop.run_forever() -
def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None! )!
return f! ! def print_and_schedule(future):! print('Hello World')! f = sleep(2)! f.add_done_callback(print_and_schedule)! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_schedule()! loop.run_forever() -
def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None! )!
return f! ! def print_and_schedule(future):! print('Hello World')! f = sleep(2)! f.add_done_callback(print_and_schedule)! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_schedule()! loop.run_forever() -
class Future:! # …! def add_done_callback(self, fn):! # …! self._callbacks.append(fn)!
! def set_result(self, result):! # …! self._schedule_callbacks()! ! def _schedule_callbacks(self):! callbacks = self._callbacks[:]! # …! for cb in callbacks:! self._loop.call_soon(cb, self) http://hg.python.org/cpython/file/3.4/Lib/asyncio/futures.py -
def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None! )!
return f! ! def print_and_schedule(future):! print('Hello World')! f = sleep(2)! f.add_done_callback(print_and_schedule)! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_schedule(None)! loop.run_forever() -
def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None! )!
return f! ! def print_and_schedule(future):! print('Hello World')! f = sleep(2)! f.add_done_callback(print_and_schedule)! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_schedule()! loop.run_forever() -
def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None! )!
return f! ! def print_and_schedule(future):! print('Hello World')! f = sleep(2)! f.add_done_callback(print_and_schedule)! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_schedule()! loop.run_forever() called by event loop (after 2 secs) -
def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None! )!
return f! ! def print_and_schedule(*args, **kwargs):! print('Hello World')! f = sleep(2)! f.add_done_callback(print_and_schedule)! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_schedule()! loop.run_forever() calls -
future print_and_schedule event loop Initial invocation .__init__ .call_later .set_result .add_done_callback
call as callback Starts running -
Future • Encapsulates the asynchronous execution of a callable. •
Callbacks are bundled with it • Event loop calls them for you -
def producer():! yield 0! yield 1! ! g = producer()!
print(type(g)) # <type 'generator'>! ! for i in g:! print(i)! # 0! # 1 Generators -
def producer():! yield 0! yield 1! ! g = producer()!
print(type(g)) # <type 'generator'>! ! for i in g:! print(i)! # 0! # 1 Generators -
def producer():! yield 0! yield 1! ! g = producer()!
print(type(g)) # <type 'generator'>! ! for i in g:! print(i)! # 0! # 1 Generators -
def producer():! yield 0! yield 1! ! g = producer()!
print(type(g)) # <type 'generator'>! ! for i in g:! print(i)! # 0! # 1 Generators -
g = producer()! while True:! try:! i = next(g) #
Python 3! print(i)! except StopIteration:! break! # 0! # 1 -
g = producer()! while True:! try:! i = g.__next__() #
Python 3! print(i)! except StopIteration:! break! # 0! # 1 -
g = producer()! while True:! try:! i = g.send(None)! print(i)!
except StopIteration:! break! # 0! # 1 http://www.python.org/dev/peps/pep-0342/ -
def get_generator():! v = yield 0! print('Gen:', v)! v =
yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None) # Starts g.! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Con: 0 ! Gen: 1! Con: 1! Gen: 2 Outputs -
def get_generator():! v = yield 0! print('Gen:', v)! v =
yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None)! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Con: 0 ! Gen: 1! Con: 1! Gen: 2 Outputs -
def get_generator():! v = yield 0! print('Gen:', v)! v =
yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None)! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Con: 0 ! Gen: 1! Con: 1! Gen: 2 Outputs -
def get_generator():! v = yield 0! print('Gen:', v)! v =
yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None)! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Con: 0 ! Gen: 1! Con: 1! Gen: 2 Outputs -
def get_generator():! v = yield 0! print('Gen:', v)! v =
yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None)! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Con: 0 ! Gen: 1! Con: 1! Gen: 2 Outputs -
def get_generator():! v = yield 0! print('Gen:', v)! v =
yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None)! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Con: 0 ! Gen: 1! Con: 1! Gen: 2 Outputs -
def get_generator():! v = yield 0! print('Gen:', v)! v =
yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None)! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Con: 0 ! Gen: 1! Con: 1! Gen: 2 Outputs -
def get_generator():! v = yield 0! print('Gen:', v)! v =
yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None)! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Con: 0 ! Gen: 1! Con: 1! Gen: 2 Outputs -
def get_generator():! v = yield 0! print('Gen:', v)! v =
yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None)! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Con: 0 ! Gen: 1! Con: 1! Gen: 2 Outputs -
def get_generator():! v = yield 0! print('Gen:', v)! v =
yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None)! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Con: 0 ! Gen: 1! Con: 1! Gen: 2 Outputs -
def get_generator():! v = yield 0! print('Gen:', v)! v =
yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None)! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Con: 0 ! Gen: 1! Con: 1! Gen: 2 Outputs No more yielding. -
def get_generator():! v = yield 0! print('Gen:', v)! v =
yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None)! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Con: 0 ! Gen: 1! Con: 1! Gen: 2 Outputs No more yielding. -
def get_generator():! v = yield 0! print('Gen:', v)! v =
yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None)! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Con: 0 ! Gen: 1! Con: 1! Gen: 2 Outputs -
[Coroutines] allow multiple entry points for suspending and resuming execution
at certain locations. http://en.wikipedia.org/wiki/Coroutine -
def get_generator():! v = yield 0! print('Gen:', v)! v =
yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None)! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Coroutines? -
Generators • “Semi-coroutines” • Always yield to its caller •
yield is for passing back a value to parent -
Generators • “Semi-coroutines” • Always yield to its caller •
yield is for passing back a value to parent -
Generators • “Semi-coroutines” • Always yield to its caller •
yield is for passing back a value to parent Event loop Future instance -
yield from • PEP 380 and Python 3.3 • Sub-generator
delegation • yield the results from another generator -
yield from • Exceptions bubbling • Return value without round-trip
to scheduler • Cleaner/simpler API -
def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None! )!
return f! ! def print_and_schedule(future):! print('Hello World')! f = sleep(2)! f.add_done_callback(print_and_schedule)! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_schedule()! loop.run_forever() -
def print_and_schedule(future):! print('Hello World')! f = sleep(2)! f.add_done_callback(print_and_schedule)! ! if
__name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_schedule()! loop.run_forever() -
@asyncio.coroutine! def print_and_schedule():! while True:! print('Hello World')! f = sleep(2)!
yield from f! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! loop.run_until_complete(! print_and_schedule! ) -
@asyncio.coroutine! def print_and_schedule():! while True:! print('Hello World')! f = sleep(2)!
yield from f! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! loop.run_until_complete(! print_and_schedule! ) -
@asyncio.coroutine! def print_and_schedule():! while True:! print('Hello World')! f = sleep(2)!
yield from f! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! loop.run_until_complete(! print_and_schedule! ) -
@asyncio.coroutine! def sleep(s):! f = asyncio.Future()! f._loop.call_later(s, f.set_result, None)! yield
from f! ! @asyncio.coroutine! def print_and_schedule():! while True:! print('Hello World')! yield from sleep(2)! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! loop.run_until_complete(! print_and_schedule! ) -
@asyncio.coroutine! def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None!
)! yield from f! ! @asyncio.coroutine! def print_and_schedule():! while True:! print('Hello World')! yield from sleep(2) -
@asyncio.coroutine! def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None!
)! yield from f! ! @asyncio.coroutine! def print_and_schedule():! while True:! print('Hello World')! yield from sleep(2) called by event loop -
@asyncio.coroutine! def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None!
)! yield from f! ! @asyncio.coroutine! def print_and_schedule():! while True:! print('Hello World')! yield from sleep(2) -
@asyncio.coroutine! def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None!
)! yield from f! ! @asyncio.coroutine! def print_and_schedule():! while True:! print('Hello World')! yield from sleep(2) yield to event loop -
@asyncio.coroutine! def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None!
)! yield from f! ! @asyncio.coroutine! def print_and_schedule():! while True:! print('Hello World')! yield from sleep(2) Later, by event loop -
@asyncio.coroutine! def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None!
)! yield from f! ! @asyncio.coroutine! def print_and_schedule():! while True:! print('Hello World')! yield from sleep(2) Resumes with g.send(None) -
@asyncio.coroutine! def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None!
)! yield from f! ! @asyncio.coroutine! def print_and_schedule():! while True:! print('Hello World')! yield from sleep(2) -
import asyncio! from aiohttp import request! ! @asyncio.coroutine! def get_content(url):!
resp = yield from request('GET', url)! content = yield from resp.read_and_close()! return content! ! @asyncio.roroutine! def get_them(url):! url = yield from get_content(url)! url = yield from get_content(url)! url = yield from get_content(url)! print(url) -
Bottom Line • Asynchrony is cool • Promises are pluscool
• Coroutines are doublepluscool -
The Rest • How asyncio works • Return values from
coroutines • Error-handling with yield from • … -
Further Reading • PEPs 255, 342, 380, and 3148 •
PyCon 2013 Keynote - Guido van Rossum • The difference between yield and yield-from • Deconstructing Deferred