Yield

Asynchronised operations are good. Callbacks are not. Named callbacks clutter scopes and are disruptive. Anonymous callbacks create deeply nested code and become totally unreadable—Python does not even support that because it just lacks style! We deserve better. Let’s take a look on how Python 3.4’s new “asyncio” (aka Tulip) brings elegance to async tasks.

Transcript

1. Yield a
2. Yield from a
3. There Are Things • asyncio fundementals • asyncio.Future • Generators (yield) • PEP 380 (yield from) • Coroutines • asyncio examples
4. There’s No Time • asyncio fundementals • asyncio.Future • Generators (yield) • PEP 380 (yield from) • Coroutines • asyncio examples
5. Me • Call me TP • Follow @uranusjr • uranusjr.com
7. www. .com
8. Asynchrony • “Tell me when you’re done.” • Non-blocking API • Event loop
9. Event Loop?
10. • “Reactor” • In charge of dispatching events for tasks • Tasks give up control when they don’t need it Event Loop
11. import asyncio! ! def print_and_repeat(loop):! print('Hello World')! loop.call_later(! 2,! print_and_repeat,! loop! )! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_repeat(loop)! loop.run_forever() https://code.google.com/p/tulip/source/browse/examples/hello_callback.py
12. import asyncio! ! def print_and_repeat(loop):! print('Hello World')! loop.call_later(! 2,! print_and_repeat,! loop! )! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_repeat(loop)! loop.run_forever()
13. import asyncio! ! def print_and_repeat(loop):! print('Hello World')! loop.call_later(! 2,! print_and_repeat,! loop! )! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_repeat(loop)! loop.run_forever()
14. import asyncio! ! def print_and_repeat(loop):! print('Hello World')! loop.call_later(! 2,! print_and_repeat,! loop! )! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_repeat(loop)! loop.run_forever()
15. • Takes registration for callbacks • Calls them when it should • Hard to persist states Event Loop
16. Future
17. Future • Deferred? Promises/A+? • “Run this for me and tell me the result.” • Like concurrent.futures.Future
18. def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None! )! return f! ! def print_and_schedule(future):! print('Hello World')! f = sleep(2)! f.add_done_callback(print_and_schedule)! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_schedule()! loop.run_forever()
19. def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None! )! return f! ! def print_and_schedule(future):! print('Hello World')! f = sleep(2)! f.add_done_callback(print_and_schedule)! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_schedule()! loop.run_forever()
20. def sleep(secs):! f = asyncio.Future(loop=None)! f._loop.call_later(! secs, f.set_result, None! )! return f! ! def print_and_schedule(future):! print('Hello World')! f = sleep(2)! f.add_done_callback(print_and_schedule)! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_schedule()! loop.run_forever()
21. def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None! )! return f! ! def print_and_schedule(future):! print('Hello World')! f = sleep(2)! f.add_done_callback(print_and_schedule)! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_schedule()! loop.run_forever()
22. def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None! )! return f! ! def print_and_schedule(future):! print('Hello World')! f = sleep(2)! f.add_done_callback(print_and_schedule)! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_schedule()! loop.run_forever()
23. def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None! )! return f! ! def print_and_schedule(future):! print('Hello World')! f = sleep(2)! f.add_done_callback(print_and_schedule)! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_schedule()! loop.run_forever()
24. class Future:! # …! def add_done_callback(self, fn):! # …! self._callbacks.append(fn)! ! def set_result(self, result):! # …! self._schedule_callbacks()! ! def _schedule_callbacks(self):! callbacks = self._callbacks[:]! # …! for cb in callbacks:! self._loop.call_soon(cb, self) http://hg.python.org/cpython/file/3.4/Lib/asyncio/futures.py
25. def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None! )! return f! ! def print_and_schedule(future):! print('Hello World')! f = sleep(2)! f.add_done_callback(print_and_schedule)! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_schedule(None)! loop.run_forever()
26. def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None! )! return f! ! def print_and_schedule(future):! print('Hello World')! f = sleep(2)! f.add_done_callback(print_and_schedule)! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_schedule()! loop.run_forever()
27. def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None! )! return f! ! def print_and_schedule(future):! print('Hello World')! f = sleep(2)! f.add_done_callback(print_and_schedule)! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_schedule()! loop.run_forever() called by event loop (after 2 secs)
28. def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None! )! return f! ! def print_and_schedule(*args, **kwargs):! print('Hello World')! f = sleep(2)! f.add_done_callback(print_and_schedule)! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_schedule()! loop.run_forever() calls
29. future print_and_schedule event loop Initial invocation .__init__ .call_later .set_result .add_done_callback call as callback Starts running
30. Future • Encapsulates the asynchronous execution of a callable. • Callbacks are bundled with it • Event loop calls them for you
31. Coroutine
32. def producer():! yield 0! yield 1! ! g = producer()! print(type(g)) # <type 'generator'>! ! for i in g:! print(i)! # 0! # 1 Generators
33. def producer():! yield 0! yield 1! ! g = producer()! print(type(g)) # <type 'generator'>! ! for i in g:! print(i)! # 0! # 1 Generators
34. def producer():! yield 0! yield 1! ! g = producer()! print(type(g)) # <type 'generator'>! ! for i in g:! print(i)! # 0! # 1 Generators
35. def producer():! yield 0! yield 1! ! g = producer()! print(type(g)) # <type 'generator'>! ! for i in g:! print(i)! # 0! # 1 Generators
36. g = producer()! while True:! try:! i = next(g) # Python 3! print(i)! except StopIteration:! break! # 0! # 1
37. g = producer()! while True:! try:! i = g.__next__() # Python 3! print(i)! except StopIteration:! break! # 0! # 1
38. g = producer()! while True:! try:! i = g.send(None)! print(i)! except StopIteration:! break! # 0! # 1 http://www.python.org/dev/peps/pep-0342/
39. def get_generator():! v = yield 0! print('Gen:', v)! v = yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None) # Starts g.! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Con: 0 ! Gen: 1! Con: 1! Gen: 2 Outputs
40. def get_generator():! v = yield 0! print('Gen:', v)! v = yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None)! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Con: 0 ! Gen: 1! Con: 1! Gen: 2 Outputs
41. def get_generator():! v = yield 0! print('Gen:', v)! v = yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None)! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Con: 0 ! Gen: 1! Con: 1! Gen: 2 Outputs
42. def get_generator():! v = yield 0! print('Gen:', v)! v = yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None)! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Con: 0 ! Gen: 1! Con: 1! Gen: 2 Outputs
43. def get_generator():! v = yield 0! print('Gen:', v)! v = yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None)! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Con: 0 ! Gen: 1! Con: 1! Gen: 2 Outputs
44. def get_generator():! v = yield 0! print('Gen:', v)! v = yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None)! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Con: 0 ! Gen: 1! Con: 1! Gen: 2 Outputs
45. def get_generator():! v = yield 0! print('Gen:', v)! v = yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None)! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Con: 0 ! Gen: 1! Con: 1! Gen: 2 Outputs
46. def get_generator():! v = yield 0! print('Gen:', v)! v = yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None)! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Con: 0 ! Gen: 1! Con: 1! Gen: 2 Outputs
47. def get_generator():! v = yield 0! print('Gen:', v)! v = yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None)! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Con: 0 ! Gen: 1! Con: 1! Gen: 2 Outputs
48. def get_generator():! v = yield 0! print('Gen:', v)! v = yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None)! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Con: 0 ! Gen: 1! Con: 1! Gen: 2 Outputs
49. def get_generator():! v = yield 0! print('Gen:', v)! v = yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None)! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Con: 0 ! Gen: 1! Con: 1! Gen: 2 Outputs No more yielding.
50. def get_generator():! v = yield 0! print('Gen:', v)! v = yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None)! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Con: 0 ! Gen: 1! Con: 1! Gen: 2 Outputs No more yielding.
51. def get_generator():! v = yield 0! print('Gen:', v)! v = yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None)! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Con: 0 ! Gen: 1! Con: 1! Gen: 2 Outputs
52. [Coroutines] allow multiple entry points for suspending and resuming execution at certain locations. http://en.wikipedia.org/wiki/Coroutine
53. def get_generator():! v = yield 0! print('Gen:', v)! v = yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None)! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Coroutines?
54. Generators • “Semi-coroutines” • Always yield to its caller • yield is for passing back a value to parent
56. Generators • “Semi-coroutines” • Always yield to its caller • yield is for passing back a value to parent
57. Generators • “Semi-coroutines” • Always yield to its caller • yield is for passing back a value to parent Event loop Future instance
58. yield from • PEP 380 and Python 3.3 • Sub-generator delegation • yield the results from another generator
59. I cannot possibly do this justice. — Guido, PyCon 2013
60. yield from • Exceptions bubbling • Return value without round-trip to scheduler • Cleaner/simpler API
61. def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None! )! return f! ! def print_and_schedule(future):! print('Hello World')! f = sleep(2)! f.add_done_callback(print_and_schedule)! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_schedule()! loop.run_forever()
62. ! def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None! )! return f
63. @asyncio.coroutine! def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None! )! yield from f
64. def print_and_schedule(future):! print('Hello World')! f = sleep(2)! f.add_done_callback(print_and_schedule)! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_schedule()! loop.run_forever()
65. @asyncio.coroutine! def print_and_schedule():! while True:! print('Hello World')! f = sleep(2)! yield from f! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! loop.run_until_complete(! print_and_schedule! )
66. @asyncio.coroutine! def print_and_schedule():! while True:! print('Hello World')! f = sleep(2)! yield from f! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! loop.run_until_complete(! print_and_schedule! )
67. @asyncio.coroutine! def print_and_schedule():! while True:! print('Hello World')! f = sleep(2)! yield from f! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! loop.run_until_complete(! print_and_schedule! )
68. @asyncio.coroutine! def sleep(s):! f = asyncio.Future()! f._loop.call_later(s, f.set_result, None)! yield from f! ! @asyncio.coroutine! def print_and_schedule():! while True:! print('Hello World')! yield from sleep(2)! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! loop.run_until_complete(! print_and_schedule! )
69. @asyncio.coroutine! def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None! )! yield from f! ! @asyncio.coroutine! def print_and_schedule():! while True:! print('Hello World')! yield from sleep(2)
70. @asyncio.coroutine! def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None! )! yield from f! ! @asyncio.coroutine! def print_and_schedule():! while True:! print('Hello World')! yield from sleep(2) called by event loop
71. @asyncio.coroutine! def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None! )! yield from f! ! @asyncio.coroutine! def print_and_schedule():! while True:! print('Hello World')! yield from sleep(2)
72. @asyncio.coroutine! def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None! )! yield from f! ! @asyncio.coroutine! def print_and_schedule():! while True:! print('Hello World')! yield from sleep(2) yield to event loop
73. @asyncio.coroutine! def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None! )! yield from f! ! @asyncio.coroutine! def print_and_schedule():! while True:! print('Hello World')! yield from sleep(2) Later, by event loop
74. @asyncio.coroutine! def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None! )! yield from f! ! @asyncio.coroutine! def print_and_schedule():! while True:! print('Hello World')! yield from sleep(2) Resumes with g.send(None)
75. @asyncio.coroutine! def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None! )! yield from f! ! @asyncio.coroutine! def print_and_schedule():! while True:! print('Hello World')! yield from sleep(2)
76. I got this.
78. import asyncio! from aiohttp import request! ! @asyncio.coroutine! def get_content(url):! resp = yield from request('GET', url)! content = yield from resp.read_and_close()! return content! ! @asyncio.roroutine! def get_them(url):! url = yield from get_content(url)! url = yield from get_content(url)! url = yield from get_content(url)! print(url)
79. Bottom Line • Asynchrony is cool • Promises are pluscool • Coroutines are doublepluscool
80. The Rest • How asyncio works • Return values from coroutines • Error-handling with yield from • …
81. Further Reading • PEPs 255, 342, 380, and 3148 • PyCon 2013 Keynote - Guido van Rossum • The difference between yield and yield-from • Deconstructing Deferred