Yield

 Yield

Asynchronised operations are good. Callbacks are not. Named callbacks clutter scopes and are disruptive. Anonymous callbacks create deeply nested code and become totally unreadable—Python does not even support that because it just lacks style! We deserve better. Let’s take a look on how Python 3.4’s new “asyncio” (aka Tulip) brings elegance to async tasks.

9dafad54b5b4f360b7aae5f482bc1c91?s=128

Tzu-ping Chung

May 17, 2014
Tweet

Transcript

  1. Yield a

  2. Yield from a

  3. There Are Things • asyncio fundementals • asyncio.Future • Generators

    (yield) • PEP 380 (yield from) • Coroutines • asyncio examples
  4. There’s No Time • asyncio fundementals • asyncio.Future • Generators

    (yield) • PEP 380 (yield from) • Coroutines • asyncio examples
  5. Me • Call me TP • Follow @uranusjr • uranusjr.com

  6. None
  7. www. .com

  8. Asynchrony • “Tell me when you’re done.” • Non-blocking API

    • Event loop
  9. Event Loop?

  10. • “Reactor” • In charge of dispatching events for tasks

    • Tasks give up control when they don’t need it Event Loop
  11. import asyncio! ! def print_and_repeat(loop):! print('Hello World')! loop.call_later(! 2,! print_and_repeat,!

    loop! )! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_repeat(loop)! loop.run_forever() https://code.google.com/p/tulip/source/browse/examples/hello_callback.py
  12. import asyncio! ! def print_and_repeat(loop):! print('Hello World')! loop.call_later(! 2,! print_and_repeat,!

    loop! )! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_repeat(loop)! loop.run_forever()
  13. import asyncio! ! def print_and_repeat(loop):! print('Hello World')! loop.call_later(! 2,! print_and_repeat,!

    loop! )! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_repeat(loop)! loop.run_forever()
  14. import asyncio! ! def print_and_repeat(loop):! print('Hello World')! loop.call_later(! 2,! print_and_repeat,!

    loop! )! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_repeat(loop)! loop.run_forever()
  15. • Takes registration for callbacks • Calls them when it

    should • Hard to persist states Event Loop
  16. Future

  17. Future • Deferred? Promises/A+? • “Run this for me and

    tell me the result.” • Like concurrent.futures.Future
  18. def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None! )!

    return f! ! def print_and_schedule(future):! print('Hello World')! f = sleep(2)! f.add_done_callback(print_and_schedule)! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_schedule()! loop.run_forever()
  19. def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None! )!

    return f! ! def print_and_schedule(future):! print('Hello World')! f = sleep(2)! f.add_done_callback(print_and_schedule)! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_schedule()! loop.run_forever()
  20. def sleep(secs):! f = asyncio.Future(loop=None)! f._loop.call_later(! secs, f.set_result, None! )!

    return f! ! def print_and_schedule(future):! print('Hello World')! f = sleep(2)! f.add_done_callback(print_and_schedule)! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_schedule()! loop.run_forever()
  21. def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None! )!

    return f! ! def print_and_schedule(future):! print('Hello World')! f = sleep(2)! f.add_done_callback(print_and_schedule)! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_schedule()! loop.run_forever()
  22. def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None! )!

    return f! ! def print_and_schedule(future):! print('Hello World')! f = sleep(2)! f.add_done_callback(print_and_schedule)! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_schedule()! loop.run_forever()
  23. def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None! )!

    return f! ! def print_and_schedule(future):! print('Hello World')! f = sleep(2)! f.add_done_callback(print_and_schedule)! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_schedule()! loop.run_forever()
  24. class Future:! # …! def add_done_callback(self, fn):! # …! self._callbacks.append(fn)!

    ! def set_result(self, result):! # …! self._schedule_callbacks()! ! def _schedule_callbacks(self):! callbacks = self._callbacks[:]! # …! for cb in callbacks:! self._loop.call_soon(cb, self) http://hg.python.org/cpython/file/3.4/Lib/asyncio/futures.py
  25. def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None! )!

    return f! ! def print_and_schedule(future):! print('Hello World')! f = sleep(2)! f.add_done_callback(print_and_schedule)! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_schedule(None)! loop.run_forever()
  26. def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None! )!

    return f! ! def print_and_schedule(future):! print('Hello World')! f = sleep(2)! f.add_done_callback(print_and_schedule)! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_schedule()! loop.run_forever()
  27. def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None! )!

    return f! ! def print_and_schedule(future):! print('Hello World')! f = sleep(2)! f.add_done_callback(print_and_schedule)! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_schedule()! loop.run_forever() called by event loop (after 2 secs)
  28. def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None! )!

    return f! ! def print_and_schedule(*args, **kwargs):! print('Hello World')! f = sleep(2)! f.add_done_callback(print_and_schedule)! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_schedule()! loop.run_forever() calls
  29. future print_and_schedule event loop Initial invocation .__init__ .call_later .set_result .add_done_callback

    call as callback Starts running
  30. Future • Encapsulates the asynchronous execution of a callable. •

    Callbacks are bundled with it • Event loop calls them for you
  31. Coroutine

  32. def producer():! yield 0! yield 1! ! g = producer()!

    print(type(g)) # <type 'generator'>! ! for i in g:! print(i)! # 0! # 1 Generators
  33. def producer():! yield 0! yield 1! ! g = producer()!

    print(type(g)) # <type 'generator'>! ! for i in g:! print(i)! # 0! # 1 Generators
  34. def producer():! yield 0! yield 1! ! g = producer()!

    print(type(g)) # <type 'generator'>! ! for i in g:! print(i)! # 0! # 1 Generators
  35. def producer():! yield 0! yield 1! ! g = producer()!

    print(type(g)) # <type 'generator'>! ! for i in g:! print(i)! # 0! # 1 Generators
  36. g = producer()! while True:! try:! i = next(g) #

    Python 3! print(i)! except StopIteration:! break! # 0! # 1
  37. g = producer()! while True:! try:! i = g.__next__() #

    Python 3! print(i)! except StopIteration:! break! # 0! # 1
  38. g = producer()! while True:! try:! i = g.send(None)! print(i)!

    except StopIteration:! break! # 0! # 1 http://www.python.org/dev/peps/pep-0342/
  39. def get_generator():! v = yield 0! print('Gen:', v)! v =

    yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None) # Starts g.! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Con: 0 ! Gen: 1! Con: 1! Gen: 2 Outputs
  40. def get_generator():! v = yield 0! print('Gen:', v)! v =

    yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None)! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Con: 0 ! Gen: 1! Con: 1! Gen: 2 Outputs
  41. def get_generator():! v = yield 0! print('Gen:', v)! v =

    yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None)! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Con: 0 ! Gen: 1! Con: 1! Gen: 2 Outputs
  42. def get_generator():! v = yield 0! print('Gen:', v)! v =

    yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None)! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Con: 0 ! Gen: 1! Con: 1! Gen: 2 Outputs
  43. def get_generator():! v = yield 0! print('Gen:', v)! v =

    yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None)! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Con: 0 ! Gen: 1! Con: 1! Gen: 2 Outputs
  44. def get_generator():! v = yield 0! print('Gen:', v)! v =

    yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None)! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Con: 0 ! Gen: 1! Con: 1! Gen: 2 Outputs
  45. def get_generator():! v = yield 0! print('Gen:', v)! v =

    yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None)! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Con: 0 ! Gen: 1! Con: 1! Gen: 2 Outputs
  46. def get_generator():! v = yield 0! print('Gen:', v)! v =

    yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None)! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Con: 0 ! Gen: 1! Con: 1! Gen: 2 Outputs
  47. def get_generator():! v = yield 0! print('Gen:', v)! v =

    yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None)! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Con: 0 ! Gen: 1! Con: 1! Gen: 2 Outputs
  48. def get_generator():! v = yield 0! print('Gen:', v)! v =

    yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None)! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Con: 0 ! Gen: 1! Con: 1! Gen: 2 Outputs
  49. def get_generator():! v = yield 0! print('Gen:', v)! v =

    yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None)! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Con: 0 ! Gen: 1! Con: 1! Gen: 2 Outputs No more yielding.
  50. def get_generator():! v = yield 0! print('Gen:', v)! v =

    yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None)! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Con: 0 ! Gen: 1! Con: 1! Gen: 2 Outputs No more yielding.
  51. def get_generator():! v = yield 0! print('Gen:', v)! v =

    yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None)! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Con: 0 ! Gen: 1! Con: 1! Gen: 2 Outputs
  52. [Coroutines] allow multiple entry points for suspending and resuming execution

    at certain locations. http://en.wikipedia.org/wiki/Coroutine
  53. def get_generator():! v = yield 0! print('Gen:', v)! v =

    yield v! print('Gen:', v)! ! g = get_generator()! v = g.send(None)! ! while True:! try:! print('Con:', v)! v = g.send(v + 1)! except StopIteration:! break Coroutines?
  54. Generators • “Semi-coroutines” • Always yield to its caller •

    yield is for passing back a value to parent
  55. None
  56. Generators • “Semi-coroutines” • Always yield to its caller •

    yield is for passing back a value to parent
  57. Generators • “Semi-coroutines” • Always yield to its caller •

    yield is for passing back a value to parent Event loop Future instance
  58. yield from • PEP 380 and Python 3.3 • Sub-generator

    delegation • yield the results from another generator
  59. I cannot possibly do this justice. — Guido, PyCon 2013

  60. yield from • Exceptions bubbling • Return value without round-trip

    to scheduler • Cleaner/simpler API
  61. def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None! )!

    return f! ! def print_and_schedule(future):! print('Hello World')! f = sleep(2)! f.add_done_callback(print_and_schedule)! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_schedule()! loop.run_forever()
  62. ! def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None!

    )! return f
  63. @asyncio.coroutine! def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None!

    )! yield from f
  64. def print_and_schedule(future):! print('Hello World')! f = sleep(2)! f.add_done_callback(print_and_schedule)! ! if

    __name__ == '__main__':! loop = asyncio.get_event_loop()! print_and_schedule()! loop.run_forever()
  65. @asyncio.coroutine! def print_and_schedule():! while True:! print('Hello World')! f = sleep(2)!

    yield from f! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! loop.run_until_complete(! print_and_schedule! )
  66. @asyncio.coroutine! def print_and_schedule():! while True:! print('Hello World')! f = sleep(2)!

    yield from f! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! loop.run_until_complete(! print_and_schedule! )
  67. @asyncio.coroutine! def print_and_schedule():! while True:! print('Hello World')! f = sleep(2)!

    yield from f! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! loop.run_until_complete(! print_and_schedule! )
  68. @asyncio.coroutine! def sleep(s):! f = asyncio.Future()! f._loop.call_later(s, f.set_result, None)! yield

    from f! ! @asyncio.coroutine! def print_and_schedule():! while True:! print('Hello World')! yield from sleep(2)! ! if __name__ == '__main__':! loop = asyncio.get_event_loop()! loop.run_until_complete(! print_and_schedule! )
  69. @asyncio.coroutine! def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None!

    )! yield from f! ! @asyncio.coroutine! def print_and_schedule():! while True:! print('Hello World')! yield from sleep(2)
  70. @asyncio.coroutine! def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None!

    )! yield from f! ! @asyncio.coroutine! def print_and_schedule():! while True:! print('Hello World')! yield from sleep(2) called by event loop
  71. @asyncio.coroutine! def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None!

    )! yield from f! ! @asyncio.coroutine! def print_and_schedule():! while True:! print('Hello World')! yield from sleep(2)
  72. @asyncio.coroutine! def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None!

    )! yield from f! ! @asyncio.coroutine! def print_and_schedule():! while True:! print('Hello World')! yield from sleep(2) yield to event loop
  73. @asyncio.coroutine! def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None!

    )! yield from f! ! @asyncio.coroutine! def print_and_schedule():! while True:! print('Hello World')! yield from sleep(2) Later, by event loop
  74. @asyncio.coroutine! def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None!

    )! yield from f! ! @asyncio.coroutine! def print_and_schedule():! while True:! print('Hello World')! yield from sleep(2) Resumes with g.send(None)
  75. @asyncio.coroutine! def sleep(secs):! f = asyncio.Future()! f._loop.call_later(! secs, f.set_result, None!

    )! yield from f! ! @asyncio.coroutine! def print_and_schedule():! while True:! print('Hello World')! yield from sleep(2)
  76. I got this.

  77. None
  78. import asyncio! from aiohttp import request! ! @asyncio.coroutine! def get_content(url):!

    resp = yield from request('GET', url)! content = yield from resp.read_and_close()! return content! ! @asyncio.roroutine! def get_them(url):! url = yield from get_content(url)! url = yield from get_content(url)! url = yield from get_content(url)! print(url)
  79. Bottom Line • Asynchrony is cool • Promises are pluscool

    • Coroutines are doublepluscool
  80. The Rest • How asyncio works • Return values from

    coroutines • Error-handling with yield from • …
  81. Further Reading • PEPs 255, 342, 380, and 3148 •

    PyCon 2013 Keynote - Guido van Rossum • The difference between yield and yield-from • Deconstructing Deferred
  82. None