Практическое применение Akka Streams

Несколько жизненных примеров использования Akka Streams

Transcript

1. Практика Akka Streams Алексей Романчук
2. 2 Обо мне
4. 4 Обо мне
5. • Знаете синтаксис Scala • Знакомы со стандартной библиотекой Scala • Слышали про Reactive Streams • Видели Akka Streams* 5 О вас
6. В предыдущих сериях
7. T
8. T Iterable[T]
9. T Iterable[T] Future[T]
10. T Iterable[T] Future[T] Source[T]
11. • Реализуют спецификацию Reactive Streams • Основаны на Akka • Умеют Backpressure • Статически типизированы 8 Akka Streams
12. 9 Ключевые абстракции Source Sink Flow
13. Базовые комбинаторы
14. 11 Source Source.single(1)
 Source.failed(new Exception())
 Source.empty[Int]
 
 Source.repeat(1)
 Source.fromIterator(() => Iterator(1, 2, 3))
 Source.cycle(() => Iterator(1, 2, 3))
 
 Source.tick(1 second, 100 millis, ())
15. 11 Source Source.single(1)
 Source.failed(new Exception())
 Source.empty[Int]
 
 Source.repeat(1)
 Source.fromIterator(() => Iterator(1, 2, 3))
 Source.cycle(() => Iterator(1, 2, 3))
 
 Source.tick(1 second, 100 millis, ())
16. 11 Source Source.single(1)
 Source.failed(new Exception())
 Source.empty[Int]
 
 Source.repeat(1)
 Source.fromIterator(() => Iterator(1, 2, 3))
 Source.cycle(() => Iterator(1, 2, 3))
 
 Source.tick(1 second, 100 millis, ())
17. 11 Source Source.single(1)
 Source.failed(new Exception())
 Source.empty[Int]
 
 Source.repeat(1)
 Source.fromIterator(() => Iterator(1, 2, 3))
 Source.cycle(() => Iterator(1, 2, 3))
 
 Source.tick(1 second, 100 millis, ())
18. 11 Source Source.single(1)
 Source.failed(new Exception())
 Source.empty[Int]
 
 Source.repeat(1)
 Source.fromIterator(() => Iterator(1, 2, 3))
 Source.cycle(() => Iterator(1, 2, 3))
 
 Source.tick(1 second, 100 millis, ())
19. 11 Source Source.single(1)
 Source.failed(new Exception())
 Source.empty[Int]
 
 Source.repeat(1)
 Source.fromIterator(() => Iterator(1, 2, 3))
 Source.cycle(() => Iterator(1, 2, 3))
 
 Source.tick(1 second, 100 millis, ())
20. 11 Source Source.single(1)
 Source.failed(new Exception())
 Source.empty[Int]
 
 Source.repeat(1)
 Source.fromIterator(() => Iterator(1, 2, 3))
 Source.cycle(() => Iterator(1, 2, 3))
 
 Source.tick(1 second, 100 millis, ())
21. 11 Source Source.single(1)
 Source.failed(new Exception())
 Source.empty[Int]
 
 Source.repeat(1)
 Source.fromIterator(() => Iterator(1, 2, 3))
 Source.cycle(() => Iterator(1, 2, 3))
 
 Source.tick(1 second, 100 millis, ())
22. 11 Source Source.single(1)
 Source.failed(new Exception())
 Source.empty[Int]
 
 Source.repeat(1)
 Source.fromIterator(() => Iterator(1, 2, 3))
 Source.cycle(() => Iterator(1, 2, 3))
 
 Source.tick(1 second, 100 millis, ())
23. 12 Sink Sink.ignore
 Sink.cancelled
 Sink.foreach[Int](println)
 
 Sink.head
 Sink.headOption
 Sink.last
 Sink.lastOption
 
 Sink.fold(0)(_ + _)
 Sink.reduce[Int](_ + _)
24. 12 Sink Sink.ignore
 Sink.cancelled
 Sink.foreach[Int](println)
 
 Sink.head
 Sink.headOption
 Sink.last
 Sink.lastOption
 
 Sink.fold(0)(_ + _)
 Sink.reduce[Int](_ + _)
25. 12 Sink Sink.ignore
 Sink.cancelled
 Sink.foreach[Int](println)
 
 Sink.head
 Sink.headOption
 Sink.last
 Sink.lastOption
 
 Sink.fold(0)(_ + _)
 Sink.reduce[Int](_ + _)
26. 12 Sink Sink.ignore
 Sink.cancelled
 Sink.foreach[Int](println)
 
 Sink.head
 Sink.headOption
 Sink.last
 Sink.lastOption
 
 Sink.fold(0)(_ + _)
 Sink.reduce[Int](_ + _)
27. 12 Sink Sink.ignore
 Sink.cancelled
 Sink.foreach[Int](println)
 
 Sink.head
 Sink.headOption
 Sink.last
 Sink.lastOption
 
 Sink.fold(0)(_ + _)
 Sink.reduce[Int](_ + _)
28. 12 Sink Sink.ignore
 Sink.cancelled
 Sink.foreach[Int](println)
 
 Sink.head
 Sink.headOption
 Sink.last
 Sink.lastOption
 
 Sink.fold(0)(_ + _)
 Sink.reduce[Int](_ + _)
29. 12 Sink Sink.ignore
 Sink.cancelled
 Sink.foreach[Int](println)
 
 Sink.head
 Sink.headOption
 Sink.last
 Sink.lastOption
 
 Sink.fold(0)(_ + _)
 Sink.reduce[Int](_ + _)
30. 13 Flow Flow[Int].map(identity)
 Flow[Int].mapAsync(10)(Future.successful)
 Flow[Int].mapAsyncUnordered(10)(Future.successful)
 
 Flow[Int].filter(_ % 2 == 0)
 Flow[Int].collect {
 case n if n % 2 == 0 => n
 } Flow[Int].take(10)
 Flow[Int].takeWhile(_ > 10)
 Flow[Int].drop(10)
 Flow[Int].dropWhile(_ > 10)
31. 13 Flow Flow[Int].map(identity)
 Flow[Int].mapAsync(10)(Future.successful)
 Flow[Int].mapAsyncUnordered(10)(Future.successful)
 
 Flow[Int].filter(_ % 2 == 0)
 Flow[Int].collect {
 case n if n % 2 == 0 => n
 } Flow[Int].take(10)
 Flow[Int].takeWhile(_ > 10)
 Flow[Int].drop(10)
 Flow[Int].dropWhile(_ > 10)
32. 13 Flow Flow[Int].map(identity)
 Flow[Int].mapAsync(10)(Future.successful)
 Flow[Int].mapAsyncUnordered(10)(Future.successful)
 
 Flow[Int].filter(_ % 2 == 0)
 Flow[Int].collect {
 case n if n % 2 == 0 => n
 } Flow[Int].take(10)
 Flow[Int].takeWhile(_ > 10)
 Flow[Int].drop(10)
 Flow[Int].dropWhile(_ > 10)
33. 13 Flow Flow[Int].map(identity)
 Flow[Int].mapAsync(10)(Future.successful)
 Flow[Int].mapAsyncUnordered(10)(Future.successful)
 
 Flow[Int].filter(_ % 2 == 0)
 Flow[Int].collect {
 case n if n % 2 == 0 => n
 } Flow[Int].take(10)
 Flow[Int].takeWhile(_ > 10)
 Flow[Int].drop(10)
 Flow[Int].dropWhile(_ > 10)
34. 13 Flow Flow[Int].map(identity)
 Flow[Int].mapAsync(10)(Future.successful)
 Flow[Int].mapAsyncUnordered(10)(Future.successful)
 
 Flow[Int].filter(_ % 2 == 0)
 Flow[Int].collect {
 case n if n % 2 == 0 => n
 } Flow[Int].take(10)
 Flow[Int].takeWhile(_ > 10)
 Flow[Int].drop(10)
 Flow[Int].dropWhile(_ > 10)
35. 13 Flow Flow[Int].map(identity)
 Flow[Int].mapAsync(10)(Future.successful)
 Flow[Int].mapAsyncUnordered(10)(Future.successful)
 
 Flow[Int].filter(_ % 2 == 0)
 Flow[Int].collect {
 case n if n % 2 == 0 => n
 } Flow[Int].take(10)
 Flow[Int].takeWhile(_ > 10)
 Flow[Int].drop(10)
 Flow[Int].dropWhile(_ > 10)
36. 13 Flow Flow[Int].map(identity)
 Flow[Int].mapAsync(10)(Future.successful)
 Flow[Int].mapAsyncUnordered(10)(Future.successful)
 
 Flow[Int].filter(_ % 2 == 0)
 Flow[Int].collect {
 case n if n % 2 == 0 => n
 } Flow[Int].take(10)
 Flow[Int].takeWhile(_ > 10)
 Flow[Int].drop(10)
 Flow[Int].dropWhile(_ > 10)
37. 13 Flow Flow[Int].map(identity)
 Flow[Int].mapAsync(10)(Future.successful)
 Flow[Int].mapAsyncUnordered(10)(Future.successful)
 
 Flow[Int].filter(_ % 2 == 0)
 Flow[Int].collect {
 case n if n % 2 == 0 => n
 } Flow[Int].take(10)
 Flow[Int].takeWhile(_ > 10)
 Flow[Int].drop(10)
 Flow[Int].dropWhile(_ > 10)
38. 14 Flow Flow[Int].flatMapConcat(n =>
 Source.fromIterator(() => Iterator.range(1, n))
 ) 1 2 3 Х
39. 14 Flow Flow[Int].flatMapConcat(n =>
 Source.fromIterator(() => Iterator.range(1, n))
 ) 1 2 3 Х 1 Х
40. 14 Flow Flow[Int].flatMapConcat(n =>
 Source.fromIterator(() => Iterator.range(1, n))
 ) 1 2 3 Х 1 Х 1 2 Х
41. 14 Flow Flow[Int].flatMapConcat(n =>
 Source.fromIterator(() => Iterator.range(1, n))
 ) 1 2 3 Х 1 Х 1 2 Х 1 2 3 Х
42. 14 Flow Flow[Int].flatMapConcat(n =>
 Source.fromIterator(() => Iterator.range(1, n))
 ) 1 2 3 Х 1 Х 1 2 Х 1 2 3 Х 1 1 2 1 2 3 Х
43. Пример №1
44. • Прочитать с хадупа логи транзакций и найти там 100 примеров фродных транзакций 16 Задача List Files API Read Line Result
45. 17 Наивная реализация def checkFraud(in: String): Future[Option[TransactionId]] = ??? val result: List[TransactionId] = (for {
 file <- hdfs.listStatusIterator(path)
 if file.isFile
 line <-fromInputStream(fs.open(file.getPath)).getLines()
 resultOpt = Await.result(checkFraud(line), 10 seconds)
 result <- resultOpt
 } yield result) .take(100).toList
46. 17 Наивная реализация def checkFraud(in: String): Future[Option[TransactionId]] = ??? val result: List[TransactionId] = (for {
 file <- hdfs.listStatusIterator(path)
 if file.isFile
 line <-fromInputStream(fs.open(file.getPath)).getLines()
 resultOpt = Await.result(checkFraud(line), 10 seconds)
 result <- resultOpt
 } yield result) .take(100).toList
47. 17 Наивная реализация def checkFraud(in: String): Future[Option[TransactionId]] = ??? val result: List[TransactionId] = (for {
 file <- hdfs.listStatusIterator(path)
 if file.isFile
 line <-fromInputStream(fs.open(file.getPath)).getLines()
 resultOpt = Await.result(checkFraud(line), 10 seconds)
 result <- resultOpt
 } yield result) .take(100).toList
48. 17 Наивная реализация def checkFraud(in: String): Future[Option[TransactionId]] = ??? val result: List[TransactionId] = (for {
 file <- hdfs.listStatusIterator(path)
 if file.isFile
 line <-fromInputStream(fs.open(file.getPath)).getLines()
 resultOpt = Await.result(checkFraud(line), 10 seconds)
 result <- resultOpt
 } yield result) .take(100).toList
49. 17 Наивная реализация def checkFraud(in: String): Future[Option[TransactionId]] = ??? val result: List[TransactionId] = (for {
 file <- hdfs.listStatusIterator(path)
 if file.isFile
 line <-fromInputStream(fs.open(file.getPath)).getLines()
 resultOpt = Await.result(checkFraud(line), 10 seconds)
 result <- resultOpt
 } yield result) .take(100).toList
50. 17 Наивная реализация def checkFraud(in: String): Future[Option[TransactionId]] = ??? val result: List[TransactionId] = (for {
 file <- hdfs.listStatusIterator(path)
 if file.isFile
 line <-fromInputStream(fs.open(file.getPath)).getLines()
 resultOpt = Await.result(checkFraud(line), 10 seconds)
 result <- resultOpt
 } yield result) .take(100).toList
51. 17 Наивная реализация def checkFraud(in: String): Future[Option[TransactionId]] = ??? val result: List[TransactionId] = (for {
 file <- hdfs.listStatusIterator(path)
 if file.isFile
 line <-fromInputStream(fs.open(file.getPath)).getLines()
 resultOpt = Await.result(checkFraud(line), 10 seconds)
 result <- resultOpt
 } yield result) .take(100).toList
52. 17 Наивная реализация def checkFraud(in: String): Future[Option[TransactionId]] = ??? val result: List[TransactionId] = (for {
 file <- hdfs.listStatusIterator(path)
 if file.isFile
 line <-fromInputStream(fs.open(file.getPath)).getLines()
 resultOpt = Await.result(checkFraud(line), 10 seconds)
 result <- resultOpt
 } yield result) .take(100).toList
53. 17 Наивная реализация def checkFraud(in: String): Future[Option[TransactionId]] = ??? val result: List[TransactionId] = (for {
 file <- hdfs.listStatusIterator(path)
 if file.isFile
 line <-fromInputStream(fs.open(file.getPath)).getLines()
 resultOpt = Await.result(checkFraud(line), 10 seconds)
 result <- resultOpt
 } yield result) .take(100).toList
54. 17 Наивная реализация def checkFraud(in: String): Future[Option[TransactionId]] = ??? val result: List[TransactionId] = (for {
 file <- hdfs.listStatusIterator(path)
 if file.isFile
 line <-fromInputStream(fs.open(file.getPath)).getLines()
 resultOpt = Await.result(checkFraud(line), 10 seconds)
 result <- resultOpt
 } yield result) .take(100).toList
55. 17 Наивная реализация def checkFraud(in: String): Future[Option[TransactionId]] = ??? val result: List[TransactionId] = (for {
 file <- hdfs.listStatusIterator(path)
 if file.isFile
 line <-fromInputStream(fs.open(file.getPath)).getLines()
 resultOpt = Await.result(checkFraud(line), 10 seconds)
 result <- resultOpt
 } yield result) .take(100).toList
56. Ок?
57. • Не терять лишнего времени на скачивание • Параллельные запросы к сервису • Не превышать ограничения в 10 запросов в секунду 19 Требования
58. 20 Красивая картинка со схемой List Files API Read Line Result
59. 21 Красивый код на стримах val stream =
 Source
 .fromIterator { () => hdfs.listStatusIterator(path) }
 .filter(_.isFile)
 .flatMapConcat { file =>
 val s = fromInputStream(hdfs.open(file.getPath))
 Source.fromIterator(() => s.getLines())
 }
 .throttle(10, 1 second, 0, ThrottleMode.Shaping)
 .mapAsyncUnordered(50)(checkFraud)
 .collect { case Some(value) => value }
 .take(100)
 .fold(List.empty[TransactionId])(_ :+ _)
 .toMat(Sink.head)(Keep.right)

60. 21 Красивый код на стримах val stream =
 Source
 .fromIterator { () => hdfs.listStatusIterator(path) }
 .filter(_.isFile)
 .flatMapConcat { file =>
 val s = fromInputStream(hdfs.open(file.getPath))
 Source.fromIterator(() => s.getLines())
 }
 .throttle(10, 1 second, 0, ThrottleMode.Shaping)
 .mapAsyncUnordered(50)(checkFraud)
 .collect { case Some(value) => value }
 .take(100)
 .fold(List.empty[TransactionId])(_ :+ _)
 .toMat(Sink.head)(Keep.right)

61. 21 Красивый код на стримах val stream =
 Source
 .fromIterator { () => hdfs.listStatusIterator(path) }
 .filter(_.isFile)
 .flatMapConcat { file =>
 val s = fromInputStream(hdfs.open(file.getPath))
 Source.fromIterator(() => s.getLines())
 }
 .throttle(10, 1 second, 0, ThrottleMode.Shaping)
 .mapAsyncUnordered(50)(checkFraud)
 .collect { case Some(value) => value }
 .take(100)
 .fold(List.empty[TransactionId])(_ :+ _)
 .toMat(Sink.head)(Keep.right)

62. 21 Красивый код на стримах val stream =
 Source
 .fromIterator { () => hdfs.listStatusIterator(path) }
 .filter(_.isFile)
 .flatMapConcat { file =>
 val s = fromInputStream(hdfs.open(file.getPath))
 Source.fromIterator(() => s.getLines())
 }
 .throttle(10, 1 second, 0, ThrottleMode.Shaping)
 .mapAsyncUnordered(50)(checkFraud)
 .collect { case Some(value) => value }
 .take(100)
 .fold(List.empty[TransactionId])(_ :+ _)
 .toMat(Sink.head)(Keep.right)

63. 21 Красивый код на стримах val stream =
 Source
 .fromIterator { () => hdfs.listStatusIterator(path) }
 .filter(_.isFile)
 .flatMapConcat { file =>
 val s = fromInputStream(hdfs.open(file.getPath))
 Source.fromIterator(() => s.getLines())
 }
 .throttle(10, 1 second, 0, ThrottleMode.Shaping)
 .mapAsyncUnordered(50)(checkFraud)
 .collect { case Some(value) => value }
 .take(100)
 .fold(List.empty[TransactionId])(_ :+ _)
 .toMat(Sink.head)(Keep.right)

64. 21 Красивый код на стримах val stream =
 Source
 .fromIterator { () => hdfs.listStatusIterator(path) }
 .filter(_.isFile)
 .flatMapConcat { file =>
 val s = fromInputStream(hdfs.open(file.getPath))
 Source.fromIterator(() => s.getLines())
 }
 .throttle(10, 1 second, 0, ThrottleMode.Shaping)
 .mapAsyncUnordered(50)(checkFraud)
 .collect { case Some(value) => value }
 .take(100)
 .fold(List.empty[TransactionId])(_ :+ _)
 .toMat(Sink.head)(Keep.right)

65. 21 Красивый код на стримах val stream =
 Source
 .fromIterator { () => hdfs.listStatusIterator(path) }
 .filter(_.isFile)
 .flatMapConcat { file =>
 val s = fromInputStream(hdfs.open(file.getPath))
 Source.fromIterator(() => s.getLines())
 }
 .throttle(10, 1 second, 0, ThrottleMode.Shaping)
 .mapAsyncUnordered(50)(checkFraud)
 .collect { case Some(value) => value }
 .take(100)
 .fold(List.empty[TransactionId])(_ :+ _)
 .toMat(Sink.head)(Keep.right)

66. 21 Красивый код на стримах val stream =
 Source
 .fromIterator { () => hdfs.listStatusIterator(path) }
 .filter(_.isFile)
 .flatMapConcat { file =>
 val s = fromInputStream(hdfs.open(file.getPath))
 Source.fromIterator(() => s.getLines())
 }
 .throttle(10, 1 second, 0, ThrottleMode.Shaping)
 .mapAsyncUnordered(50)(checkFraud)
 .collect { case Some(value) => value }
 .take(100)
 .fold(List.empty[TransactionId])(_ :+ _)
 .toMat(Sink.head)(Keep.right)

67. 21 Красивый код на стримах val stream =
 Source
 .fromIterator { () => hdfs.listStatusIterator(path) }
 .filter(_.isFile)
 .flatMapConcat { file =>
 val s = fromInputStream(hdfs.open(file.getPath))
 Source.fromIterator(() => s.getLines())
 }
 .throttle(10, 1 second, 0, ThrottleMode.Shaping)
 .mapAsyncUnordered(50)(checkFraud)
 .collect { case Some(value) => value }
 .take(100)
 .fold(List.empty[TransactionId])(_ :+ _)
 .toMat(Sink.head)(Keep.right)

68. 21 Красивый код на стримах val stream =
 Source
 .fromIterator { () => hdfs.listStatusIterator(path) }
 .filter(_.isFile)
 .flatMapConcat { file =>
 val s = fromInputStream(hdfs.open(file.getPath))
 Source.fromIterator(() => s.getLines())
 }
 .throttle(10, 1 second, 0, ThrottleMode.Shaping)
 .mapAsyncUnordered(50)(checkFraud)
 .collect { case Some(value) => value }
 .take(100)
 .fold(List.empty[TransactionId])(_ :+ _)
 .toMat(Sink.head)(Keep.right)

69. 21 Красивый код на стримах val stream =
 Source
 .fromIterator { () => hdfs.listStatusIterator(path) }
 .filter(_.isFile)
 .flatMapConcat { file =>
 val s = fromInputStream(hdfs.open(file.getPath))
 Source.fromIterator(() => s.getLines())
 }
 .throttle(10, 1 second, 0, ThrottleMode.Shaping)
 .mapAsyncUnordered(50)(checkFraud)
 .collect { case Some(value) => value }
 .take(100)
 .fold(List.empty[TransactionId])(_ :+ _)
 .toMat(Sink.head)(Keep.right)

70. 21 Красивый код на стримах val stream =
 Source
 .fromIterator { () => hdfs.listStatusIterator(path) }
 .filter(_.isFile)
 .flatMapConcat { file =>
 val s = fromInputStream(hdfs.open(file.getPath))
 Source.fromIterator(() => s.getLines())
 }
 .throttle(10, 1 second, 0, ThrottleMode.Shaping)
 .mapAsyncUnordered(50)(checkFraud)
 .collect { case Some(value) => value }
 .take(100)
 .fold(List.empty[TransactionId])(_ :+ _)
 .toMat(Sink.head)(Keep.right)

71. 21 Красивый код на стримах val stream =
 Source
 .fromIterator { () => hdfs.listStatusIterator(path) }
 .filter(_.isFile)
 .flatMapConcat { file =>
 val s = fromInputStream(hdfs.open(file.getPath))
 Source.fromIterator(() => s.getLines())
 }
 .throttle(10, 1 second, 0, ThrottleMode.Shaping)
 .mapAsyncUnordered(50)(checkFraud)
 .collect { case Some(value) => value }
 .take(100)
 .fold(List.empty[TransactionId])(_ :+ _)
 .toMat(Sink.head)(Keep.right)

72. 21 Красивый код на стримах val stream =
 Source
 .fromIterator { () => hdfs.listStatusIterator(path) }
 .filter(_.isFile)
 .flatMapConcat { file =>
 val s = fromInputStream(hdfs.open(file.getPath))
 Source.fromIterator(() => s.getLines())
 }
 .throttle(10, 1 second, 0, ThrottleMode.Shaping)
 .mapAsyncUnordered(50)(checkFraud)
 .collect { case Some(value) => value }
 .take(100)
 .fold(List.empty[TransactionId])(_ :+ _)
 .toMat(Sink.head)(Keep.right)

73. 21 Красивый код на стримах val stream =
 Source
 .fromIterator { () => hdfs.listStatusIterator(path) }
 .filter(_.isFile)
 .flatMapConcat { file =>
 val s = fromInputStream(hdfs.open(file.getPath))
 Source.fromIterator(() => s.getLines())
 }
 .throttle(10, 1 second, 0, ThrottleMode.Shaping)
 .mapAsyncUnordered(50)(checkFraud)
 .collect { case Some(value) => value }
 .take(100)
 .fold(List.empty[TransactionId])(_ :+ _)
 .toMat(Sink.head)(Keep.right)

74. 21 Красивый код на стримах val stream =
 Source
 .fromIterator { () => hdfs.listStatusIterator(path) }
 .filter(_.isFile)
 .flatMapConcat { file =>
 val s = fromInputStream(hdfs.open(file.getPath))
 Source.fromIterator(() => s.getLines())
 }
 .throttle(10, 1 second, 0, ThrottleMode.Shaping)
 .mapAsyncUnordered(50)(checkFraud)
 .collect { case Some(value) => value }
 .take(100)
 .fold(List.empty[TransactionId])(_ :+ _)
 .toMat(Sink.head)(Keep.right)

75. Все на стандартных комбинаторах
76. 23 Запуск val result: Future[List[TransactionId]] = stream.run() 
 val result2: Future[List[TransactionId]] = stream.run()
77. 24 Запуск List Files API Read Line Result
78. 24 Запуск List Files API Read Line Result run() Future Stream
79. 24 Запуск List Files API Read Line Result run() Future Stream run() Future Stream
80. Материализация
81. Чертеж → Деталь
82. 27 RunnableGraph RunnableGraph[+Mat] { def run(): Mat }
83. 28 Mat final class Source[+Out, +Mat] final class Flow[-In, +Out, +Mat] final class Sink[-In, +Mat]
84. 29 Sources case object NotUsed def single[T](element: T) def empty[T] def failed[T](cause: Throwable) def repeat[T](element: T) def fromIterator[T](f: () => Iterator[T]) def cycle[T](f: () => Iterator[T])
85. 30 Sinks def ignore: Sink[Any, Future[Done]] def foreach[T](f: T => Unit): Sink[T, Future[Done]] def head[T]: Sink[T, Future[T]] def headOption[T]: Sink[T, Future[Option[T]]] def fold[U, T](z: U)(f: (U, T) => U): Sink[T, Future[U]] def reduce[T](f: (T, T) => T): Sink[T, Future[T]]
86. 30 Sinks def ignore: Sink[Any, Future[Done]] def foreach[T](f: T => Unit): Sink[T, Future[Done]] def head[T]: Sink[T, Future[T]] def headOption[T]: Sink[T, Future[Option[T]]] def fold[U, T](z: U)(f: (U, T) => U): Sink[T, Future[U]] def reduce[T](f: (T, T) => T): Sink[T, Future[T]]
87. 30 Sinks def ignore: Sink[Any, Future[Done]] def foreach[T](f: T => Unit): Sink[T, Future[Done]] def head[T]: Sink[T, Future[T]] def headOption[T]: Sink[T, Future[Option[T]]] def fold[U, T](z: U)(f: (U, T) => U): Sink[T, Future[U]] def reduce[T](f: (T, T) => T): Sink[T, Future[T]]
88. 30 Sinks def ignore: Sink[Any, Future[Done]] def foreach[T](f: T => Unit): Sink[T, Future[Done]] def head[T]: Sink[T, Future[T]] def headOption[T]: Sink[T, Future[Option[T]]] def fold[U, T](z: U)(f: (U, T) => U): Sink[T, Future[U]] def reduce[T](f: (T, T) => T): Sink[T, Future[T]]
89. 30 Sinks def ignore: Sink[Any, Future[Done]] def foreach[T](f: T => Unit): Sink[T, Future[Done]] def head[T]: Sink[T, Future[T]] def headOption[T]: Sink[T, Future[Option[T]]] def fold[U, T](z: U)(f: (U, T) => U): Sink[T, Future[U]] def reduce[T](f: (T, T) => T): Sink[T, Future[T]]
90. 30 Sinks def ignore: Sink[Any, Future[Done]] def foreach[T](f: T => Unit): Sink[T, Future[Done]] def head[T]: Sink[T, Future[T]] def headOption[T]: Sink[T, Future[Option[T]]] def fold[U, T](z: U)(f: (U, T) => U): Sink[T, Future[U]] def reduce[T](f: (T, T) => T): Sink[T, Future[T]]
91. 30 Sinks def ignore: Sink[Any, Future[Done]] def foreach[T](f: T => Unit): Sink[T, Future[Done]] def head[T]: Sink[T, Future[T]] def headOption[T]: Sink[T, Future[Option[T]]] def fold[U, T](z: U)(f: (U, T) => U): Sink[T, Future[U]] def reduce[T](f: (T, T) => T): Sink[T, Future[T]]
92. 30 Sinks def ignore: Sink[Any, Future[Done]] def foreach[T](f: T => Unit): Sink[T, Future[Done]] def head[T]: Sink[T, Future[T]] def headOption[T]: Sink[T, Future[Option[T]]] def fold[U, T](z: U)(f: (U, T) => U): Sink[T, Future[U]] def reduce[T](f: (T, T) => T): Sink[T, Future[T]]
93. 31 Композиция val source: Source[Int, NotUsed] = Source.single(1)
 val sink: Sink[Any, Future[Done]] = Sink.ignore
 
 val s1: RunnableGraph[NotUsed] = source.to(sink)
 
 val s2: RunnableGraph[Future[Done]] =
 source.toMat(sink)(Keep.right)
 
 val s3: RunnableGraph[(NotUsed, Future[Done])] =
 source.toMat(sink)(Keep.both)

94. 31 Композиция val source: Source[Int, NotUsed] = Source.single(1)
 val sink: Sink[Any, Future[Done]] = Sink.ignore
 
 val s1: RunnableGraph[NotUsed] = source.to(sink)
 
 val s2: RunnableGraph[Future[Done]] =
 source.toMat(sink)(Keep.right)
 
 val s3: RunnableGraph[(NotUsed, Future[Done])] =
 source.toMat(sink)(Keep.both)

95. 31 Композиция val source: Source[Int, NotUsed] = Source.single(1)
 val sink: Sink[Any, Future[Done]] = Sink.ignore
 
 val s1: RunnableGraph[NotUsed] = source.to(sink)
 
 val s2: RunnableGraph[Future[Done]] =
 source.toMat(sink)(Keep.right)
 
 val s3: RunnableGraph[(NotUsed, Future[Done])] =
 source.toMat(sink)(Keep.both)

96. 31 Композиция val source: Source[Int, NotUsed] = Source.single(1)
 val sink: Sink[Any, Future[Done]] = Sink.ignore
 
 val s1: RunnableGraph[NotUsed] = source.to(sink)
 
 val s2: RunnableGraph[Future[Done]] =
 source.toMat(sink)(Keep.right)
 
 val s3: RunnableGraph[(NotUsed, Future[Done])] =
 source.toMat(sink)(Keep.both)

97. 31 Композиция val source: Source[Int, NotUsed] = Source.single(1)
 val sink: Sink[Any, Future[Done]] = Sink.ignore
 
 val s1: RunnableGraph[NotUsed] = source.to(sink)
 
 val s2: RunnableGraph[Future[Done]] =
 source.toMat(sink)(Keep.right)
 
 val s3: RunnableGraph[(NotUsed, Future[Done])] =
 source.toMat(sink)(Keep.both)

98. 31 Композиция val source: Source[Int, NotUsed] = Source.single(1)
 val sink: Sink[Any, Future[Done]] = Sink.ignore
 
 val s1: RunnableGraph[NotUsed] = source.to(sink)
 
 val s2: RunnableGraph[Future[Done]] =
 source.toMat(sink)(Keep.right)
 
 val s3: RunnableGraph[(NotUsed, Future[Done])] =
 source.toMat(sink)(Keep.both)

99. 32 Композиция
100. 33 Красивый код на стримах val stream: RunnableGraph[Future[List[TransactionId]]] = Source
 .fromIterator { () => hdfs.listStatusIterator(path) }
 .filter(_.isFile)
 .flatMapConcat { file =>
 val s = fromInputStream(hdfs.open(file.getPath))
 Source.fromIterator(() => s.getLines())
 }
 .throttle(10, 1 second, 0, ThrottleMode.Shaping)
 .mapAsyncUnordered(50)(checkFraud)
 .collect { case Some(value) => value }
 .take(100)
 .fold(List.empty[TransactionId])(_ :+ _)
 .toMat(Sink.head)(Keep.right)

101. 33 Красивый код на стримах val stream: RunnableGraph[Future[List[TransactionId]]] = Source
 .fromIterator { () => hdfs.listStatusIterator(path) }
 .filter(_.isFile)
 .flatMapConcat { file =>
 val s = fromInputStream(hdfs.open(file.getPath))
 Source.fromIterator(() => s.getLines())
 }
 .throttle(10, 1 second, 0, ThrottleMode.Shaping)
 .mapAsyncUnordered(50)(checkFraud)
 .collect { case Some(value) => value }
 .take(100)
 .fold(List.empty[TransactionId])(_ :+ _)
 .toMat(Sink.head)(Keep.right)

102. 33 Красивый код на стримах val stream: RunnableGraph[Future[List[TransactionId]]] = Source
 .fromIterator { () => hdfs.listStatusIterator(path) }
 .filter(_.isFile)
 .flatMapConcat { file =>
 val s = fromInputStream(hdfs.open(file.getPath))
 Source.fromIterator(() => s.getLines())
 }
 .throttle(10, 1 second, 0, ThrottleMode.Shaping)
 .mapAsyncUnordered(50)(checkFraud)
 .collect { case Some(value) => value }
 .take(100)
 .fold(List.empty[TransactionId])(_ :+ _)
 .toMat(Sink.head)(Keep.right)

103. 33 Красивый код на стримах val stream: RunnableGraph[Future[List[TransactionId]]] = Source
 .fromIterator { () => hdfs.listStatusIterator(path) }
 .filter(_.isFile)
 .flatMapConcat { file =>
 val s = fromInputStream(hdfs.open(file.getPath))
 Source.fromIterator(() => s.getLines())
 }
 .throttle(10, 1 second, 0, ThrottleMode.Shaping)
 .mapAsyncUnordered(50)(checkFraud)
 .collect { case Some(value) => value }
 .take(100)
 .fold(List.empty[TransactionId])(_ :+ _)
 .toMat(Sink.head)(Keep.right)

104. 34 Красивая картинка со схемой List Files API Read Line Result run() Stream run() Future Stream Future
105. 34 Красивая картинка со схемой List Files API Read Line Result run() Future Stream run() Future Stream Future
106. Пример №2
107. • Читать данные из очереди и обрабатывать. 36 Задача Source Process Sink Future
108. 37 Красивый код на стримах val source = ZMQSource(context,
 mode = ZMQ.PULL,
 timeout = 1 second,
 addresses = List("tcp: //127.0.0.1:12345")
 )
 
 def process(in: ByteString): Future[Int] = ???
109. 37 Красивый код на стримах val source = ZMQSource(context,
 mode = ZMQ.PULL,
 timeout = 1 second,
 addresses = List("tcp: //127.0.0.1:12345")
 )
 
 def process(in: ByteString): Future[Int] = ???
110. 37 Красивый код на стримах val source = ZMQSource(context,
 mode = ZMQ.PULL,
 timeout = 1 second,
 addresses = List("tcp: //127.0.0.1:12345")
 )
 
 def process(in: ByteString): Future[Int] = ???
111. 37 Красивый код на стримах val source = ZMQSource(context,
 mode = ZMQ.PULL,
 timeout = 1 second,
 addresses = List("tcp: //127.0.0.1:12345")
 )
 
 def process(in: ByteString): Future[Int] = ???
112. 38 Красивый код на стримах val stream = source
 .mapAsync(10)(process)
 .toMat(Sink.last)(Keep.right)
 
 val result: Future[Int] = stream.run()
 result.onComplete(_ => "We are done here")

113. 38 Красивый код на стримах val stream = source
 .mapAsync(10)(process)
 .toMat(Sink.last)(Keep.right)
 
 val result: Future[Int] = stream.run()
 result.onComplete(_ => "We are done here")

114. 38 Красивый код на стримах val stream = source
 .mapAsync(10)(process)
 .toMat(Sink.last)(Keep.right)
 
 val result: Future[Int] = stream.run()
 result.onComplete(_ => "We are done here")

115. 38 Красивый код на стримах val stream = source
 .mapAsync(10)(process)
 .toMat(Sink.last)(Keep.right)
 
 val result: Future[Int] = stream.run()
 result.onComplete(_ => "We are done here")

116. 38 Красивый код на стримах val stream = source
 .mapAsync(10)(process)
 .toMat(Sink.last)(Keep.right)
 
 val result: Future[Int] = stream.run()
 result.onComplete(_ => "We are done here")

117. 39 Красивая картинка со схемой Source Process Sink Future
118. 39 Красивая картинка со схемой Source Process Sink Future Stream Future
119. Как это остановить?
120. 41 Проблема остановки val stream = source
 .mapAsync(10)(process)
 .toMat(Sink.last)(Keep.right)
 
 val result: Future[Int] = stream.run()
 result.onComplete(_ => "We are done here")

121. 42 Проблема остановки Source Process Sink Future
122. 42 Проблема остановки Source Process Sink Future Control
123. 42 Проблема остановки Source Process Sink Future Stream Control Control Future
124. 43 Проблема остановки trait Control {
 /**
 * Disconnect the underlying ZMQ socket, deliver the remaining data and finally close the socket.
 */
 def gracefulStop(): Unit
 }
125. 44 Проблема остановки val source: Source[ByteString, Control] = ??? val stream: RunnableGraph[(Control, Future[Int])] = source
 .mapAsync(10)(process)
 .toMat(Sink.head)(Keep.both)
 
 val (control: Control, result: Future[Int]) = stream.run()
 result.onComplete(_ => "We are done here")
 control.gracefulStop()
126. 44 Проблема остановки val source: Source[ByteString, Control] = ??? val stream: RunnableGraph[(Control, Future[Int])] = source
 .mapAsync(10)(process)
 .toMat(Sink.head)(Keep.both)
 
 val (control: Control, result: Future[Int]) = stream.run()
 result.onComplete(_ => "We are done here")
 control.gracefulStop()
127. 44 Проблема остановки val source: Source[ByteString, Control] = ??? val stream: RunnableGraph[(Control, Future[Int])] = source
 .mapAsync(10)(process)
 .toMat(Sink.head)(Keep.both)
 
 val (control: Control, result: Future[Int]) = stream.run()
 result.onComplete(_ => "We are done here")
 control.gracefulStop()
128. 44 Проблема остановки val source: Source[ByteString, Control] = ??? val stream: RunnableGraph[(Control, Future[Int])] = source
 .mapAsync(10)(process)
 .toMat(Sink.head)(Keep.both)
 
 val (control: Control, result: Future[Int]) = stream.run()
 result.onComplete(_ => "We are done here")
 control.gracefulStop()
129. 44 Проблема остановки val source: Source[ByteString, Control] = ??? val stream: RunnableGraph[(Control, Future[Int])] = source
 .mapAsync(10)(process)
 .toMat(Sink.head)(Keep.both)
 
 val (control: Control, result: Future[Int]) = stream.run()
 result.onComplete(_ => "We are done here")
 control.gracefulStop()
130. 44 Проблема остановки val source: Source[ByteString, Control] = ??? val stream: RunnableGraph[(Control, Future[Int])] = source
 .mapAsync(10)(process)
 .toMat(Sink.head)(Keep.both)
 
 val (control: Control, result: Future[Int]) = stream.run()
 result.onComplete(_ => "We are done here")
 control.gracefulStop()
131. Пример №3
132. 46 Механика Flow[T]
133. 46 Механика Flow[T] Request Cancel
134. 46 Механика Flow[T] Request Request Cancel Cancel
135. 46 Механика Flow[T] Request Request T Error Complete Cancel Cancel
136. 46 Механика Flow[T] T Error Complete Request Request T Error Complete Cancel Cancel
137. • Emit элемента • Завершение • Ошибка • Backpressure • Отмена 47 Что нужно знать про stages
138. 48 Свой Source val s: Source[Int, ActorRef] =
 Source.actorPublisher[Int](Props(new MySourceActor))
 
 class MySourceActor extends ActorPublisher[Int] {
 override def receive: Receive = {
 case ActorPublisherMessage.Request(n) =>
 onNext(100)
 onCompleteThenStop()
 onErrorThenStop(new Exception)
 case ActorPublisherMessage.Cancel =>
 context.stop(self)
 }
 }

139. 48 Свой Source val s: Source[Int, ActorRef] =
 Source.actorPublisher[Int](Props(new MySourceActor))
 
 class MySourceActor extends ActorPublisher[Int] {
 override def receive: Receive = {
 case ActorPublisherMessage.Request(n) =>
 onNext(100)
 onCompleteThenStop()
 onErrorThenStop(new Exception)
 case ActorPublisherMessage.Cancel =>
 context.stop(self)
 }
 }

140. 48 Свой Source val s: Source[Int, ActorRef] =
 Source.actorPublisher[Int](Props(new MySourceActor))
 
 class MySourceActor extends ActorPublisher[Int] {
 override def receive: Receive = {
 case ActorPublisherMessage.Request(n) =>
 onNext(100)
 onCompleteThenStop()
 onErrorThenStop(new Exception)
 case ActorPublisherMessage.Cancel =>
 context.stop(self)
 }
 }

141. 48 Свой Source val s: Source[Int, ActorRef] =
 Source.actorPublisher[Int](Props(new MySourceActor))
 
 class MySourceActor extends ActorPublisher[Int] {
 override def receive: Receive = {
 case ActorPublisherMessage.Request(n) =>
 onNext(100)
 onCompleteThenStop()
 onErrorThenStop(new Exception)
 case ActorPublisherMessage.Cancel =>
 context.stop(self)
 }
 }

142. 48 Свой Source val s: Source[Int, ActorRef] =
 Source.actorPublisher[Int](Props(new MySourceActor))
 
 class MySourceActor extends ActorPublisher[Int] {
 override def receive: Receive = {
 case ActorPublisherMessage.Request(n) =>
 onNext(100)
 onCompleteThenStop()
 onErrorThenStop(new Exception)
 case ActorPublisherMessage.Cancel =>
 context.stop(self)
 }
 }

143. 48 Свой Source val s: Source[Int, ActorRef] =
 Source.actorPublisher[Int](Props(new MySourceActor))
 
 class MySourceActor extends ActorPublisher[Int] {
 override def receive: Receive = {
 case ActorPublisherMessage.Request(n) =>
 onNext(100)
 onCompleteThenStop()
 onErrorThenStop(new Exception)
 case ActorPublisherMessage.Cancel =>
 context.stop(self)
 }
 }

144. 48 Свой Source val s: Source[Int, ActorRef] =
 Source.actorPublisher[Int](Props(new MySourceActor))
 
 class MySourceActor extends ActorPublisher[Int] {
 override def receive: Receive = {
 case ActorPublisherMessage.Request(n) =>
 onNext(100)
 onCompleteThenStop()
 onErrorThenStop(new Exception)
 case ActorPublisherMessage.Cancel =>
 context.stop(self)
 }
 }

145. 48 Свой Source val s: Source[Int, ActorRef] =
 Source.actorPublisher[Int](Props(new MySourceActor))
 
 class MySourceActor extends ActorPublisher[Int] {
 override def receive: Receive = {
 case ActorPublisherMessage.Request(n) =>
 onNext(100)
 onCompleteThenStop()
 onErrorThenStop(new Exception)
 case ActorPublisherMessage.Cancel =>
 context.stop(self)
 }
 }

146. 48 Свой Source val s: Source[Int, ActorRef] =
 Source.actorPublisher[Int](Props(new MySourceActor))
 
 class MySourceActor extends ActorPublisher[Int] {
 override def receive: Receive = {
 case ActorPublisherMessage.Request(n) =>
 onNext(100)
 onCompleteThenStop()
 onErrorThenStop(new Exception)
 case ActorPublisherMessage.Cancel =>
 context.stop(self)
 }
 }

147. 48 Свой Source val s: Source[Int, ActorRef] =
 Source.actorPublisher[Int](Props(new MySourceActor))
 
 class MySourceActor extends ActorPublisher[Int] {
 override def receive: Receive = {
 case ActorPublisherMessage.Request(n) =>
 onNext(100)
 onCompleteThenStop()
 onErrorThenStop(new Exception)
 case ActorPublisherMessage.Cancel =>
 context.stop(self)
 }
 }

148. 49 Reactive Zmq case object GracefulStop
 case object DeliverMore
 class MySourceActor(addr: String) extends ActorPublisher[Array[Byte]] {
 var conn: ZMQ.Socket = ???
 
 override def preStart(): Unit = {
 conn.connect(addr)
 }
 
 override def postStop(): Unit = {
 conn.disconnect(addr)
 }

149. 49 Reactive Zmq case object GracefulStop
 case object DeliverMore
 class MySourceActor(addr: String) extends ActorPublisher[Array[Byte]] {
 var conn: ZMQ.Socket = ???
 
 override def preStart(): Unit = {
 conn.connect(addr)
 }
 
 override def postStop(): Unit = {
 conn.disconnect(addr)
 }

150. 49 Reactive Zmq case object GracefulStop
 case object DeliverMore
 class MySourceActor(addr: String) extends ActorPublisher[Array[Byte]] {
 var conn: ZMQ.Socket = ???
 
 override def preStart(): Unit = {
 conn.connect(addr)
 }
 
 override def postStop(): Unit = {
 conn.disconnect(addr)
 }

151. 49 Reactive Zmq case object GracefulStop
 case object DeliverMore
 class MySourceActor(addr: String) extends ActorPublisher[Array[Byte]] {
 var conn: ZMQ.Socket = ???
 
 override def preStart(): Unit = {
 conn.connect(addr)
 }
 
 override def postStop(): Unit = {
 conn.disconnect(addr)
 }

152. 49 Reactive Zmq case object GracefulStop
 case object DeliverMore
 class MySourceActor(addr: String) extends ActorPublisher[Array[Byte]] {
 var conn: ZMQ.Socket = ???
 
 override def preStart(): Unit = {
 conn.connect(addr)
 }
 
 override def postStop(): Unit = {
 conn.disconnect(addr)
 }

153. 49 Reactive Zmq case object GracefulStop
 case object DeliverMore
 class MySourceActor(addr: String) extends ActorPublisher[Array[Byte]] {
 var conn: ZMQ.Socket = ???
 
 override def preStart(): Unit = {
 conn.connect(addr)
 }
 
 override def postStop(): Unit = {
 conn.disconnect(addr)
 }

154. 49 Reactive Zmq case object GracefulStop
 case object DeliverMore
 class MySourceActor(addr: String) extends ActorPublisher[Array[Byte]] {
 var conn: ZMQ.Socket = ???
 
 override def preStart(): Unit = {
 conn.connect(addr)
 }
 
 override def postStop(): Unit = {
 conn.disconnect(addr)
 }

155. 50 Reactive Zmq override def receive: Receive = {
 case ActorPublisherMessage.Request(_) | DeliverMore =>
 Option(conn.recv()) match {
 case Some(msg) =>
 onNext(msg)
 if(totalDemand > 0) self ! DeliverMore
 case None => self ! DeliverMore
 }
 case ActorPublisherMessage.Cancel | GracefulStop =>
 conn.disconnect(addr)
 onCompleteThenStop()
 }

156. 50 Reactive Zmq override def receive: Receive = {
 case ActorPublisherMessage.Request(_) | DeliverMore =>
 Option(conn.recv()) match {
 case Some(msg) =>
 onNext(msg)
 if(totalDemand > 0) self ! DeliverMore
 case None => self ! DeliverMore
 }
 case ActorPublisherMessage.Cancel | GracefulStop =>
 conn.disconnect(addr)
 onCompleteThenStop()
 }

157. 50 Reactive Zmq override def receive: Receive = {
 case ActorPublisherMessage.Request(_) | DeliverMore =>
 Option(conn.recv()) match {
 case Some(msg) =>
 onNext(msg)
 if(totalDemand > 0) self ! DeliverMore
 case None => self ! DeliverMore
 }
 case ActorPublisherMessage.Cancel | GracefulStop =>
 conn.disconnect(addr)
 onCompleteThenStop()
 }

158. 50 Reactive Zmq override def receive: Receive = {
 case ActorPublisherMessage.Request(_) | DeliverMore =>
 Option(conn.recv()) match {
 case Some(msg) =>
 onNext(msg)
 if(totalDemand > 0) self ! DeliverMore
 case None => self ! DeliverMore
 }
 case ActorPublisherMessage.Cancel | GracefulStop =>
 conn.disconnect(addr)
 onCompleteThenStop()
 }

159. 50 Reactive Zmq override def receive: Receive = {
 case ActorPublisherMessage.Request(_) | DeliverMore =>
 Option(conn.recv()) match {
 case Some(msg) =>
 onNext(msg)
 if(totalDemand > 0) self ! DeliverMore
 case None => self ! DeliverMore
 }
 case ActorPublisherMessage.Cancel | GracefulStop =>
 conn.disconnect(addr)
 onCompleteThenStop()
 }

160. 50 Reactive Zmq override def receive: Receive = {
 case ActorPublisherMessage.Request(_) | DeliverMore =>
 Option(conn.recv()) match {
 case Some(msg) =>
 onNext(msg)
 if(totalDemand > 0) self ! DeliverMore
 case None => self ! DeliverMore
 }
 case ActorPublisherMessage.Cancel | GracefulStop =>
 conn.disconnect(addr)
 onCompleteThenStop()
 }

161. 50 Reactive Zmq override def receive: Receive = {
 case ActorPublisherMessage.Request(_) | DeliverMore =>
 Option(conn.recv()) match {
 case Some(msg) =>
 onNext(msg)
 if(totalDemand > 0) self ! DeliverMore
 case None => self ! DeliverMore
 }
 case ActorPublisherMessage.Cancel | GracefulStop =>
 conn.disconnect(addr)
 onCompleteThenStop()
 }

162. 50 Reactive Zmq override def receive: Receive = {
 case ActorPublisherMessage.Request(_) | DeliverMore =>
 Option(conn.recv()) match {
 case Some(msg) =>
 onNext(msg)
 if(totalDemand > 0) self ! DeliverMore
 case None => self ! DeliverMore
 }
 case ActorPublisherMessage.Cancel | GracefulStop =>
 conn.disconnect(addr)
 onCompleteThenStop()
 }

163. 50 Reactive Zmq override def receive: Receive = {
 case ActorPublisherMessage.Request(_) | DeliverMore =>
 Option(conn.recv()) match {
 case Some(msg) =>
 onNext(msg)
 if(totalDemand > 0) self ! DeliverMore
 case None => self ! DeliverMore
 }
 case ActorPublisherMessage.Cancel | GracefulStop =>
 conn.disconnect(addr)
 onCompleteThenStop()
 }

164. 50 Reactive Zmq override def receive: Receive = {
 case ActorPublisherMessage.Request(_) | DeliverMore =>
 Option(conn.recv()) match {
 case Some(msg) =>
 onNext(msg)
 if(totalDemand > 0) self ! DeliverMore
 case None => self ! DeliverMore
 }
 case ActorPublisherMessage.Cancel | GracefulStop =>
 conn.disconnect(addr)
 onCompleteThenStop()
 }

165. 50 Reactive Zmq override def receive: Receive = {
 case ActorPublisherMessage.Request(_) | DeliverMore =>
 Option(conn.recv()) match {
 case Some(msg) =>
 onNext(msg)
 if(totalDemand > 0) self ! DeliverMore
 case None => self ! DeliverMore
 }
 case ActorPublisherMessage.Cancel | GracefulStop =>
 conn.disconnect(addr)
 onCompleteThenStop()
 }

166. 51 Reactive Zmq val source: Source[Array[Byte], Control] =
 Source.actorPublisher[Array[Byte]](
 Props(new MySourceActor( ???))
 ).mapMaterializedValue { ref =>
 new Control {
 override def gracefulStop() = ref ! GracefulStop
 }
 }
167. 51 Reactive Zmq val source: Source[Array[Byte], Control] =
 Source.actorPublisher[Array[Byte]](
 Props(new MySourceActor( ???))
 ).mapMaterializedValue { ref =>
 new Control {
 override def gracefulStop() = ref ! GracefulStop
 }
 }
168. 51 Reactive Zmq val source: Source[Array[Byte], Control] =
 Source.actorPublisher[Array[Byte]](
 Props(new MySourceActor( ???))
 ).mapMaterializedValue { ref =>
 new Control {
 override def gracefulStop() = ref ! GracefulStop
 }
 }
169. 51 Reactive Zmq val source: Source[Array[Byte], Control] =
 Source.actorPublisher[Array[Byte]](
 Props(new MySourceActor( ???))
 ).mapMaterializedValue { ref =>
 new Control {
 override def gracefulStop() = ref ! GracefulStop
 }
 }
170. 51 Reactive Zmq val source: Source[Array[Byte], Control] =
 Source.actorPublisher[Array[Byte]](
 Props(new MySourceActor( ???))
 ).mapMaterializedValue { ref =>
 new Control {
 override def gracefulStop() = ref ! GracefulStop
 }
 }
171. 51 Reactive Zmq val source: Source[Array[Byte], Control] =
 Source.actorPublisher[Array[Byte]](
 Props(new MySourceActor( ???))
 ).mapMaterializedValue { ref =>
 new Control {
 override def gracefulStop() = ref ! GracefulStop
 }
 }
172. 51 Reactive Zmq val source: Source[Array[Byte], Control] =
 Source.actorPublisher[Array[Byte]](
 Props(new MySourceActor( ???))
 ).mapMaterializedValue { ref =>
 new Control {
 override def gracefulStop() = ref ! GracefulStop
 }
 }
173. 52 Свой Sink val s = Sink.actorSubscriber[Int](Props(new MySinkActor))
 
 class MySinkActor extends ActorSubscriber {
 override def receive: Receive = {
 case ActorSubscriberMessage.OnNext =>
 cancel()
 request(100)
 case ActorSubscriberMessage.OnComplete =>
 case ActorSubscriberMessage.OnError =>
 }
174. 52 Свой Sink val s = Sink.actorSubscriber[Int](Props(new MySinkActor))
 
 class MySinkActor extends ActorSubscriber {
 override def receive: Receive = {
 case ActorSubscriberMessage.OnNext =>
 cancel()
 request(100)
 case ActorSubscriberMessage.OnComplete =>
 case ActorSubscriberMessage.OnError =>
 }
175. 52 Свой Sink val s = Sink.actorSubscriber[Int](Props(new MySinkActor))
 
 class MySinkActor extends ActorSubscriber {
 override def receive: Receive = {
 case ActorSubscriberMessage.OnNext =>
 cancel()
 request(100)
 case ActorSubscriberMessage.OnComplete =>
 case ActorSubscriberMessage.OnError =>
 }
176. 52 Свой Sink val s = Sink.actorSubscriber[Int](Props(new MySinkActor))
 
 class MySinkActor extends ActorSubscriber {
 override def receive: Receive = {
 case ActorSubscriberMessage.OnNext =>
 cancel()
 request(100)
 case ActorSubscriberMessage.OnComplete =>
 case ActorSubscriberMessage.OnError =>
 }
177. 52 Свой Sink val s = Sink.actorSubscriber[Int](Props(new MySinkActor))
 
 class MySinkActor extends ActorSubscriber {
 override def receive: Receive = {
 case ActorSubscriberMessage.OnNext =>
 cancel()
 request(100)
 case ActorSubscriberMessage.OnComplete =>
 case ActorSubscriberMessage.OnError =>
 }
178. 52 Свой Sink val s = Sink.actorSubscriber[Int](Props(new MySinkActor))
 
 class MySinkActor extends ActorSubscriber {
 override def receive: Receive = {
 case ActorSubscriberMessage.OnNext =>
 cancel()
 request(100)
 case ActorSubscriberMessage.OnComplete =>
 case ActorSubscriberMessage.OnError =>
 }
179. 52 Свой Sink val s = Sink.actorSubscriber[Int](Props(new MySinkActor))
 
 class MySinkActor extends ActorSubscriber {
 override def receive: Receive = {
 case ActorSubscriberMessage.OnNext =>
 cancel()
 request(100)
 case ActorSubscriberMessage.OnComplete =>
 case ActorSubscriberMessage.OnError =>
 }
180. 52 Свой Sink val s = Sink.actorSubscriber[Int](Props(new MySinkActor))
 
 class MySinkActor extends ActorSubscriber {
 override def receive: Receive = {
 case ActorSubscriberMessage.OnNext =>
 cancel()
 request(100)
 case ActorSubscriberMessage.OnComplete =>
 case ActorSubscriberMessage.OnError =>
 }
181. 52 Свой Sink val s = Sink.actorSubscriber[Int](Props(new MySinkActor))
 
 class MySinkActor extends ActorSubscriber {
 override def receive: Receive = {
 case ActorSubscriberMessage.OnNext =>
 cancel()
 request(100)
 case ActorSubscriberMessage.OnComplete =>
 case ActorSubscriberMessage.OnError =>
 }
182. Итоги
183. • Стандартные комбинаторы • Материализация • Управление жизненным циклом • Свои stages 54 Итоги
184. Серебряная пуля
185. • Более простая технология не подходит • Backpressure • Упорядоченные потоки • Локальность • Фиксированная топология 56 Когда стоит использовать
186. • Документация Akka Streams • Альпака • reactive-zmq • reactive-kafka • Первое видео про Akka Streams • Слайды этой презентации - goo.gl/F7EYoE 57 Ссылки
187. Вопросы? alexey.romanchuk@gmail.com @1esha
188. Backpressure
189. 60 Событий слишком много
190. 60 Событий слишком много Отправитель
191. 60 Событий слишком много Отправитель Получатель
192. 60 Событий слишком много Отправитель Получатель
193. 61 Событий слишком много Отправитель Получатель
194. 62 Событий слишком много Отправитель Получатель
195. 63 Событий слишком много Отправитель Получатель
196. 64 Событий слишком много Отправитель Получатель
197. 65 Событий слишком много Отправитель Получатель
198. 66 Событий слишком много Отправитель Получатель
199. 67 Событий слишком много Отправитель Получатель
200. 68 Блокирующий вызов Отправитель Получатель
201. 68 Блокирующий вызов Отправитель Получатель
202. 68 Блокирующий вызов Отправитель Получатель
203. 69 Pull Отправитель Получатель
204. 69 Pull Отправитель Получатель
205. 69 Pull Отправитель Получатель
206. 70 Negative Acknowledge Отправитель Получатель
207. 70 Negative Acknowledge Отправитель Получатель
208. 70 Negative Acknowledge Отправитель Получатель
209. 71 Negative Acknowledge Отправитель Получатель
210. 72 Dynamic pull-push Отправитель Получатель
211. 72 Dynamic pull-push Отправитель Получатель 4
212. 73 Dynamic pull-push 4 Отправитель Получатель
213. 74 Dynamic pull-push 3 Отправитель Получатель
214. 75 Dynamic pull-push 3 Отправитель Получатель
215. 76 Dynamic pull-push 0 Отправитель Получатель
216. 77 Dynamic pull-push 0 Отправитель Получатель
217. 78 Dynamic pull-push 0 Отправитель Получатель
218. 78 Dynamic pull-push 0 Отправитель Получатель 2
219. 79 Dynamic pull-push 2 Отправитель Получатель
220. 80 Dynamic pull-push 0 Отправитель Получатель
221. 81 Balance Balance
222. 81 Balance Balance
223. 81 Balance Balance
224. 82 Broadcast Broadcast
225. 82 Broadcast Broadcast
226. 82 Broadcast Broadcast
227. 83 Zip Zip
228. 83 Zip Zip
229. 83 Zip Zip
230. 84 Unzip Unzip
231. 84 Unzip Unzip
232. 84 Unzip Unzip
233. 85 Красивый код на стримах val stream: RunnableGraph[Future[List[TransactionId]]] =
 Source
 .fromIterator { () => hdfs.listStatusIterator(path) }
 .filter(_.isFile)
 .flatMapConcat { file =>
 val s = fromInputStream(hdfs.open(file.getPath))
 Source.fromIterator(() => s.getLines())
 }
 .throttle(10, 1 second, 0, ThrottleMode.Shaping)
 .mapAsyncUnordered(50)(checkFraud)
 .collect { case Some(value) => value }
 .take(100)
 .fold(List.empty[TransactionId])(_ :+ _)
 .toMat(Sink.head)(Keep.right)

234. 85 Красивый код на стримах val stream: RunnableGraph[Future[List[TransactionId]]] =
 Source
 .fromIterator { () => hdfs.listStatusIterator(path) }
 .filter(_.isFile)
 .flatMapConcat { file =>
 val s = fromInputStream(hdfs.open(file.getPath))
 Source.fromIterator(() => s.getLines())
 }
 .throttle(10, 1 second, 0, ThrottleMode.Shaping)
 .mapAsyncUnordered(50)(checkFraud)
 .collect { case Some(value) => value }
 .take(100)
 .fold(List.empty[TransactionId])(_ :+ _)
 .toMat(Sink.head)(Keep.right)

235. 85 Красивый код на стримах val stream: RunnableGraph[Future[List[TransactionId]]] =
 Source
 .fromIterator { () => hdfs.listStatusIterator(path) }
 .filter(_.isFile)
 .flatMapConcat { file =>
 val s = fromInputStream(hdfs.open(file.getPath))
 Source.fromIterator(() => s.getLines())
 }
 .throttle(10, 1 second, 0, ThrottleMode.Shaping)
 .mapAsyncUnordered(50)(checkFraud)
 .collect { case Some(value) => value }
 .take(100)
 .fold(List.empty[TransactionId])(_ :+ _)
 .toMat(Sink.head)(Keep.right)

236. 85 Красивый код на стримах val stream: RunnableGraph[Future[List[TransactionId]]] =
 Source
 .fromIterator { () => hdfs.listStatusIterator(path) }
 .filter(_.isFile)
 .flatMapConcat { file =>
 val s = fromInputStream(hdfs.open(file.getPath))
 Source.fromIterator(() => s.getLines())
 }
 .throttle(10, 1 second, 0, ThrottleMode.Shaping)
 .mapAsyncUnordered(50)(checkFraud)
 .collect { case Some(value) => value }
 .take(100)
 .fold(List.empty[TransactionId])(_ :+ _)
 .toMat(Sink.head)(Keep.right)

237. 85 Красивый код на стримах val stream: RunnableGraph[Future[List[TransactionId]]] =
 Source
 .fromIterator { () => hdfs.listStatusIterator(path) }
 .filter(_.isFile)
 .flatMapConcat { file =>
 val s = fromInputStream(hdfs.open(file.getPath))
 Source.fromIterator(() => s.getLines())
 }
 .throttle(10, 1 second, 0, ThrottleMode.Shaping)
 .mapAsyncUnordered(50)(checkFraud)
 .collect { case Some(value) => value }
 .take(100)
 .fold(List.empty[TransactionId])(_ :+ _)
 .toMat(Sink.head)(Keep.right)

238. 85 Красивый код на стримах val stream: RunnableGraph[Future[List[TransactionId]]] =
 Source
 .fromIterator { () => hdfs.listStatusIterator(path) }
 .filter(_.isFile)
 .flatMapConcat { file =>
 val s = fromInputStream(hdfs.open(file.getPath))
 Source.fromIterator(() => s.getLines())
 }
 .throttle(10, 1 second, 0, ThrottleMode.Shaping)
 .mapAsyncUnordered(50)(checkFraud)
 .collect { case Some(value) => value }
 .take(100)
 .fold(List.empty[TransactionId])(_ :+ _)
 .toMat(Sink.head)(Keep.right)

239. 85 Красивый код на стримах val stream: RunnableGraph[Future[List[TransactionId]]] =
 Source
 .fromIterator { () => hdfs.listStatusIterator(path) }
 .filter(_.isFile)
 .flatMapConcat { file =>
 val s = fromInputStream(hdfs.open(file.getPath))
 Source.fromIterator(() => s.getLines())
 }
 .throttle(10, 1 second, 0, ThrottleMode.Shaping)
 .mapAsyncUnordered(50)(checkFraud)
 .collect { case Some(value) => value }
 .take(100)
 .fold(List.empty[TransactionId])(_ :+ _)
 .toMat(Sink.head)(Keep.right)

240. 85 Красивый код на стримах val stream: RunnableGraph[Future[List[TransactionId]]] =
 Source
 .fromIterator { () => hdfs.listStatusIterator(path) }
 .filter(_.isFile)
 .flatMapConcat { file =>
 val s = fromInputStream(hdfs.open(file.getPath))
 Source.fromIterator(() => s.getLines())
 }
 .throttle(10, 1 second, 0, ThrottleMode.Shaping)
 .mapAsyncUnordered(50)(checkFraud)
 .collect { case Some(value) => value }
 .take(100)
 .fold(List.empty[TransactionId])(_ :+ _)
 .toMat(Sink.head)(Keep.right)

241. 85 Красивый код на стримах val stream: RunnableGraph[Future[List[TransactionId]]] =
 Source
 .fromIterator { () => hdfs.listStatusIterator(path) }
 .filter(_.isFile)
 .flatMapConcat { file =>
 val s = fromInputStream(hdfs.open(file.getPath))
 Source.fromIterator(() => s.getLines())
 }
 .throttle(10, 1 second, 0, ThrottleMode.Shaping)
 .mapAsyncUnordered(50)(checkFraud)
 .collect { case Some(value) => value }
 .take(100)
 .fold(List.empty[TransactionId])(_ :+ _)
 .toMat(Sink.head)(Keep.right)

242. 85 Красивый код на стримах val stream: RunnableGraph[Future[List[TransactionId]]] =
 Source
 .fromIterator { () => hdfs.listStatusIterator(path) }
 .filter(_.isFile)
 .flatMapConcat { file =>
 val s = fromInputStream(hdfs.open(file.getPath))
 Source.fromIterator(() => s.getLines())
 }
 .throttle(10, 1 second, 0, ThrottleMode.Shaping)
 .mapAsyncUnordered(50)(checkFraud)
 .collect { case Some(value) => value }
 .take(100)
 .fold(List.empty[TransactionId])(_ :+ _)
 .toMat(Sink.head)(Keep.right)

243. 85 Красивый код на стримах val stream: RunnableGraph[Future[List[TransactionId]]] =
 Source
 .fromIterator { () => hdfs.listStatusIterator(path) }
 .filter(_.isFile)
 .flatMapConcat { file =>
 val s = fromInputStream(hdfs.open(file.getPath))
 Source.fromIterator(() => s.getLines())
 }
 .throttle(10, 1 second, 0, ThrottleMode.Shaping)
 .mapAsyncUnordered(50)(checkFraud)
 .collect { case Some(value) => value }
 .take(100)
 .fold(List.empty[TransactionId])(_ :+ _)
 .toMat(Sink.head)(Keep.right)

244. 85 Красивый код на стримах val stream: RunnableGraph[Future[List[TransactionId]]] =
 Source
 .fromIterator { () => hdfs.listStatusIterator(path) }
 .filter(_.isFile)
 .flatMapConcat { file =>
 val s = fromInputStream(hdfs.open(file.getPath))
 Source.fromIterator(() => s.getLines())
 }
 .throttle(10, 1 second, 0, ThrottleMode.Shaping)
 .mapAsyncUnordered(50)(checkFraud)
 .collect { case Some(value) => value }
 .take(100)
 .fold(List.empty[TransactionId])(_ :+ _)
 .toMat(Sink.head)(Keep.right)

245. 85 Красивый код на стримах val stream: RunnableGraph[Future[List[TransactionId]]] =
 Source
 .fromIterator { () => hdfs.listStatusIterator(path) }
 .filter(_.isFile)
 .flatMapConcat { file =>
 val s = fromInputStream(hdfs.open(file.getPath))
 Source.fromIterator(() => s.getLines())
 }
 .throttle(10, 1 second, 0, ThrottleMode.Shaping)
 .mapAsyncUnordered(50)(checkFraud)
 .collect { case Some(value) => value }
 .take(100)
 .fold(List.empty[TransactionId])(_ :+ _)
 .toMat(Sink.head)(Keep.right)

246. 85 Красивый код на стримах val stream: RunnableGraph[Future[List[TransactionId]]] =
 Source
 .fromIterator { () => hdfs.listStatusIterator(path) }
 .filter(_.isFile)
 .flatMapConcat { file =>
 val s = fromInputStream(hdfs.open(file.getPath))
 Source.fromIterator(() => s.getLines())
 }
 .throttle(10, 1 second, 0, ThrottleMode.Shaping)
 .mapAsyncUnordered(50)(checkFraud)
 .collect { case Some(value) => value }
 .take(100)
 .fold(List.empty[TransactionId])(_ :+ _)
 .toMat(Sink.head)(Keep.right)

247. 85 Красивый код на стримах val stream: RunnableGraph[Future[List[TransactionId]]] =
 Source
 .fromIterator { () => hdfs.listStatusIterator(path) }
 .filter(_.isFile)
 .flatMapConcat { file =>
 val s = fromInputStream(hdfs.open(file.getPath))
 Source.fromIterator(() => s.getLines())
 }
 .throttle(10, 1 second, 0, ThrottleMode.Shaping)
 .mapAsyncUnordered(50)(checkFraud)
 .collect { case Some(value) => value }
 .take(100)
 .fold(List.empty[TransactionId])(_ :+ _)
 .toMat(Sink.head)(Keep.right)

248. 85 Красивый код на стримах val stream: RunnableGraph[Future[List[TransactionId]]] =
 Source
 .fromIterator { () => hdfs.listStatusIterator(path) }
 .filter(_.isFile)
 .flatMapConcat { file =>
 val s = fromInputStream(hdfs.open(file.getPath))
 Source.fromIterator(() => s.getLines())
 }
 .throttle(10, 1 second, 0, ThrottleMode.Shaping)
 .mapAsyncUnordered(50)(checkFraud)
 .collect { case Some(value) => value }
 .take(100)
 .fold(List.empty[TransactionId])(_ :+ _)
 .toMat(Sink.head)(Keep.right)