Functional Flowing

KotlinX’s Flow data type is extremely powerful, and offers an API that is quite familiar for functional programmers.
This talk will showcase how we can leverage KotlinX Flow to describe powerful programs, and build pipelines to transform and manipulate data in an efficient streaming way.

Transcript

1. Functional Flowing
   47deg.com
   

   View Slide

2. Who am I?
   

   View Slide

3. KotlinX Flow
   Pull based
   Reasoning about Flow
   Streaming Resources
   Parallel Processing
   

   View Slide

4. Pull Based Streaming
   Upstream
   Pipeline Downstream
   

   View Slide

5. Pull Based Streaming
   Upstream
   Pipeline Downstream
   

   View Slide

6. Pull Based Streaming
   public fun interface FlowCollector {
   public suspend fun emit(value: T)
   }
   

   View Slide

7. Creating Flows
   class Circle(val count: Int)
   suspend fun FlowCollector.produceCircles(): Unit {
   var counter: Int = 0
   while(true) {
   val circle = Circle(counter>+)
   println("Creating $circle")
   emit(circle)
   }
   }
   

   View Slide

8. Creating Flows
   class Circle(val count: Int)
   suspend fun FlowCollector.produceCircles(): Unit {
   var counter: Int = 0
   while(true) {
   val circle = Circle(counter>+)
   println("Creating $circle")
   emit(circle)
   }
   }
   val circles: Flow =
   flow { produceCircles() }
   

   View Slide

9. Creating Flows
   class Circle(val count: Int)
   val circles: Flow = flow {
   var counter: Int = 0
   while(true) {
   val circle = Circle(counter>+)
   println("Creating $circle")
   emit(circle)
   }
   }
   

   View Slide

10. Pipeline
    class Square private constructor(val circle: Circle) {
    companion object {
    suspend fun Circle.toSquare(): Square {
    print("Creating Square($count)")
    delay(1.seconds)
    return Square(this).also { println("Created $this") }
    }
    }
    }
    

    View Slide

11. Putting it all together
    fun main() = runBlocking {
    circles
    .take(3)
    .collect { circle ->
    circle.toSquare()
    }
    }
    Creating Circle@63e31ee
    Creating Square(0)
    Created Square@6adca536
    Creating Circle@23223dd8
    

    View Slide

12. Reasoning about Flows
    suspend fun FlowCollector.produceCircles(): Unit {
    var counter: Int = 0
    while(true) {
    val circle = Circle(counter>+)
    println("Creating $circle")
    emit(circle)
    }
    }
    

    View Slide

13. Reasoning about Flows
    class Square private constructor(val circle: Circle) {
    companion object {
    suspend fun Circle.toSquare(): Square {
    print("Creating Square($count)")
    delay(1.seconds)
    return Square(this).also { println("Created $this") }
    }
    }
    }
    

    View Slide

14. Reasoning about Flows
    while(true)
    0 1 2
    Coroutine
    

    View Slide

15. Flowing resources
    fun Path.readAll(): Flow
    

    View Slide

16. Flowing resources
    fun Path.readAll(): Flow = flow { >/ this: FlowCollector
    useLines { lines ->
    lines.forEach { line -> emit(line) }
    }
    }
    

    View Slide

17. Flowing resources
    suspend fun insertIntoBlobStorage(line: String): Unit
    

    View Slide

18. Flowing resources
    /** Java SDK */
    class BlobStorage(val blobName: String) : AutoCloseable {
    fun uploadBlob(data: ByteArray): CompletableFuture
    override fun close() = Unit
    }
    suspend fun BlobStorage.insertIntoBlobStorage(line: String): UploadResult =
    uploadBlob(line.toByteArray(Charsets.UTF_8)).await()
    

    View Slide

19. Flowing resources
    suspend fun uploadData(path: String): Unit =
    BlobStorage(path).use { storage ->
    Path(path)
    .readAll()
    .collect { line ->
    storage.insertIntoBlobStorage(line)
    }
    }
    

    View Slide

20. Flowing resources
    fun blobStorage(path: String): Flow = flow {
    BlobStorage(path).use { emit(it) }
    }
    

    View Slide

21. Flowing resources
    suspend fun uploadData2(path: String): Flow =
    blobStorage().flatMapConcat { blobStorage ->
    Path("data.txt")
    .readAll()
    .map { line ->
    blobStorage.insertIntoBlobStorage(line)
    }
    }
    

    View Slide

22. Parallel Processing
    Read line Upload blob
    Coroutine
    …
    

    View Slide

23. Parallel Processing
    Read line Coroutine Upload blob Coroutine
    Channel(3)
    

    View Slide

24. Parallel Processing
    suspend fun uploadData3(path: String): Flow =
    blobStorage().flatMapConcat { blobStorage ->
    Path("data.txt")
    .readAll()
    .parMapUnordered(concurrency = 3) { line ->
    blobStorage.insertIntoBlobStorage(line)
    }
    }
    

    View Slide

25. Parallel Processing
    @FlowPreview
    public inline fun Flow.parMapUnordered(
    concurrency: Int = DEFAULT_CONCURRENCY,
    crossinline transform: suspend (a: A) >> B
    ): Flow {
    val flowOfFlows: Flow> = map { o ->
    flow {
    emit(transform(o))
    }
    }
    return parallelFlows.flattenMerge(concurrency)
    }
    

    View Slide

26. Parallel Processing
    suspend fun uploadData3(path: String): Flow =
    blobStorage().flatMapConcat { blobStorage ->
    Path("data.txt")
    .readAll()
    .parMap(concurrency = 3) { line ->
    blobStorage.insertIntoBlobStorage(line)
    }
    }
    

    View Slide

27. Compose-able streaming
    Automatic back-pressure
    Conclusion
    Suspend everywhere
    

    View Slide

28. Thanks!
    

    View Slide