Upgrade to Pro — share decks privately, control downloads, hide ads and more …

Functional Flowing

Functional Flowing

KotlinX’s Flow data type is extremely powerful, and offers an API that is quite familiar for functional programmers.
This talk will showcase how we can leverage KotlinX Flow to describe powerful programs, and build pipelines to transform and manipulate data in an efficient streaming way.

Simon Vergauwen

November 26, 2022
Tweet

More Decks by Simon Vergauwen

Other Decks in Programming

Transcript

  1. Functional Flowing
    47deg.com

    View Slide

  2. Who am I?

    View Slide

  3. KotlinX Flow
    Pull based
    Reasoning about Flow
    Streaming Resources
    Parallel Processing

    View Slide

  4. Pull Based Streaming
    Upstream
    Pipeline Downstream

    View Slide

  5. Pull Based Streaming
    Upstream
    Pipeline Downstream

    View Slide

  6. Pull Based Streaming
    public fun interface FlowCollector {
    public suspend fun emit(value: T)
    }

    View Slide

  7. Creating Flows
    class Circle(val count: Int)
    suspend fun FlowCollector.produceCircles(): Unit {
    var counter: Int = 0
    while(true) {
    val circle = Circle(counter>+)
    println("Creating $circle")
    emit(circle)
    }
    }

    View Slide

  8. Creating Flows
    class Circle(val count: Int)
    suspend fun FlowCollector.produceCircles(): Unit {
    var counter: Int = 0
    while(true) {
    val circle = Circle(counter>+)
    println("Creating $circle")
    emit(circle)
    }
    }
    val circles: Flow =
    flow { produceCircles() }

    View Slide

  9. Creating Flows
    class Circle(val count: Int)
    val circles: Flow = flow {
    var counter: Int = 0
    while(true) {
    val circle = Circle(counter>+)
    println("Creating $circle")
    emit(circle)
    }
    }

    View Slide

  10. Pipeline
    class Square private constructor(val circle: Circle) {
    companion object {
    suspend fun Circle.toSquare(): Square {
    print("Creating Square($count)")
    delay(1.seconds)
    return Square(this).also { println("Created $this") }
    }
    }
    }

    View Slide

  11. Putting it all together
    fun main() = runBlocking {
    circles
    .take(3)
    .collect { circle ->
    circle.toSquare()
    }
    }
    Creating [email protected]
    Creating Square(0)
    Created [email protected]
    Creating [email protected]

    View Slide

  12. Reasoning about Flows
    suspend fun FlowCollector.produceCircles(): Unit {
    var counter: Int = 0
    while(true) {
    val circle = Circle(counter>+)
    println("Creating $circle")
    emit(circle)
    }
    }

    View Slide

  13. Reasoning about Flows
    class Square private constructor(val circle: Circle) {
    companion object {
    suspend fun Circle.toSquare(): Square {
    print("Creating Square($count)")
    delay(1.seconds)
    return Square(this).also { println("Created $this") }
    }
    }
    }

    View Slide

  14. Reasoning about Flows
    while(true)
    0 1 2
    Coroutine

    View Slide

  15. Flowing resources
    fun Path.readAll(): Flow

    View Slide

  16. Flowing resources
    fun Path.readAll(): Flow = flow { >/ this: FlowCollector
    useLines { lines ->
    lines.forEach { line -> emit(line) }
    }
    }

    View Slide

  17. Flowing resources
    suspend fun insertIntoBlobStorage(line: String): Unit

    View Slide

  18. Flowing resources
    /** Java SDK */
    class BlobStorage(val blobName: String) : AutoCloseable {
    fun uploadBlob(data: ByteArray): CompletableFuture
    override fun close() = Unit
    }
    suspend fun BlobStorage.insertIntoBlobStorage(line: String): UploadResult =
    uploadBlob(line.toByteArray(Charsets.UTF_8)).await()

    View Slide

  19. Flowing resources
    suspend fun uploadData(path: String): Unit =
    BlobStorage(path).use { storage ->
    Path(path)
    .readAll()
    .collect { line ->
    storage.insertIntoBlobStorage(line)
    }
    }

    View Slide

  20. Flowing resources
    fun blobStorage(path: String): Flow = flow {
    BlobStorage(path).use { emit(it) }
    }

    View Slide

  21. Flowing resources
    suspend fun uploadData2(path: String): Flow =
    blobStorage().flatMapConcat { blobStorage ->
    Path("data.txt")
    .readAll()
    .map { line ->
    blobStorage.insertIntoBlobStorage(line)
    }
    }

    View Slide

  22. Parallel Processing
    Read line Upload blob
    Coroutine

    View Slide

  23. Parallel Processing
    Read line Coroutine Upload blob Coroutine
    Channel(3)

    View Slide

  24. Parallel Processing
    suspend fun uploadData3(path: String): Flow =
    blobStorage().flatMapConcat { blobStorage ->
    Path("data.txt")
    .readAll()
    .parMapUnordered(concurrency = 3) { line ->
    blobStorage.insertIntoBlobStorage(line)
    }
    }

    View Slide

  25. Parallel Processing
    @FlowPreview
    public inline fun Flow.parMapUnordered(
    concurrency: Int = DEFAULT_CONCURRENCY,
    crossinline transform: suspend (a: A) >> B
    ): Flow {
    val flowOfFlows: Flow> = map { o ->
    flow {
    emit(transform(o))
    }
    }
    return parallelFlows.flattenMerge(concurrency)
    }

    View Slide

  26. Parallel Processing
    suspend fun uploadData3(path: String): Flow =
    blobStorage().flatMapConcat { blobStorage ->
    Path("data.txt")
    .readAll()
    .parMap(concurrency = 3) { line ->
    blobStorage.insertIntoBlobStorage(line)
    }
    }

    View Slide

  27. Compose-able streaming
    Automatic back-pressure
    Conclusion
    Suspend everywhere

    View Slide

  28. Thanks!

    View Slide