Upgrade to Pro — share decks privately, control downloads, hide ads and more …

Functional Flowing

Functional Flowing

KotlinX’s Flow data type is extremely powerful, and offers an API that is quite familiar for functional programmers.
This talk will showcase how we can leverage KotlinX Flow to describe powerful programs, and build pipelines to transform and manipulate data in an efficient streaming way.

Simon Vergauwen

November 26, 2022
Tweet

More Decks by Simon Vergauwen

Other Decks in Programming

Transcript

  1. Functional Flowing
    47deg.com

    View full-size slide

  2. KotlinX Flow
    Pull based
    Reasoning about Flow
    Streaming Resources
    Parallel Processing

    View full-size slide

  3. Pull Based Streaming
    Upstream
    Pipeline Downstream

    View full-size slide

  4. Pull Based Streaming
    Upstream
    Pipeline Downstream

    View full-size slide

  5. Pull Based Streaming
    public fun interface FlowCollector {
    public suspend fun emit(value: T)
    }

    View full-size slide

  6. Creating Flows
    class Circle(val count: Int)
    suspend fun FlowCollector.produceCircles(): Unit {
    var counter: Int = 0
    while(true) {
    val circle = Circle(counter>+)
    println("Creating $circle")
    emit(circle)
    }
    }

    View full-size slide

  7. Creating Flows
    class Circle(val count: Int)
    suspend fun FlowCollector.produceCircles(): Unit {
    var counter: Int = 0
    while(true) {
    val circle = Circle(counter>+)
    println("Creating $circle")
    emit(circle)
    }
    }
    val circles: Flow =
    flow { produceCircles() }

    View full-size slide

  8. Creating Flows
    class Circle(val count: Int)
    val circles: Flow = flow {
    var counter: Int = 0
    while(true) {
    val circle = Circle(counter>+)
    println("Creating $circle")
    emit(circle)
    }
    }

    View full-size slide

  9. Pipeline
    class Square private constructor(val circle: Circle) {
    companion object {
    suspend fun Circle.toSquare(): Square {
    print("Creating Square($count)")
    delay(1.seconds)
    return Square(this).also { println("Created $this") }
    }
    }
    }

    View full-size slide

  10. Putting it all together
    fun main() = runBlocking {
    circles
    .take(3)
    .collect { circle ->
    circle.toSquare()
    }
    }
    Creating Circle@63e31ee
    Creating Square(0)
    Created Square@6adca536
    Creating Circle@23223dd8

    View full-size slide

  11. Reasoning about Flows
    suspend fun FlowCollector.produceCircles(): Unit {
    var counter: Int = 0
    while(true) {
    val circle = Circle(counter>+)
    println("Creating $circle")
    emit(circle)
    }
    }

    View full-size slide

  12. Reasoning about Flows
    class Square private constructor(val circle: Circle) {
    companion object {
    suspend fun Circle.toSquare(): Square {
    print("Creating Square($count)")
    delay(1.seconds)
    return Square(this).also { println("Created $this") }
    }
    }
    }

    View full-size slide

  13. Reasoning about Flows
    while(true)
    0 1 2
    Coroutine

    View full-size slide

  14. Flowing resources
    fun Path.readAll(): Flow

    View full-size slide

  15. Flowing resources
    fun Path.readAll(): Flow = flow { >/ this: FlowCollector
    useLines { lines ->
    lines.forEach { line -> emit(line) }
    }
    }

    View full-size slide

  16. Flowing resources
    suspend fun insertIntoBlobStorage(line: String): Unit

    View full-size slide

  17. Flowing resources
    /** Java SDK */
    class BlobStorage(val blobName: String) : AutoCloseable {
    fun uploadBlob(data: ByteArray): CompletableFuture
    override fun close() = Unit
    }
    suspend fun BlobStorage.insertIntoBlobStorage(line: String): UploadResult =
    uploadBlob(line.toByteArray(Charsets.UTF_8)).await()

    View full-size slide

  18. Flowing resources
    suspend fun uploadData(path: String): Unit =
    BlobStorage(path).use { storage ->
    Path(path)
    .readAll()
    .collect { line ->
    storage.insertIntoBlobStorage(line)
    }
    }

    View full-size slide

  19. Flowing resources
    fun blobStorage(path: String): Flow = flow {
    BlobStorage(path).use { emit(it) }
    }

    View full-size slide

  20. Flowing resources
    suspend fun uploadData2(path: String): Flow =
    blobStorage().flatMapConcat { blobStorage ->
    Path("data.txt")
    .readAll()
    .map { line ->
    blobStorage.insertIntoBlobStorage(line)
    }
    }

    View full-size slide

  21. Parallel Processing
    Read line Upload blob
    Coroutine

    View full-size slide

  22. Parallel Processing
    Read line Coroutine Upload blob Coroutine
    Channel(3)

    View full-size slide

  23. Parallel Processing
    suspend fun uploadData3(path: String): Flow =
    blobStorage().flatMapConcat { blobStorage ->
    Path("data.txt")
    .readAll()
    .parMapUnordered(concurrency = 3) { line ->
    blobStorage.insertIntoBlobStorage(line)
    }
    }

    View full-size slide

  24. Parallel Processing
    @FlowPreview
    public inline fun Flow.parMapUnordered(
    concurrency: Int = DEFAULT_CONCURRENCY,
    crossinline transform: suspend (a: A) >> B
    ): Flow {
    val flowOfFlows: Flow> = map { o ->
    flow {
    emit(transform(o))
    }
    }
    return parallelFlows.flattenMerge(concurrency)
    }

    View full-size slide

  25. Parallel Processing
    suspend fun uploadData3(path: String): Flow =
    blobStorage().flatMapConcat { blobStorage ->
    Path("data.txt")
    .readAll()
    .parMap(concurrency = 3) { line ->
    blobStorage.insertIntoBlobStorage(line)
    }
    }

    View full-size slide

  26. Compose-able streaming
    Automatic back-pressure
    Conclusion
    Suspend everywhere

    View full-size slide