Upgrade to Pro — share decks privately, control downloads, hide ads and more …

Graphics Programming with Kotlin

237be48129b762b31847d6167597366d?s=47 Romain Guy
October 04, 2018

Graphics Programming with Kotlin

Graphics programming is a field dominated by traditional languages like C and C++, or specialized languages like GLSL and HLSL. In this talk, we'll go over how Kotlin allows you to take advantage of a managed language while preserving the conciseness and expressiveness of low-level languages like C++ when writing math and graphics oriented code.

237be48129b762b31847d6167597366d?s=128

Romain Guy

October 04, 2018
Tweet

Transcript

  1. GRAPHICS KOTLIN
 for @romainguy

  2. Android KTX

  3. Dealing with legacy

  4. // Pack a color int val color = ((a and

    0xff) shl 24) or ((r and 0xff) shl 16) or ((g and 0xff) shl 8) or ((b and 0xff) ) // Unpack a color int val a = (color shr 24) and 0xff val r = (color shr 16) and 0xff val g = (color shr 8) and 0xff val b = (color ) and 0xff
  5. inline val @receiver:ColorInt Int.alpha get() = (this shr 24) and

    0xff inline val @receiver:ColorInt Int.red get() = (this shr 16) and 0xff inline val @receiver:ColorInt Int.green get() = (this shr 8) and 0xff inline val @receiver:ColorInt Int.blue get() = (this ) and 0xff
  6. val a = color.alpha val r = color.red val g

    = color.green val b = color.blue
  7. inline operator fun @receiver:ColorInt Int.component1() = this.alpha inline operator fun

    @receiver:ColorInt Int.component2() = this.red inline operator fun @receiver:ColorInt Int.component3() = this.green inline operator fun @receiver:ColorInt Int.component4() = this.blue
  8. val (a, r, g, b) = color

  9. Destructure everything

  10. // Points val (x, y) = PointF(1.0f, 2.0f) + PointF(3.0f,

    4.0f) // Rectangles val (l, t, r, b) = Rect(0, 0, 4, 4) and Rect(2, 2, 6, 6)
  11. // Points val (x, y) = PointF(1.0f, 2.0f) + PointF(3.0f,

    4.0f) // Rectangles val (l, t, r, b) = Rect(0, 0, 4, 4) and Rect(2, 2, 6, 6) // Matrix val (right, up, forward, eye) = viewMatrix
  12. // Points
 val (x, y) = PointF(1.0f, 2.0f) + PointF(3.0f,

    4.0f)
 
 // Rectangles
 val (l, t, r, b) = Rect(0, 0, 4, 4) and Rect(2, 2, 6, 6)
 // Matrix
 val (right, up, forward, eye) = viewMatrix val (x, y, z) = eye
  13. Implement operators

  14. Kotlin Arithmetic operators Set operators plus + ∪ (union) minus

    – – (difference) times × div / or ∪ (union) and ∩ (intersection) xor ⊖ (symmetric difference) not U \ (complement)
  15. // Union val r = RectF(0.0f, 0.0f, 4.0f, 4.0f) or

    RectF(2.0f, 2.0f, 6.0f, 6.0f) // Symmetric difference val r = Rect(0, 0, 4, 4) xor Rect(2, 2, 6, 6) // Difference val path = circle - square // Offset val (l, t, r, b) = Rect(0, 0, 2, 2) + 2 val (l, t, r, b) = Rect(0, 0, 2, 2) - Point(1, 2)
  16. inline operator fun Rect.contains(p: PointF) = contains(p.x, p.y)

  17. if (PointF(x, y) in path.bounds) { // Hit detected }

  18. None
  19. inline operator fun Bitmap.get(x: Int, y: Int) = getPixel(x, y)

  20. inline operator fun Bitmap.get(x: Int, y: Int) = getPixel(x, y)

  21. inline operator fun Bitmap.get(x: Int, y: Int) = getPixel(x, y)

    inline operator fun Bitmap.set(x: Int, y: Int, @ColorInt color: Int) =
 setPixel(x, y, color)
  22. val b = getBitmap() val a = b[1, 1].alpha if

    (a < 255) { b[1, 1] = 0xff_00_00_00 }
  23. inline operator fun Bitmap.get(x: IntRange, y: IntRange): IntArray

  24. bitmap[16..32, 16..32].forEach { val (_, r, g, b) = it

    }
  25. All together

  26. None
  27. if (bounds.contains(hit.x, hit.y)) {

  28. if (bounds.contains(hit.x, hit.y)) { val pixel = Bitmap.createBitmap(width, height, ARGB_8888).let

    {
  29. if (bounds.contains(hit.x, hit.y)) { val pixel = Bitmap.createBitmap(width, height, ARGB_8888).let

    { with(Canvas(it)) { save() scale(2.0f, 2.0f) val path = Path().apply { op(path1, path2, DIFFERENCE) } drawPath(path, paint) restore() }
  30. if (bounds.contains(hit.x, hit.y)) { val pixel = Bitmap.createBitmap(width, height, ARGB_8888).let

    { with(Canvas(it)) { save() scale(2.0f, 2.0f) val path = Path().apply { op(path1, path2, DIFFERENCE) } drawPath(path, paint) restore() } it.getPixel(hit.x, hit.y) }
  31. if (bounds.contains(hit.x, hit.y)) { val pixel = Bitmap.createBitmap(width, height, ARGB_8888).let

    { with(Canvas(it)) {
 save()
 scale(2.0f, 2.0f)
 val path = Path().apply { op(path1, path2, DIFFERENCE) }
 drawPath(path, paint)
 restore()
 } it.getPixel(hit.x, hit.y)
 } val r = Color.red(pixel)
 val g = Color.green(pixel)
 val b = Color.blue(pixel)
 }
  32. if (hit in bounds) { val (_, r, g, b)

    = createBitmap(width, height).applyCanvas { withScale(2.0f, 2.0f) { drawPath(path1 - path2, paint) } }[hit.x, hit.y] }
  33. None
  34. vec3 color = vec3(0.65, 0.85, 1.0) + direction.y * 0.72;

    // (distance, material) vec2 hit = traceRay(origin, direction); float distance = hit.x; float material = hit.y; // We've hit something in the scene if (material > 0.0) { vec3 lightDir = vec3(0.6, 0.7, -0.7); vec3 position = origin + distance * direction; vec3 v = normalize(-direction); vec3 n = normal(position); vec3 l = normalize(lightDir); vec3 h = normalize(v + l); vec3 r = normalize(reflect(direction, n)); float NoV = abs(dot(n, v)) + 1e-5; float NoL = saturate(dot(n, l)); float NoH = saturate(dot(n, h)); float LoH = saturate(dot(l, h)); // ... }
  35. var color = Float3(0.65f, 0.85f, 1.0f) + direction.y * 0.72f

    // (distance, material) val hit = traceRay(origin, direction) val distance = hit.x val material = hit.y // We've hit something in the scene if (material > 0.0f) { val lightDir = Float3(0.6f, 0.7f, -0.7f) val position = origin + distance * direction val v = normalize(-direction) val n = normal(position) val l = normalize(lightDir) val h = normalize(v + l) val r = normalize(reflect(direction, n)) val NoV = abs(dot(n, v)) + 1e-5f val NoL = saturate(dot(n, l)) val NoH = saturate(dot(n, h)) val LoH = saturate(dot(l, h)) // ... }
  36. Data first Math is functional

  37. data class Float3(var x: Float = 0.0f, var y: Float

    = 0.0f, var z: Float = 0.0f) + operators
  38. inline fun abs(v: Float3) = Float3(abs(v.x), abs(v.y), abs(v.z)) inline fun

    length(v: Float3) = sqrt(v.x * v.x + v.y * v.y + v.z * v.z) inline fun length2(v: Float3) = v.x * v.x + v.y * v.y + v.z * v.z inline fun distance(a: Float3, b: Float3) = length(a - b) inline fun dot(a: Float3, b: Float3) = a.x * b.x + a.y * b.y + a.z * b.z
  39. fun lookAt( eye: Float3, target: Float3, up: Float3 = Float3(z

    = 1.0f) ): Mat4 { val f = normalize(target - eye)
 val r = normalize(f x up)
 val u = normalize(r x f)
 return Mat4(r, u, f, eye)
 }
  40. fun lookAt( eye: Float3, target: Float3, up: Float3 = Float3(z

    = 1.0f) ): Mat4 { val f = normalize(target - eye)
 val r = normalize(f x up)
 val u = normalize(r x f)
 return Mat4(r, u, f, eye)
 } val h = normalize(v + l) val r = normalize(reflect(direction, n)) val NoV = abs(dot(n, v)) + 1e-5f val NoL = saturate(dot(n, l))
  41. Aliasing & swizzling

  42. vec4 v = vec4(…) // Use XYZ for coordinates vec3

    position = v.xyz // Use RGB for color data vec3 color = v.rgb // Swizzling vec3 reverseColor = v.bgr
 vec3 grayColor = v.ggg vec4 twoPositions = v.xyxy
  43. // In Float4 inline var xy: Float2 get() = Float2(x,

    y) set(value) { x = value.x y = value.y } inline var rg: Float2 get() = Float2(x, y) set(value) { x = value.x y = value.y }
  44. None
  45. enum class VectorComponent { X, Y, Z, W, R, G,

    B, A, S, T, P, Q }
  46. enum class VectorComponent { X, Y, Z, W, R, G,

    B, A, S, T, P, Q } operator fun get(index: VectorComponent) = when (index) { VectorComponent.X, VectorComponent.R, VectorComponent.S -> x VectorComponent.Y, VectorComponent.G, VectorComponent.T -> y VectorComponent.Z, VectorComponent.B, VectorComponent.P -> z else -> throw IllegalArgumentException(“Unknown index”) }
  47. enum class VectorComponent {
 X, Y, Z, W,
 R, G,

    B, A,
 S, T, P, Q
 } 
 operator fun get(index: VectorComponent) = when (index) {
 VectorComponent.X, VectorComponent.R, VectorComponent.S -> x
 VectorComponent.Y, VectorComponent.G, VectorComponent.T -> y
 VectorComponent.Z, VectorComponent.B, VectorComponent.P -> z
 else -> throw IllegalArgumentException(“Unknown index”)
 } 
 operator fun get(
 index1: VectorComponent,
 index2: VectorComponent,
 index3: VectorComponent
 ): Float3 {
 return Float3(get(index1), get(index2), get(index3))
 }
  48. val v = Float4(…)
 val position = v.xyz
 val color

    = v.rgb
 val reverseColor = v[B, G, R]
 val grayColor = v[G, G, G] val twoPositions = v[X, Y, X, Y]
  49. Row major Column major

  50. None
  51. right

  52. right up

  53. right up forward

  54. right up forward translation

  55. val transform: Mat4 // 3rd column, first element val x3

    = transform[2, 0]
 val x3 = transform.z.x // Math notation // Row-major, 1-based val x3 = transform(1, 3)
  56. Where there is one there are many

  57. if (color.r > 0.0f && color.g > 0.0f && color.b

    > 0.0f) { // ... }
  58. if (all(color gt Float3(0.0f))) { // ... }

  59. if (all(color gt Float3(0.0f))) { // ... } // any()

    for || if (any(color lt black)) { // ... }
  60. https://github.com/google/filament

  61. https://github.com/google/filament

  62. Local extension functions

  63. None
  64. private fun createMesh() { data class Vertex(val x: Float, val

    y: Float, val z: Float, val n: Float3)
  65. private fun createMesh() { data class Vertex(val x: Float, val

    y: Float, val z: Float, val n: Float3) fun ByteBuffer.put(v: Vertex): ByteBuffer { putFloat(v.x) putFloat(v.y) putFloat(v.z) v.n.forEach { putFloat(it) } return this }
  66. private fun createMesh() { data class Vertex(val x: Float, val

    y: Float, val z: Float, val n: Float3) fun ByteBuffer.put(v: Vertex): ByteBuffer { putFloat(v.x) putFloat(v.y) putFloat(v.z) v.n.forEach { putFloat(it) } return this } val vertexData = ByteBuffer.allocate(vertexCount * vertexSize) .order(ByteOrder.nativeOrder()) // Face -Z .put(Vertex(-1.0f, -1.0f, -1.0f, Float3(0.0f, 0.0f, -1.0f))) .put(Vertex(-1.0f, 1.0f, -1.0f, Float3(0.0f, 0.0f, -1.0f))) // ...
  67. private fun createMesh() {
 data class Vertex(val x: Float, val

    y: Float, val z: Float, val n: Float3)
 fun ByteBuffer.put(v: Vertex): ByteBuffer {
 putFloat(v.x)
 putFloat(v.y)
 putFloat(v.z)
 v.n.forEach { putFloat(it) }
 return this
 }
 val vertexData = ByteBuffer.allocate(vertexCount * vertexSize)
 .order(ByteOrder.nativeOrder())
 // Face -Z
 .put(Vertex(-1.0f, -1.0f, -1.0f, Float3(0.0f, 0.0f, -1.0f)))
 .put(Vertex(-1.0f, 1.0f, -1.0f, Float3(0.0f, 0.0f, -1.0f)))
 // ...
 // Build mesh with vertexData
 }
  68. 1.3

  69. Unsigned Integers

  70. UInt

  71. UInt represents natural numbers ℕ

  72. @ColorInt operator fun Bitmap.get(x: Int, y: Int): Int

  73. @ColorInt operator fun Bitmap.get(x: Int, y: Int): Int // Can

    x and y be < 0? // Will it throw, clamp or wrap around? myBitmap[-1, -1]
  74. @ColorInt operator fun Bitmap.get(x: UInt, y: UInt): Int

  75. @ColorInt operator fun Bitmap.get(x: UInt, y: UInt): Int // No

    more ambiguity myBitmap[-1, -1]
  76. @ColorInt operator fun Bitmap.get(x: UInt, y: UInt): Int // No

    more ambiguity myBitmap[-1, -1] Conversion of signed constants to unsigned ones is prohibited
  77. Kotlin Signed Byte Short Int Long Unsigned Char UInt ULong

  78. JVM/ART Signed byte short int long Unsigned char

  79. val a: UInt = //... val b: UInt = //...

    val c = a + b
  80. None
  81. 1: iload 4 // load a

  82. 1: iload 4 // load a 2: iload 5 //

    load b
  83. 1: iload 4 // load a 2: iload 5 //

    load b 3: iadd
  84. 1: iload 4 // load a 2: iload 5 //

    load b 3: iadd 4: invokestatic #13 // kotlin/UInt."constructor-impl":(I)I
  85. 1: iload 4 // load a 2: iload 5 //

    load b 3: iadd 4: invokestatic #13 // kotlin/UInt."constructor-impl":(I)I 5: istore_3
  86. 4: invokestatic #13 // kotlin/UInt."constructor-impl":(I)I

  87. public static int constructor-impl(int); Code: 0: iload_0 1: ireturn

  88. 1: iload 4 // load a 2: iload 5 //

    load b 3: isub 4: invokestatic #13 // kotlin/UInt."constructor-impl":(I)I 5: istore_3
  89. 1: iload 4 // load a 2: iload 5 //

    load b 3: imul 4: invokestatic #13 // kotlin/UInt."constructor-impl":(I)I 5: istore_3
  90. 1: iload 4 // load a 2: iload 5 //

    load b 3: invokestatic #91 // kotlin/UnsignedKt."uintDivide-J1ME1BU":(II)I 4: invokestatic #13 // kotlin/UInt."constructor-impl":(I)I 5: istore_3
  91. UInt

  92. UInt

  93. UInt —— — — — — — — — —

  94. Inline Classes

  95. Inline classes are a zero-cost* abstraction

  96. Inline classes are a zero-cost* abstraction * When used right

  97. None
  98. • Wraps a single value

  99. • Wraps a single value • The underlying value is

    read-only
  100. • Wraps a single value • The underlying value is

    read-only • Can only implement interfaces
  101. • Wraps a single value • The underlying value is

    read-only • Can only implement interfaces • equals/hashcode/toString for free
  102. • Wraps a single value • The underlying value is

    read-only • Can only implement interfaces • equals/hashcode/toString for free • Cannot be used as vararg* * Unless you are UInt/ULong
  103. inline class Color(private val c: Int) { val a: Int

    get() = (c shr 24) and 0xff val r: Int get() = (c shr 16) and 0xff val g: Int get() = (c shr 8) and 0xff val b: Int get() = (c ) and 0xff }
  104. fun printColor(color: Color) { println(""" red = ${color.r / 255f}

    green = ${color.g / 255f} blue = ${color.b / 255f} alpha = ${color.a / 255f} """.trimIndent()) } val color = Color(0x7f_10_20_30) printColor(color)
  105. val color = Color(0x7f_10_20_30)

  106. 0: ldc #163 // int 2131763248 val color = Color(0x7f_10_20_30)

  107. 0: ldc #163 // int 2131763248 1: invokestatic #164 //

    Method Color."constructor-impl":(I)I val color = Color(0x7f_10_20_30)
  108. 0: ldc #163 // int 2131763248 1: invokestatic #164 //

    Method Color."constructor-impl":(I)I 2: istore 7 val color = Color(0x7f_10_20_30)
  109. 0: ldc #163 // int 2131763248 1: invokestatic #164 //

    Method Color."constructor-impl":(I)I 2: istore 7 3: iload 7 val color = Color(0x7f_10_20_30)
  110. 0: ldc #163 // int 2131763248 1: invokestatic #164 //

    Method Color."constructor-impl":(I)I 2: istore 7 3: iload 7 4: invokestatic #166 // Method "printColor-M0a6fYQ":(I)V val color = Color(0x7f_10_20_30)
  111. fun printColor(color: Color)

  112. public static final void printColor-M0a6fYQ(int); 0: iload_0 fun printColor(color: Color)

  113. public static final void printColor-M0a6fYQ(int);
 0: iload_0 1: invokestatic #112

    // Method Color.”getR-impl":(I)I fun printColor(color: Color)
  114. public static final void printColor-M0a6fYQ(int);
 0: iload_0 1: invokestatic #112

    // Method Color.”getR-impl":(I)I fun printColor(color: Color) public static final int getR-impl(int); 0: iload_0 1: bipush 16 2: ishr 3: sipush 255 4: iand 5: ireturn
  115. Be careful with auto-boxing

  116. None
  117. • Nullable types

  118. • Nullable types • Inside collections, arrays, etc.

  119. • Nullable types • Inside collections, arrays, etc. • When

    Object or Any is expected
  120. val a = Color(0x7f_10_20_30) val b = Color(0x7f_30_20_10)

  121. val a = Color(0x7f_10_20_30) val b = Color(0x7f_30_20_10) println(a ==

    b)
  122. val a = Color(0x7f_10_20_30)
 val b = Color(0x7f_30_20_10)
 println(a ==

    b) println(a.equals(b))
  123. a == b

  124. a == b

  125. 1: iload_2 a == b

  126. 1: iload_2 2: invokestatic #152 // Method Color."box-impl":(I)LColor; a ==

    b
  127. 1: iload_2 2: invokestatic #152 // Method Color."box-impl":(I)LColor; 3: iload_1

    a == b
  128. 1: iload_2 2: invokestatic #152 // Method Color."box-impl":(I)LColor; 3: iload_1

    4: invokestatic #152 // Method Color."box-impl":(I)LColor; a == b
  129. 1: iload_2 2: invokestatic #152 // Method Color."box-impl":(I)LColor; 3: iload_1

    4: invokestatic #152 // Method Color."box-impl":(I)LColor; 5: invokestatic #163 // Method kotlin/jvm/internal/Intrinsics.
 //.areEqual:(Ljava/lang/Object;Ljava/lang/Object;)Z a == b
  130. a.equals(b)

  131. a.equals(b)

  132. 1: iload_2 a.equals(b)

  133. 1: iload_2 2: iload_1 a.equals(b)

  134. 1: iload_2 2: iload_1 3: invokestatic #152 // Method Color."box-impl":(I)LColor;

    a.equals(b)
  135. 1: iload_2 2: iload_1 3: invokestatic #152 // Method Color."box-impl":(I)LColor;

    4: invokestatic #156 // Method Color."equals-impl":(ILjava/lang/Object;)Z a.equals(b)
  136. inline class Color(private val value: Int) { val a: Int

    get() = (value shr 24) and 0xff val r: Int get() = (value shr 16) and 0xff val g: Int get() = (value shr 8) and 0xff val b: Int get() = (value ) and 0xff }
  137. inline class Color(val value: Int) { val a: Int get()

    = (value shr 24) and 0xff val r: Int get() = (value shr 16) and 0xff val g: Int get() = (value shr 8) and 0xff val b: Int get() = (value ) and 0xff }
  138. val a = Color(0x7f_10_20_30) val b = Color(0x7f_30_20_10)

  139. val a = Color(0x7f_10_20_30)
 val b = Color(0x7f_30_20_10)
 println(a.value ==

    b.value)
  140. var s = “” // Boxing!
 s += a 


    // No boxing
 s += a.toString()
  141. Coroutines

  142. Render Culling 0 N Sorting Shadow Color GC Lights 0

    N 0 N 0 N 0 N 0 N Driver N
  143. private fun startRender(image: BufferedImage, viewer: ImageViewer) { GlobalScope.launch { while

    (true) { runBlocking { renderTiles(image, viewer) } } } }
  144. private val NumCpu = Runtime.getRuntime().availableProcessors() private val TilesDispatcher = newFixedThreadPoolContext(NumCpu

    * 3, "Renderer") private suspend fun renderTiles(image: BufferedImage, viewer: ImageViewer) { // ... coroutineScope { repeat(NumCpu) { i -> repeat(NumCpu) { j -> // ... launch(TilesDispatcher) { val pixels = renderTile(x, y, w, h, resolution, time) viewer.pushUpdate(x, height - h - y, w, h, pixels) } } } } }
  145. Kotlin scripting

  146. My wishlist

  147. User-defined literals

  148. inline class Half(private val s: Short) { fun toFloat(): Float

    = // ... }
  149. inline class Half(private val s: Short) { fun toFloat(): Float

    = // ... } fun Float.toHalf(): Half = // ...
  150. inline class Half(private val s: Short) {
 fun toFloat(): Float

    = // ...
 } 
 fun Float.toHalf(): Half = // ... 
 suffix operator fun Float._h() = this.toHalf()
  151. // pi is of type Half val pi = 3.1415_h

  152. // 2.0_s returns 2,000 ms suffix operator fun Float._ns() =

    this / 1_000f suffix operator fun Float._ms() = this suffix operator fun Float._s() = this * 1_000f suffix operator fun Float._m() = this * 60_000f suffix operator fun Float._h() = this * 3_600_000f
  153. Short-form constructors

  154. data class Vec3(x: Float, y: Float, z: Float) fun lookAt(pos:

    Vec3) = // ... lookAt([1f, 2f, 3f])
  155. data class Vertex(pos: Float3, normal: Float3) addVertex([[1f, 2f, 3f], [0f,

    0f, 1f]])
  156. Linear allocations

  157. Kotlin/GPU

  158. Where to find some code kotlin-math
 https://github.com/romainguy/kotlin-math filament
 https://github.com/google/filament https://google.github.io/filament/Filament.md.html

    kotlin-shadertoy https://goo.gl/TFKUwP
  159. Talks this week Mathematical modeling with Kotlin
 Today @13:00 Build

    a game using libGDX and Kotlin
 Today @15:15 Porting D3.js to Kotlin Multiplatform
 Tomorrow @13:00
  160. Questions?