scodec for Scala 3 Michael Pilquist // @mpilquist

ascii.decode( ) == Successful(DecodeResult(scodec,BitVector(empty)))

AGENDA 3 Macros Numeric Literals Tuples Revisited Match Types Mirrors Derivation

LITERAL INTERPOLATORS 4 val x = hex"00112233445566778899aabbccddeeff00112233" // x: ByteVector = ByteVector(20 bytes, 0x00112233445566778899aabbccddeeff00112233) val y = bin"1000101011011000100101" // y: BitVector = BitVector(22 bits, 0x8ad894) val z = hex"Cow!" // foo.scala: hexadecimal string literal may only contain characters [0-9a-fA-f] // val z = hex"Cow!" // ^ // Compilation Failed

LITERAL INTERPOLATORS 5 package scodec.bits import scala.quoted._ inline def (inline ctx: StringContext).hex (inline args: Any*): ByteVector = ${validateHex('ctx, 'args)}

LITERAL INTERPOLATORS 6 package scodec.bits._ import scala.quoted._ import scala.quoted.matching._ inline def (ctx: StringContext).hex (inline args: Any*): ByteVector = ${validateHex('ctx, 'args)} def validateHex( strCtxExpr: Expr[StringContext], argsExpr: Expr[Seq[Any]] )(using qctx: QuoteContext): Expr[ByteVector] = strCtxExpr.unlift match { case Some(sc) => validateHexImpl(sc.parts, argsExpr) case None => qctx.error("StringContext args must be statically known") }

LITERAL INTERPOLATORS 7 private def validateHexImpl( parts: Seq[String], argsExpr: Expr[Seq[Any]] )(using qctx: QuoteContext): Expr[ByteVector] = { if (parts.size == 1) { val literal = parts.head ByteVector.fromHex(literal) match { case Some(_) => '{ByteVector.fromValidHex(${Expr(literal)})} case None => qctx.error( "hexadecimal string literal may only contain characters [0-9a-fA-f]", ) ??? } } else { qctx.error("interpolation not supported", argsExpr) ??? } }

NUMERIC LITERALS 8 val x: ByteVector = 0x00112233445566778899aabbccddeeff00112233 // x: ByteVector = ByteVector(20 bytes, 0x00112233445566778899aabbccddeeff00112233) val y: BitVector = 0xdeadbeef // y: BitVector = BitVector(32 bits, 0xdeadbeef)

NUMERIC LITERALS 9 import scala.util.FromDigits import java.math.BigInteger object ByteVector { given FromDigits.WithRadix[ByteVector] { def fromDigits(digits: String, radix: Int): ByteVector = radix match { case 16 => ByteVector.fromValidHex(digits) case _ => ByteVector.fromValidHex(new BigInteger(digits, radix).toString(16)) } } }

PRODUCT CODECS 1 0 val a = int8 ~ bool ~ cstring // a: Codec[((Int, Boolean), String)] = … val b = int8 ~~ bool ~~ cstring // b: Codec[(Int, Boolean, String)] = … val c = int8 :: bool :: cstring // val c: Codec[Int :: Boolean :: String :: HNil] = … Scala 2 Lots of ways to do roughly the same thing Combinators have to pick which to support (flatZip vs flatPrepend) In practice, HList variant is the most common ☹

PRODUCT CODECS 1 1 val a = int8 :: bool :: cstring // val a: Codec[(Int, Boolean, String)] = … val b = int64 :: a // val b: Codec[(Long, Int, Boolean, String)] = … Scala 3 Unify all of these APIs in to a single one that creates tuples of expected arity

TUPLES REVISITED 1 2 sealed trait Tuple extends Any { inline def *: [H, This >: this.type <: Tuple] (x: H): H *: This = foo } type EmptyTuple = EmptyTuple.type object EmptyTuple extends Tuple sealed trait NonEmptyTuple extends Tuple @showAsInfix sealed abstract class *:[+H, +T <: Tuple] extends NonEmptyTuple object *: { def unapply[H, T <: Tuple](x: H *: T): (H, T) = (x.head, x.tail) }

TUPLES REVISITED 1 3 val a = 1 *: (true, 3, "Hi")
 // a: (Int, Boolean, Int, String) = (1,true,3,Hi)
 val b = (1, true) ++ (3, "Hi")
 // b: (Int, Boolean, Int, String) = (1,true,3,Hi)
 val h *: t = a
 // h: Int = 1
 // t: (Boolean, Int, String) = (true,3,Hi)


CODEC CONS 1 4 How can we implement :: for Codec? int8 :: (bool :: cstring) Codec[(Boolean, String)] Codec[Int] Codec[(Int, Boolean, String)] We need two operations: - for all A, B <: Tuple: (Codec[A], Codec[B]) => Codec[A *: B] - for all A, B: (Codec[A], Codec[B]) => Codec[(A, B)]

CODEC CONS 1 5 object Codec { extension tupleOpsRightAssociative on [A, B <: Tuple](b: Codec[B]) { def ::(a: Codec[A]): Codec[A *: B] = ??? } extension on [A, B](b: Codec[B]) { def ::(a: Codec[A]): Codec[(A, B)] = ??? } }

CODEC CONS 1 6 object Codec { extension tupleOpsRightAssociative on [A, B <: Tuple](b: Codec[B]) { def ::(a: Codec[A]): Codec[A *: B] = new Codec[A *: B] { def sizeBound = a.sizeBound + b.sizeBound def encode(ab: A *: B) = encodeBoth(a, b)(ab.head, ab.tail) def decode(bv: BitVector) = decodeBoth(a, b)(bv).map(_.map(_ *: _)) override def toString = s"$a :: $b" } } extension on [A, B](b: Codec[B]) { def ::(a: Codec[A]): Codec[(A, B)] = new Codec[(A, B)] { def sizeBound = a.sizeBound + b.sizeBound def encode(ab: (A, B)) = encodeBoth(a, b)(ab._1, ab._2) def decode(bv: BitVector) = decodeBoth(a, b)(bv) override def toString = s"$a :: $b" } }

CODEC CONS 1 7 object Codec { def [A, B <: Tuple] (a: Codec[A]) :: (b: Codec[B]): Codec[A *: B] = … def [A, B] (a: Codec[A]) :: (b: Codec[B])(using DummyImplicit): Codec[(A, B)] = … } Easier to read - no need to mentally reverse :: Requires DummyImplicit to disambiguate erased signature Requires explicit import of extension method at call site (#8275) ☹ Can we use simple extension methods instead of collective extensions?

UNITS 1 8 val a = ignore(2) :: int(3) :: bool :: ignore(2) :: cstring // val a: Codec[(Unit, Int, Boolean, Unit, String)] = … These unit values are annoying Have to manually insert them when encoding and remove them when decoding ☹

UNITS 1 9 val a = ignore(2) :: int(3) :: bool :: ignore(2) :: cstring // val a: Codec[(Unit, Int, Boolean, Unit, String)] = … val b = a.dropUnits // val b: Codec[(Int, Boolean, String)] = … What’s the signature of dropUnits? How do we write "the tuple you get when you remove all units from A"?

UNITS 2 0 object Codec { extension tupleOpsNoParams on [A <: Tuple](codecA: Codec[A]) { inline def dropUnits: Codec[DropUnits[A]] = codecA.xmap(a => DropUnits.drop(a), b => DropUnits.insert(b)) } } type DropUnits[A <: Tuple] <: Tuple = A match { case hd *: tl => hd match { case Unit => DropUnits[tl] case _ => hd *: DropUnits[tl] } case EmptyTuple => EmptyTuple }

MATCH TYPES 2 1 type DropUnits[A <: Tuple] <: Tuple = A match { case hd *: tl => hd match { case Unit => DropUnits[tl] case _ => hd *: DropUnits[tl] } case EmptyTuple => EmptyTuple } DropUnits is a match type - Defined by pattern matching on types - Supports recursion - Defined inductively with: - Inductive case for non-empty tuple - Base case for empty tuple

MATCH TYPES - TERM LEVEL 2 2 inline def drop[A <: Tuple](a: A): DropUnits[A] = { inline erasedValue[A] match { case _: (Unit *: tl) => drop[tl](a.asInstanceOf[Unit *: tl].tail) case _: (hd *: tl) => val at = a.asInstanceOf[hd *: tl] at.head *: drop[tl](at.tail) case EmptyTuple => EmptyTuple } }.asInstanceOf[DropUnits[A]]

MATCH TYPES - TERM LEVEL 2 3 inline def insert[A <: Tuple](t: DropUnits[A]): A = { inline erasedValue[A] match { case _: (Unit *: tl) => (()) *: (insert[tl](t.asInstanceOf[DropUnits[tl]])) case _: (hd *: tl) => val t2 = t.asInstanceOf[NonEmptyTuple] t2.head.asInstanceOf[hd] *: insert[tl](t2.tail.asInstanceOf[DropUnits[tl]]) case EmptyTuple => EmptyTuple } }.asInstanceOf[A]

CODECS FOR CASE CLASSES 2 4 val a = int(3) :: bool :: cstring // val a: Codec[(Int, Boolean, String)] = … case class Foo(x: Int, y: Boolean, z: String) val b = a.as[Foo] // val b: Codec[Foo] = …

CODECS FOR CASE CLASSES 2 5 trait Codec[A] extends Encoder[A], Decoder[A] { def as[B](using iso: Iso[A, B]): Codec[B] = xmap(iso.to, iso.from) } trait Iso[A, B] { def to(a: A): B def from(b: B): A } How can we create an Iso instance for a case class and a tuple of its elements? ?

MIRRORS 2 6 import scala.deriving.Mirror given product[T <: Tuple, P](using m: Mirror.ProductOf[P] { type MirroredElemTypes = T }) as Iso[T, P] = instance[T, P](fromTuple)(toTuple)

MIRRORS 2 7 import scala.deriving.Mirror given product[T <: Tuple, P](using m: Mirror.ProductOf[P] { type MirroredElemTypes = T }) as Iso[T, P] = instance[T, P](fromTuple)(toTuple) def fromTuple[A, B <: Tuple](b: B)(using m: Mirror.ProductOf[A] { type MirroredElemTypes = B }): A = m.fromProduct(b.asInstanceOf[Product]).asInstanceOf[A] def toTuple[A, B <: Tuple](a: A)(using m: Mirror.ProductOf[A] { type MirroredElemTypes = B }): B = Tuple.fromProduct(a.asInstanceOf[Product]).asInstanceOf[B]

DERIVATION 2 8 case class Point(x: Int, y: Int, z: Int) derives Codec val p = summon[Codec[Point]] // p: Codec[Point] = … val q = p.encode(Point(1, 2, 3)) // q: Attempt[BitVector] = Successful(BitVector(96 bits, 
 0x000000010000000200000003)) Case Classes enum Color derives Codec { case Red, Green, Blue } val r = summon[Codec[Color]] // r: Codec[Color] = … val s = r.encode(Color.Green) // s: Attempt[BitVector] = Successful(BitVector(8 bits, 0x01)) Enums & ADTs

DERIVATION 2 9 import scala.compiletime._ import scala.deriving._ object Codec { inline def derived[A](using m: Mirror.Of[A]): Codec[A] = new Codec[A] { def sizeBound = ??? def encode(a: A) = ??? def decode(b: BitVector) = ??? } } Define a derived method in type constructor companion object 1

DERIVATION 3 0 object Codec { inline def derived[A](using m: Mirror.Of[A]): Codec[A] = new Codec[A] { def sizeBound = inline m match { case p: Mirror.ProductOf[A] => sizeBoundElems[p.MirroredElemTypes] case s: Mirror.SumOf[A] => codecs.uint8.sizeBound + sizeBoundCases[s.MirroredElemTypes] } def encode(a: A) = ??? def decode(b: BitVector) = ??? } } Pattern match on the mirror of the type param, providing an implementation for both products (case classes) and sums (enums/ADTs) 2

DERIVATION 3 1 inline def sizeBoundElems[A <: Tuple]: SizeBound = inline erasedValue[A] match { case _: (hd *: tl) => summonInline[Codec[hd]].sizeBound + sizeBoundElems[tl] case EmptyTuple => SizeBound.exact(0) }

DERIVATION 3 2 inline def sizeBoundCases[A <: Tuple]: SizeBound = inline erasedValue[A] match { case _: (hd *: tl) => val hdSize = summonFrom { case p: Mirror.ProductOf[`hd`] => sizeBoundElems[p.MirroredElemTypes] } hdSize | sizeBoundCases[tl] case EmptyTuple => SizeBound.exact(0) }

WHAT’S NEXT? 3 3 Summary • Scala 3 drastically simplified scodec without sacrificing expressiveness • Many more language features and simplifications Library Availability Increasing • munit, scalatest, scalacheck, shapeless • fastparse, sourcecode, upickle, utest • circe, cats, cats-effect, fs2 (soon) • Lots more! Language Still Evolving • Add your project to the community build • Dotty contributors get better feedback about how their changes are impacting ecosystem • Library developers get help with upgrading to newer versions