Upgrade to Pro — share decks privately, control downloads, hide ads and more …

C#11 による世界最速バイナリシリアライザー「MemoryPack」の作り方

C#11 による世界最速バイナリシリアライザー「MemoryPack」の作り方

.NET Conf 2022 Recap Event Tokyo #dotnetconf

Yoshifumi Kawai

December 06, 2022
Tweet

More Decks by Yoshifumi Kawai

Other Decks in Technology

Transcript

  1. None
  2. 河合 宜文 / Kawai Yoshifumi / @neuecc Cysharp, Inc. Cygames

    C#大統一理論 C#
  3. #1 Binary Serializer in .NET https://github.com/neuecc/MessagePack-CSharp

  4. Zero encoding extreme fast binary serializer https://github.com/Cysharp/MemoryPack/ 究極的

  5. 1リクエストに1000回実行することはある

  6. バッチやクライアントアプリケーションでも

  7. Incremental Source Generator

  8. None
  9. None
  10. No.

  11. None
  12. Serialize/MemoryPackWriter

  13. None
  14. public void WriteUnmanaged<T1>(scoped in T1 value1) where T1 : unmanaged

    { var size = Unsafe.SizeOf<T1>(); ref var spanRef = ref GetSpanReference(size); Unsafe.WriteUnaligned(ref spanRef, value1); Advance(size); } 書き込み用バッファー管理 public ref partial struct MemoryPackWriter<TBufferWriter> where TBufferWriter : IBufferWriter<byte> { ref TBufferWriter bufferWriter; ref byte bufferReference; int bufferLength; ref byte GetSpanReference(int sizeHint); void Advance(int count); public MemoryPackWriter(ref TBufferWriter writer) } public interface System.Buffers.IBufferWriter<T> { Span<T> GetSpan(int sizeHint = 0); void Advance(int count); }
  15. public void WriteUnmanaged<T1>(scoped in T1 value1) where T1 : unmanaged

    { var size = Unsafe.SizeOf<T1>(); ref var spanRef = ref GetSpanReference(size); Unsafe.WriteUnaligned(ref spanRef, value1); Advance(size); } 書き込み用バッファー管理 public ref partial struct MemoryPackWriter<TBufferWriter> where TBufferWriter : IBufferWriter<byte> { ref TBufferWriter bufferWriter; ref byte bufferReference; int bufferLength; ref byte GetSpanReference(int sizeHint); void Advance(int count); public MemoryPackWriter(ref TBufferWriter writer) } public interface System.Buffers.IBufferWriter<T> { Span<T> GetSpan(int sizeHint = 0); void Advance(int count); }
  16. public void WriteUnmanaged<T1>(scoped in T1 value1) where T1 : unmanaged

    { var size = Unsafe.SizeOf<T1>(); ref var spanRef = ref GetSpanReference(size); Unsafe.WriteUnaligned(ref spanRef, value1); Advance(size); } 書き込み用バッファー管理 public ref partial struct MemoryPackWriter<TBufferWriter> where TBufferWriter : IBufferWriter<byte> { ref TBufferWriter bufferWriter; ref byte bufferReference; int bufferLength; ref byte GetSpanReference(int sizeHint); void Advance(int count); public MemoryPackWriter(ref TBufferWriter writer) } public interface System.Buffers.IBufferWriter<T> { Span<T> GetSpan(int sizeHint = 0); void Advance(int count); }
  17. public void WriteUnmanaged<T1>(scoped in T1 value1) where T1 : unmanaged

    { var size = Unsafe.SizeOf<T1>(); ref var spanRef = ref GetSpanReference(size); Unsafe.WriteUnaligned(ref spanRef, value1); Advance(size); } 書き込み用バッファー管理 public ref partial struct MemoryPackWriter<TBufferWriter> where TBufferWriter : IBufferWriter<byte> { ref TBufferWriter bufferWriter; ref byte bufferReference; int bufferLength; ref byte GetSpanReference(int sizeHint); void Advance(int count); public MemoryPackWriter(ref TBufferWriter writer) } public interface System.Buffers.IBufferWriter<T> { Span<T> GetSpan(int sizeHint = 0); void Advance(int count); }
  18. 書き込み用バッファー管理 public ref partial struct MemoryPackWriter<TBufferWriter> where TBufferWriter : IBufferWriter<byte>

    { ref TBufferWriter bufferWriter; ref byte bufferReference; int bufferLength; ref byte GetSpanReference(int sizeHint); void Advance(int count); public MemoryPackWriter(ref TBufferWriter writer) }
  19. public static partial class MemoryPackSerializer { public static void Serialize<T,

    TBufferWriter>(in TBufferWriter bufferWriter, in T? value) public static byte[] Serialize<T>(in T? value) public static ValueTask SerializeAsync<T>(Stream stream, T? value) } var writer = new MemoryPackWriter<TBufferWriter>(ref bufferWriter); writer.WriteValue(value); writer.Flush();
  20. public static partial class MemoryPackSerializer { public static void Serialize<T,

    TBufferWriter>(in TBufferWriter bufferWriter, in T? value) public static byte[] Serialize<T>(in T? value) public static ValueTask SerializeAsync<T>(Stream stream, T? value) } var bufferWriter = ReusableLinkedArrayBufferWriterPool.Rent(); var writer = new MemoryPackWriter<ReusableLinkedArrayBufferWriter>(ref bufferWriter); writer.WriteValue(value); writer.Flush(); await bufferWriter.WriteToAndResetAsync(stream); return bufferWriter.ToArrayAndReset();
  21. byte[] byte[] byte[] ArrayPool<byte>.Shared.Rent GetSpan() public sealed class ReusableLinkedArrayBufferWriter :

    IBufferWriter<byte> { List<BufferSegment> buffers; } struct BufferSegment { byte[] buffer; int written; }
  22. IBufferWriter<byte>を優先する

  23. Deserialize/MemoryPackReader

  24. None
  25. public ref partial struct MemoryPackReader { ReadOnlySequence<byte> bufferSource; ref byte

    bufferReference; int bufferLength; ref byte GetSpanReference(int sizeHint); void Advance(int count); public MemoryPackReader(in ReadOnlySequence<byte> bufferSource) public MemoryPackReader(ReadOnlySpan<byte> buffer) } 読み込み用バッファー管理
  26. 連結されたbyte[]、のようなもの

  27. 性能は同期バッファと非同期読み書きで決まる Streaming処理は別の機構で行う

  28. Optimize for All Types

  29. C#の配列は要素がunmanaged型(参照型を含まな いstruct)の場合、全て直列に並ぶ var srcLength = Unsafe.SizeOf<T>() * value.Length; var allocSize

    = srcLength + 4; ref var dest = ref GetSpanReference(allocSize); ref var src = ref Unsafe.As<T, byte>(ref GetArrayDataReference(value)); Unsafe.WriteUnaligned(ref dest, value.Length); Unsafe.CopyBlockUnaligned(ref Unsafe.Add(ref dest, 4), ref src, (uint)srcLength); Advance(allocSize);
  30. Vector3[]など複合型になればなるほど有利

  31. public sealed class ListFormatter<T> : MemoryPackFormatter<List<T?>> { public override void

    Serialize<TBufferWriter>( ref MemoryPackWriter<TBufferWriter> writer, scoped ref List<T?>? value) { if (value == null) { writer.WriteNullCollectionHeader(); return; } var span = CollectionsMarshal.AsSpan(value); var formatter = GetFormatter<T>(); WriteCollectionHeader(span.Length); for (int i = 0; i < span.Length; i++) { formatter.Serialize(ref this, ref span[i]); } } }
  32. public override void Deserialize(ref MemoryPackReader reader, scoped ref List<T?>? value)

    { if (!reader.TryReadCollectionHeader(out var length)) { value = null; return; } value = new List<T?>(length); var span = CollectionsMarshalEx.CreateSpan(value, length); var formatter = GetFormatter<T>(); for (int i = 0; i < length; i++) { formatter.Deserialize(ref this, ref span[i]); } } internal static class CollectionsMarshalEx { public static Span<T?> CreateSpan<T>( List<T?> list, int length) { list.EnsureCapacity(length); ref var view = ref Unsafe.As<List<T?>, ListView<T?>>(ref list); view._size = length; return view._items.AsSpan(0, length); } } internal sealed class ListView<T> { public T[] _items; public int _size; public int _version; }
  33. Conclusion

  34. 究極のシリアライザーを作る MemoryPack https://github.com/Cysharp/MemoryPack/

  35. None