C#11 による世界最速バイナリシリアライザー「MemoryPack」の作り方

.NET Conf 2022 Recap Event Tokyo #dotnetconf

Transcript

2. 河合 宜文 / Kawai Yoshifumi / @neuecc Cysharp, Inc. Cygames C#大統一理論 C#
3. #1 Binary Serializer in .NET https://github.com/neuecc/MessagePack-CSharp
4. Zero encoding extreme fast binary serializer https://github.com/Cysharp/MemoryPack/ 究極的
5. 1リクエストに1000回実行することはある
6. バッチやクライアントアプリケーションでも
7. Incremental Source Generator
10. No.
12. Serialize/MemoryPackWriter
14. public void WriteUnmanaged<T1>(scoped in T1 value1) where T1 : unmanaged { var size = Unsafe.SizeOf<T1>(); ref var spanRef = ref GetSpanReference(size); Unsafe.WriteUnaligned(ref spanRef, value1); Advance(size); } 書き込み用バッファー管理 public ref partial struct MemoryPackWriter<TBufferWriter> where TBufferWriter : IBufferWriter<byte> { ref TBufferWriter bufferWriter; ref byte bufferReference; int bufferLength; ref byte GetSpanReference(int sizeHint); void Advance(int count); public MemoryPackWriter(ref TBufferWriter writer) } public interface System.Buffers.IBufferWriter<T> { Span<T> GetSpan(int sizeHint = 0); void Advance(int count); }
15. public void WriteUnmanaged<T1>(scoped in T1 value1) where T1 : unmanaged { var size = Unsafe.SizeOf<T1>(); ref var spanRef = ref GetSpanReference(size); Unsafe.WriteUnaligned(ref spanRef, value1); Advance(size); } 書き込み用バッファー管理 public ref partial struct MemoryPackWriter<TBufferWriter> where TBufferWriter : IBufferWriter<byte> { ref TBufferWriter bufferWriter; ref byte bufferReference; int bufferLength; ref byte GetSpanReference(int sizeHint); void Advance(int count); public MemoryPackWriter(ref TBufferWriter writer) } public interface System.Buffers.IBufferWriter<T> { Span<T> GetSpan(int sizeHint = 0); void Advance(int count); }
16. public void WriteUnmanaged<T1>(scoped in T1 value1) where T1 : unmanaged { var size = Unsafe.SizeOf<T1>(); ref var spanRef = ref GetSpanReference(size); Unsafe.WriteUnaligned(ref spanRef, value1); Advance(size); } 書き込み用バッファー管理 public ref partial struct MemoryPackWriter<TBufferWriter> where TBufferWriter : IBufferWriter<byte> { ref TBufferWriter bufferWriter; ref byte bufferReference; int bufferLength; ref byte GetSpanReference(int sizeHint); void Advance(int count); public MemoryPackWriter(ref TBufferWriter writer) } public interface System.Buffers.IBufferWriter<T> { Span<T> GetSpan(int sizeHint = 0); void Advance(int count); }
17. public void WriteUnmanaged<T1>(scoped in T1 value1) where T1 : unmanaged { var size = Unsafe.SizeOf<T1>(); ref var spanRef = ref GetSpanReference(size); Unsafe.WriteUnaligned(ref spanRef, value1); Advance(size); } 書き込み用バッファー管理 public ref partial struct MemoryPackWriter<TBufferWriter> where TBufferWriter : IBufferWriter<byte> { ref TBufferWriter bufferWriter; ref byte bufferReference; int bufferLength; ref byte GetSpanReference(int sizeHint); void Advance(int count); public MemoryPackWriter(ref TBufferWriter writer) } public interface System.Buffers.IBufferWriter<T> { Span<T> GetSpan(int sizeHint = 0); void Advance(int count); }
18. 書き込み用バッファー管理 public ref partial struct MemoryPackWriter<TBufferWriter> where TBufferWriter : IBufferWriter<byte> { ref TBufferWriter bufferWriter; ref byte bufferReference; int bufferLength; ref byte GetSpanReference(int sizeHint); void Advance(int count); public MemoryPackWriter(ref TBufferWriter writer) }
19. public static partial class MemoryPackSerializer { public static void Serialize<T, TBufferWriter>(in TBufferWriter bufferWriter, in T? value) public static byte[] Serialize<T>(in T? value) public static ValueTask SerializeAsync<T>(Stream stream, T? value) } var writer = new MemoryPackWriter<TBufferWriter>(ref bufferWriter); writer.WriteValue(value); writer.Flush();
20. public static partial class MemoryPackSerializer { public static void Serialize<T, TBufferWriter>(in TBufferWriter bufferWriter, in T? value) public static byte[] Serialize<T>(in T? value) public static ValueTask SerializeAsync<T>(Stream stream, T? value) } var bufferWriter = ReusableLinkedArrayBufferWriterPool.Rent(); var writer = new MemoryPackWriter<ReusableLinkedArrayBufferWriter>(ref bufferWriter); writer.WriteValue(value); writer.Flush(); await bufferWriter.WriteToAndResetAsync(stream); return bufferWriter.ToArrayAndReset();
21. byte[] byte[] byte[] ArrayPool<byte>.Shared.Rent GetSpan() public sealed class ReusableLinkedArrayBufferWriter : IBufferWriter<byte> { List<BufferSegment> buffers; } struct BufferSegment { byte[] buffer; int written; }
22. IBufferWriter<byte>を優先する
23. Deserialize/MemoryPackReader
25. public ref partial struct MemoryPackReader { ReadOnlySequence<byte> bufferSource; ref byte bufferReference; int bufferLength; ref byte GetSpanReference(int sizeHint); void Advance(int count); public MemoryPackReader(in ReadOnlySequence<byte> bufferSource) public MemoryPackReader(ReadOnlySpan<byte> buffer) } 読み込み用バッファー管理
26. 連結されたbyte[]、のようなもの
27. 性能は同期バッファと非同期読み書きで決まる Streaming処理は別の機構で行う
28. Optimize for All Types
29. C#の配列は要素がunmanaged型(参照型を含まな いstruct)の場合、全て直列に並ぶ var srcLength = Unsafe.SizeOf<T>() * value.Length; var allocSize = srcLength + 4; ref var dest = ref GetSpanReference(allocSize); ref var src = ref Unsafe.As<T, byte>(ref GetArrayDataReference(value)); Unsafe.WriteUnaligned(ref dest, value.Length); Unsafe.CopyBlockUnaligned(ref Unsafe.Add(ref dest, 4), ref src, (uint)srcLength); Advance(allocSize);
30. Vector3[]など複合型になればなるほど有利
31. public sealed class ListFormatter<T> : MemoryPackFormatter<List<T?>> { public override void Serialize<TBufferWriter>( ref MemoryPackWriter<TBufferWriter> writer, scoped ref List<T?>? value) { if (value == null) { writer.WriteNullCollectionHeader(); return; } var span = CollectionsMarshal.AsSpan(value); var formatter = GetFormatter<T>(); WriteCollectionHeader(span.Length); for (int i = 0; i < span.Length; i++) { formatter.Serialize(ref this, ref span[i]); } } }
32. public override void Deserialize(ref MemoryPackReader reader, scoped ref List<T?>? value) { if (!reader.TryReadCollectionHeader(out var length)) { value = null; return; } value = new List<T?>(length); var span = CollectionsMarshalEx.CreateSpan(value, length); var formatter = GetFormatter<T>(); for (int i = 0; i < length; i++) { formatter.Deserialize(ref this, ref span[i]); } } internal static class CollectionsMarshalEx { public static Span<T?> CreateSpan<T>( List<T?> list, int length) { list.EnsureCapacity(length); ref var view = ref Unsafe.As<List<T?>, ListView<T?>>(ref list); view._size = length; return view._items.AsSpan(0, length); } } internal sealed class ListView<T> { public T[] _items; public int _size; public int _version; }
33. Conclusion
34. 究極のシリアライザーを作る MemoryPack https://github.com/Cysharp/MemoryPack/