Faster Data Serialization with Neon

Challenges when implementing data serialization in the Zalo chat app. How Zalo team has solved these challenges internally with Neon.

Transcript

1. faster data serialization with Neon Hieu Hua
2. Challenges
3. class ZaloWidget { String id; double top; double left; double bottom; double right; ZaloWidget[] children; Style style; // ... }}
4. class ZaloWidget { String id; double top; double left; double bottom; double right; ZaloWidget[] children; Style style; // ... }}
5. class ZaloWidget { String id; double top; double left; double bottom; double right; ZaloWidget[] children; Style style; // ... Click click; LongClick longClick; Background background; BackgroundPosition backgroundPositon; BackgroundRepeat backgroundRepeat; Shadow shadow; Border border; }}
6. Requirements • Performance • Backward compatibility • Persistability • Debuggability • Productivity Library Rescue • SQLite • Serializable • Parcelable • Flatbuffers • Others…
7. X √ √ √ X X X √ X √ √ X X X √ √ √ √ X X Performance Backward compatibility Persistability Debuggability Productivity SQLite Serializable Parcelable Flatbuffers
8. Library Rescue • Neon • SQLite • Serializable • Parcelable • Flatbuffers • Others…
9. Neon
10. Neon class ZaloWidget { String id; Style style; }}
11. class ZaloWidget { String id; Style style; }}
12. @NeonObject class ZaloWidget { String id; Style style; }} // serialize byte[] output = Neon.serialize(widget); // deserialize Neon.deserialize(output, ZaloWidget.class);
13. class ZaloWidget__Neon { static void serialize(ZaloWidget widget, SerializedOutput writer) { writer.writeString(widget.id); Style__Neon.serialize(widget.style, writer); }} static ZaloWidget deserialize(SerializedInput reader) { ZaloWidget widget = new ZaloWidget(); widget.id = reader.readString(); widget.style = Style__Neon.deserialize(reader); return widget; }} }}
14. class ZaloWidget__Neon { static void serialize(ZaloWidget widget, SerializedOutput writer) { writer.writeString(widget.id); Style__Neon.serialize(widget.style, writer); }} static ZaloWidget deserialize(SerializedInput reader) { ZaloWidget widget = new ZaloWidget(); widget.id = reader.readString(); widget.style = Style__Neon.deserialize(reader); return widget; }} }}
15. class ZaloWidget__Neon { static void serialize(ZaloWidget widget, SerializedOutput writer) { writer.writeString(widget.id); Style__Neon.serialize(widget.style, writer); }} static ZaloWidget deserialize(SerializedInput reader) { ZaloWidget widget = new ZaloWidget(); widget.id = reader.readString(); widget.style = Style__Neon.deserialize(reader); return widget; }} }}
16. Object start Object fields class ZaloWidget__Neon { static void serialize(ZaloWidget widget, SerializedOutput writer) { writer.writeString(widget.id); Style__Neon.serialize(widget.style, writer); }} static ZaloWidget deserialize(SerializedInput reader) { ZaloWidget widget = new ZaloWidget(); widget.id = reader.readString(); widget.style = Style__Neon.deserialize(reader); return widget; }} }} Type header Value New fields Object end
17. class ZaloWidget__Neon { static void serialize(ZaloWidget widget, SerializedOutput writer) { writer.writeInt(0); // version writer.writeString(widget.id); Style__Neon.serialize(widget.style, writer); }} static ZaloWidget deserialize(SerializedInput reader) { int version = reader.readInt(); ZaloWidget widget = new ZaloWidget(); widget.id = reader.readString(); widget.style = Style__Neon.deserialize(reader); return widget; }} }} @NeonObject(version = 0)) public class ZaloWidget { String id; Style style; }}
18. class ZaloWidget__Neon { static void serialize(ZaloWidget widget, SerializedOutput writer) { writer.writeInt(0); // version writer.writeString(widget.id); Style__Neon.serialize(widget.style, writer); }} static ZaloWidget deserialize(SerializedInput reader) { int version = reader.readInt(); ZaloWidget widget = new ZaloWidget(); widget.id = reader.readString(); widget.style = Style__Neon.deserialize(reader); return widget; }} }} @NeonObject(version = 0)) public class ZaloWidget { String id; Style style; }}
19. class ZaloWidget__Neon { static void serialize(ZaloWidget widget, SerializedOutput writer) { writer.writeInt(1); // version writer.writeString(widget.id); Style__Neon.serialize(widget.style, writer); }} static ZaloWidget deserialize(SerializedInput reader) { int version = reader.readInt(); ZaloWidget widget = new ZaloWidget(); widget.id = reader.readString(); widget.style = Style__Neon.deserialize(reader); if (version >= 1) { widget.visible = reader.readBool(); } return widget; }} }} @NeonObject(version = 1)) public class ZaloWidget { String id; Style style; @NeonProperty(since = 1) boolean visible; }}
20. class ZaloWidget__Neon { static void serialize(ZaloWidget widget, SerializedOutput writer) { writer.writeInt(2); // version writer.writeString(widget.id); Style__Neon.serialize(widget.style, writer); }} static ZaloWidget deserialize(SerializedInput reader) { int version = reader.readInt(); ZaloWidget widget = new ZaloWidget(); widget.id = reader.readString(); widget.style = Style__Neon.deserialize(reader); if (version >= 1) { widget.visible = reader.readBool(); } if (version >= 2) { widget.color = reader.readInt(); } return widget; }} }} @NeonObject(version = 2)) public class ZaloWidget { String id; Style style; @NeonProperty(since = 1) boolean visible; @NeonProperty(since = 2) int color; }}
21. @NeonObject(version = 2) public class ZaloWidget { // ... @NeonProperty(since = 2) int color; }}
22. @NeonObject(version = 2) @Migrator(ZaloWidgetMigrator.class) public class ZaloWidget { // ... @NeonProperty(since = 2) int color; }} public class ZaloWidgetMigrator implements Migrator<ZaloWidget> { @Override void migrate(ZaloWidget object, int fromVersion, int toVersion) { if (fromVersion <= 1) { object.color = Color.RED; } } }
23. @NeonObject(version = 3) public class ZaloWidget { // ... @NeonProperty(since = 3) Date lastLayout; }}
24. @NeonObject(version = 3) public class ZaloWidget { // ... @NeonProperty(since = 3, adapter = DateNeonAdapter.class) Date lastLayout; }} public class DateNeonAdapter implements NeonAdapter<Date> { @Override public void serialize(@NonNull Date object, SerializedOutput writer) { writer.writeLong(object == null ? -1 : object.getTime()); } @Override public Date deserialize(SerializedInput reader) { long ts = reader.readLong(); return ts < 0 ? null : new Date(ts); } }
25. ZaloWidget { version : 2 id: String style: Style { version: 0 // other fields }, visible: boolean, color: int } Neon.dump(bytes, ZaloWidget.class);
26. Neon • no reflections • . • . • annotationProcessor magic Backward compatibility Persistability Debuggability Productivity Performance @NeonObject(version = n) @NeonProperty(since = k) • . byte[] data = Neon.serialize(widget); Neon.dump(bytes, ZaloWidget.class);
27. X √ √ √ X X X √ X √ √ X X X √ √ √ √ X X √ √ √ √ √ Performance Backward compatibility Persistability Debuggability Productivity SQLite Serializable Parcelable Flatbuffers Neon
28. Performance ZaloWidget Serialize Deserialize Stream size Serializable ~3.5ms ~3.8ms ~5.1KB Neon ~0.3ms ~0.5ms ~0.8KB
29. github.com/zalo-group/neon v1.0 soon
30. Questions? rockerhieu rockerhieu rockerhieu twitter.com/ github.com/ @gmail.com Keynote template by Jake Wharton