gRPC and Protobufs - JSCamp Romania

My talk that I gave on gRPC and Protocol Buffers at JSCamp Romania.

Transcript

1. gRPC and Protobufs
   Iheanyi Ekechukwu
   

   View Slide

2. Iheanyi Ekechukwu
   Software Engineer @
   DigitalOcean
   @kwuchu
   

   View Slide

3. What are protobufs?
   

   View Slide

4. Protobufs
   

   View Slide

5. Protocol Buffers
   

   View Slide

6. Language-agnostic method
   for serializing structured data.
   

   View Slide

7. How do they work?
   

   View Slide

8. Defining a Message
   

   View Slide

9. // contact.proto
   syntax = "proto3";
   message Contact {
   string name = 1;
   string email = 2;
   repeated PhoneNumber phone_numbers = 3;
   PhoneNumber home = 4;
   PhoneNumber mobile = 5;
   PhoneNumber work = 6;
   }
   message PhoneNumber {
   enum PhoneType {
   MOBILE = 0;
   HOME = 1;
   WORK = 2;
   }
   string number = 1;
   PhoneType type = 2;
   }
   

   View Slide

10. message Contact
    

    View Slide

11. string name = 1;
    

    View Slide

12. double float int32 int64 uint32
    uint64 sint32 sint64 fixed32
    fixed64 sfixed32 sfixed64 bool
    string
    

    View Slide

13. repeated PhoneNumber phone_numbers = 3;
    

    View Slide

14. message PhoneNumber {
    enum PhoneType {
    MOBILE = 0;
    HOME = 1;
    WORK = 2;
    }
    string number = 1;
    PhoneType type = 2;
    }
    

    View Slide

15. enum PhoneType {
    MOBILE = 0;
    HOME = 1;
    WORK = 2;
    }
    

    View Slide

16. Compiled to a binary
    wire format
    

    View Slide

17. Compiling
    

    View Slide

18. protoc -I=. --go_out=. --js_out=import_style=commonjs,binary:.
    contact.proto
    

    View Slide

19. Generates two files:
    contact.pb.go and contact_pb.js
    

    View Slide

20. package contact
    import proto "github.com/golang/protobuf/proto"
    import fmt "fmt"
    import math "math"
    // other code omitted for brevity
    type Contact struct {
    Name string `protobuf:"bytes,1,opt,name=name"
    json:"name,omitempty"`
    Email string `protobuf:"bytes,2,opt,name=email"
    json:"email,omitempty"`
    PhoneNumbers []*PhoneNumber `protobuf:"bytes,
    3,rep,name=phone_numbers,json=phoneNumbers"
    json:"phone_numbers,omitempty"`
    Home *PhoneNumber `protobuf:"bytes,4,opt,name=home"
    json:"home,omitempty"`
    Mobile *PhoneNumber `protobuf:"bytes,5,opt,name=mobile"
    json:"mobile,omitempty"`
    Work *PhoneNumber `protobuf:"bytes,6,opt,name=work"
    json:"work,omitempty"`
    }
    

    View Slide

21. func init() { proto.RegisterFile("contact.proto", fileDescriptor0) }
    var fileDescriptor0 = []byte{
    // 249 bytes of a gzipped FileDescriptorProto
    0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x4d, 0xce, 0xcf,
    0x2b,
    0x49, 0x4c, 0x2e, 0xd1, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0x87, 0x72, 0x95, 0x3e,
    0x30,
    0x72, 0xb1, 0x3b, 0x43, 0xd8, 0x42, 0x42, 0x5c, 0x2c, 0x79, 0x89, 0xb9, 0xa9, 0x12, 0x8c,
    0x0a,
    0x8c, 0x1a, 0x9c, 0x41, 0x60, 0xb6, 0x90, 0x08, 0x17, 0x6b, 0x6a, 0x6e, 0x62, 0x66, 0x8e,
    0x04,
    0x13, 0x58, 0x10, 0xc2, 0x11, 0xb2, 0xe4, 0xe2, 0x2d, 0xc8, 0xc8, 0xcf, 0x4b, 0x8d, 0xcf,
    0x2b,
    0xcd, 0x4d, 0x4a, 0x2d, 0x2a, 0x96, 0x60, 0x56, 0x60, 0xd6, 0xe0, 0x36, 0x12, 0xd1, 0x83,
    0xd9,
    0x12, 0x00, 0x92, 0xf5, 0x03, 0x4b, 0x06, 0xf1, 0x14, 0x20, 0x38, 0xc5, 0x42, 0x1a, 0x5c,
    0x2c,
    0x19, 0xf9, 0xb9, 0xa9, 0x12, 0x2c, 0x0a, 0x8c, 0x38, 0x75, 0x80, 0x55, 0x08, 0xe9, 0x70,
    0xb1,
    0xe5, 0xe6, 0x27, 0x65, 0xe6, 0xa4, 0x4a, 0xb0, 0xe2, 0x51, 0x0b, 0x55, 0x03, 0x32, 0xb7,
    0x3c,
    0xbf, 0x28, 0x5b, 0x82, 0x0d, 0x9f, 0xb9, 0x20, 0x15, 0x4a, 0x6d, 0x8c, 0x5c, 0xdc, 0x48,
    0xa2,
    0x42, 0x62, 0x5c, 0x6c, 0x10, 0x6f, 0x40, 0x3d, 0x0e, 0xe5, 0x09, 0x19, 0x71, 0xb1, 0x94,
    0x54,
    0x16, 0xa4, 0x82, 0x7d, 0xce, 0x67, 0x24, 0x87, 0xcd, 0x44, 0x08, 0x3b, 0xa4, 0xb2, 0x20,
    0x35,
    0x08, 0xac, 0x56, 0x49, 0x9b, 0x8b, 0x13, 0x2e, 0x24, 0xc4, 0xc5, 0xc5, 0xe6, 0xeb, 0xef,
    0xe4,
    0xe9, 0xe3, 0x2a, 0xc0, 0x20, 0xc4, 0xc1, 0xc5, 0xe2, 0xe1, 0xef, 0xeb, 0x2a, 0xc0, 0x08,
    0x62,
    0x85, 0xfb, 0x07, 0x79, 0x0b, 0x30, 0x25, 0xb1, 0x81, 0xe3, 0xc2, 0x18, 0x10, 0x00, 0x00,
    0xff,
    0xff, 0x9b, 0xec, 0x7d, 0xa3, 0x9c, 0x01, 0x00, 0x00,
    }
    

    View Slide

22. const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
    type PhoneNumber_PhoneType int32
    const (
    PhoneNumber_MOBILE PhoneNumber_PhoneType = 0
    PhoneNumber_HOME PhoneNumber_PhoneType = 1
    PhoneNumber_WORK PhoneNumber_PhoneType = 2
    )
    var PhoneNumber_PhoneType_name = map[int32]string{
    0: "MOBILE",
    1: "HOME",
    2: "WORK",
    }
    var PhoneNumber_PhoneType_value = map[string]int32{
    "MOBILE": 0,
    "HOME": 1,
    "WORK": 2,
    }
    func (x PhoneNumber_PhoneType) String() string {
    return proto.EnumName(PhoneNumber_PhoneType_name, int32(x))
    }
    func (PhoneNumber_PhoneType) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{1, 0} }
    type Contact struct {
    Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"`
    Email string `protobuf:"bytes,2,opt,name=email" json:"email,omitempty"`
    PhoneNumbers []*PhoneNumber `protobuf:"bytes,3,rep,name=phone_numbers,json=phoneNumbers" json:"phone_numbers,omitempty"`
    Home *PhoneNumber `protobuf:"bytes,4,opt,name=home" json:"home,omitempty"`
    Mobile *PhoneNumber `protobuf:"bytes,5,opt,name=mobile" json:"mobile,omitempty"`
    Work *PhoneNumber `protobuf:"bytes,6,opt,name=work" json:"work,omitempty"`
    }
    func (m *Contact) Reset() { *m = Contact{} }
    func (m *Contact) String() string { return proto.CompactTextString(m) }
    func (*Contact) ProtoMessage() {}
    func (*Contact) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }
    func (m *Contact) GetPhoneNumbers() []*PhoneNumber {
    if m != nil {
    return m.PhoneNumbers
    }
    return nil
    }
    func (m *Contact) GetHome() *PhoneNumber {
    if m != nil {
    return m.Home
    }
    return nil
    }
    func (m *Contact) GetMobile() *PhoneNumber {
    if m != nil {
    return m.Mobile
    }
    return nil
    }
    func (m *Contact) GetWork() *PhoneNumber {
    if m != nil {
    return m.Work
    }
    return nil
    }
    type PhoneNumber struct {
    Number string `protobuf:"bytes,1,opt,name=number" json:"number,omitempty"`
    Type PhoneNumber_PhoneType `protobuf:"varint,2,opt,name=type,enum=contact.PhoneNumber_PhoneType" json:"type,omitempty"`
    }
    func (m *PhoneNumber) Reset() { *m = PhoneNumber{} }
    func (m *PhoneNumber) String() string { return proto.CompactTextString(m) }
    func (*PhoneNumber) ProtoMessage() {}
    func (*PhoneNumber) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} }
    

    View Slide

23. Also, getters and setters are
    defined for the message’s
    attributes
    

    View Slide

24. // contact_pb.js
    // code omitted for brevity
    /**
    * optional string name = 1;
    * @return {string}
    */
    proto.contact.Contact.prototype.getName = function() {
    return /** @type {string} */ (jspb.Message.getFieldWithDefault(thi
    1, ""));
    };
    /** @param {string} value */
    proto.contact.Contact.prototype.setName = function(value) {
    jspb.Message.setField(this, 1, value);
    };
    

    View Slide

25. Why use them?
    

    View Slide

26. Harder Simpler.
    

    View Slide

27. Clear definition of the
    data model.
    

    View Slide

28. Language Agnostic.
    

    View Slide

29. Better.
    

    View Slide

30. Faster.
    

    View Slide

31. In comparison to XML/
    JSON…
    

    View Slide

32. 3 to 10 times smaller
    

    View Slide

33. 20 to 100 times faster.
    

    View Slide

34. Stronger.
    

    View Slide

35. Strict Contract
    

    View Slide

36. Type definitions are
    also good.
    

    View Slide

37. Simpler.
    

    View Slide

38. Simpler. Better.
    

    View Slide

39. Simpler. Better. Faster.
    

    View Slide

40. Simpler. Better. Faster.
    Stronger.
    

    View Slide

41. View Slide

42. Let’s talk gRPC.
    

    View Slide

43. What is gRPC?
    

    View Slide

44. RPC framework by Google
    that extends protocol buffers.
    

    View Slide

45. Why use it?
    

    View Slide

46. Uses HTTP/2 under the
    hood.
    

    View Slide

47. SSL/TLS for secure
    communication between server
    and clients.
    

    View Slide

48. Deadlines/timeouts can be
    specified on both the server and
    the client.
    

    View Slide

49. Cancellable RPCs
    

    View Slide

50. Anybody here ever been
    burned by JSON changes?
    

    View Slide

51. View Slide

52. Safe (to an extent), to
    rename fields.
    

    View Slide

53. Did I mention clients are
    backwards/forward-compatible?
    

    View Slide

54. View Slide

55. Old client binaries
    ignore new fields added.
    

    View Slide

56. All the benefits of
    protocol buffers.
    

    View Slide

57. Stronger contracts.
    

    View Slide

58. Faster & Better
    Performance
    

    View Slide

59. Language Agnosticism
    for server and clients.
    

    View Slide

60. Great for microservices.
    

    View Slide

61. So, how does it work?
    

    View Slide

62. Let’s build a phonebook
    service and CLI.
    

    View Slide

63. First, define the service
    

    View Slide

64. syntax = "proto3";
    package api;
    service PhoneBook {}
    

    View Slide

65. Let’s add an RPC method
    to create a contact.
    

    View Slide

66. syntax = "proto3";
    package api;
    service PhoneBook {}
    

    View Slide

67. syntax = "proto3";
    package api;
    service PhoneBook {
    rpc CreateContact(CreateContactReq) returns (CreateContactRes);
    }
    message CreateContactReq {
    string name = 1;
    string email = 2;
    repeated PhoneNumber phone_numbers = 3;
    PhoneNumber home = 4;
    PhoneNumber mobile = 5;
    PhoneNumber work = 6;
    }
    message CreateContactRes {
    Contact contact = 1;
    }
    

    View Slide

68. Compile the protobuf
    with the gRPC plugin.
    

    View Slide

69. protoc -I=. --proto_path=${GOPATH}/src:. --go_out=plugins=grpc:. \
    --js_out=import_style=commonjs,binary:. api.proto
    

    View Slide

70. Generates api.pb.go and api_pb.js,
    like before, with some differences.
    

    View Slide

71. // api/api.pb.go
    package api
    // Some code omitted for brevity
    // Server API for PhoneBook service
    type PhoneBookServer interface {
    CreateContact(context.Context, *CreateContactReq)
    (*CreateContactRes, error)
    }
    func RegisterPhoneBookServer(s *grpc.Server, srv PhoneBookServer) {
    s.RegisterService(&_PhoneBook_serviceDesc, srv)
    }
    

    View Slide

72. The client also gets
    defined within these files.
    

    View Slide

73. type PhoneBookClient interface {
    CreateContact(ctx context.Context, in *CreateContactReq,
    opts ...grpc.CallOption) (*CreateContactRes, error)
    }
    type phoneBookClient struct {
    cc *grpc.ClientConn
    }
    func NewPhoneBookClient(cc *grpc.ClientConn) PhoneBookClient {
    return &phoneBookClient{cc}
    }
    

    View Slide

74. Let’s implement the CreateContact
    method on the Go server.
    

    View Slide

75. // server/server.go
    package server
    import (
    "sync"
    "google.golang.org/grpc/codes"
    "google.golang.org/grpc/status"
    oldctx "golang.org/x/net/context"
    api "github.com/iheanyi/grpc-phonebook/api"
    )
    // Use in-memory DB for simplicity
    type server struct {
    contactsByNameMu sync.RWMutex
    contactsByName map[string]*api.Contact
    }
    

    View Slide

76. func (svc *server) CreateContact(ctx oldctx.Context, req
    *api.CreateContactReq) (*api.CreateContactRes, error) {
    if len(req.Name) == 0 {
    return nil, status.Errorf(codes.InvalidArgument, "name cannot
    be blank")
    }
    contact := &api.Contact{
    Name: req.Name,
    Email: req.Email,
    PhoneNumbers: req.PhoneNumbers,
    Home: req.Home,
    Mobile: req.Mobile,
    Work: req.Work,
    }
    svc.contactsByName[contact.Name] = contact
    res := &api.CreateContactRes{
    Contact: contact,
    }
    return res, nil
    }
    

    View Slide

77. Register the service
    with a gRPC Server
    

    View Slide

78. // server/server.go
    func New() api.PhoneBookServer {
    contactsByName := make(map[string]*api.Contact)
    return &server{
    contactsByName: contactsByName,
    }
    }
    

    View Slide

79. // cmd/pbsrv/main.go
    package main
    import (
    "log"
    "net"
    "github.com/iheanyi/grpc-phonebook/api"
    "github.com/iheanyi/grpc-phonebook/server"
    "google.golang.org/grpc"
    )
    func main() {
    lis, err := net.Listen("tcp", ":50051")
    if err != nil {
    log.Fatalf("failed to listen: %v", err)
    }
    srv := grpc.NewServer()
    svc := server.New()
    api.RegisterPhoneBookServer(srv, svc)
    srv.Serve(lis)
    }
    

    View Slide

80. Done.
    

    View Slide

81. Now, let’s implement it
    in our Node.js client.
    

    View Slide

82. #!/usr/bin/env node
    // code omitted for brevity
    // cmd/pbctl/pbctl
    var PROTO_PATH = __dirname + '/../../api/api.proto';
    var grpc = require('grpc');
    var program = require('commander');
    var path = require('path');
    var api = grpc.load(path.resolve(PROTO_PATH)).api;
    var client = new api.PhoneBook('localhost:50051',
    grpc.credentials.createInsecure());
    require('console.table');
    var PhoneType = api.PhoneNumber.PhoneType;
    // end of file
    program.parse(process.argv);
    

    View Slide

83. program
    .command('create ’)
    .option('-e, --email ', "Contact's email")
    .option('-h, --home ', "Contact's home number")
    .option('-m, --mobile ', "Contact's mobile number")
    .option('-w, --work ', "Contact's work number")
    

    View Slide

84. program.action(function(name, options) {
    var request = new api.CreateContactReq();
    var phoneArr = [];
    if (options.home) {
    var homeNum = new api.PhoneNumber();
    homeNum.type = PhoneType.HOME;
    homeNum.number = options.home;
    phoneArr.push(homeNum);
    request.home = homeNum;
    }
    // Repeat logic for mobile/work numbers.
    // Omitted for brevity.
    request.name = name;
    request.email = options.email || '';
    request.phone_numbers = phoneArr;
    client.createContact(request, function(err, response) {
    if (err) {
    console.error(err.message);
    return;
    }
    printContacts(formatContacts([response.contact]));
    });
    })
    

    View Slide

85. Done.
    !
    

    View Slide

86. Let’s test it out.
    

    View Slide

87. First, let’s compile and run
    our server in one terminal.
    

    View Slide

88. go install ./cmd/pbsrv/... && pbsrv
    

    View Slide

89. Let’s try out our create
    command.
    

    View Slide

90. ./pbctl create “Iheanyi Ekechukwu” -h 1234567890 -e test@example.com
    

    View Slide

91. Our Output!
    name email home mobile work
    ----------------- ---------------- ---------- ------ ----
    Iheanyi Ekechukwu test@example.com 1234567890
    

    View Slide

92. Nice!
    

    View Slide

93. And what if our input is
    invalid?
    

    View Slide

94. ./pbctl create "" -h 1234567890
    

    View Slide

95. Outputs to stderr
    name cannot be blank
    

    View Slide

96. View Slide

97. But it doesn’t stop there, we could
    implement other methods too!
    

    View Slide

98. service PhoneBook {
    rpc CreateContact(CreateContactReq) returns (CreateContactRes);
    rpc ListContacts(ListContactsReq) returns (ListContactsRes);
    rpc DeleteContact(DeleteContactReq) returns (DeleteContactRes);
    rpc ShowContact(ShowContactReq) returns (ShowContactRes);
    rpc UpdateContact(UpdateContactReq) returns (UpdateContactRes);
    }
    

    View Slide

99. View Slide

100. https://github.com/iheanyi/grpc-phonebook
     

     View Slide

101. Other Features of gRPC
     

     View Slide

102. Bring your own IDL.
     

     View Slide

103. Load Balancing
     

     View Slide

104. Interceptors (Go, Java)
     

     View Slide

105. Streaming
     (Client/Server/Bidirectional)
     

     View Slide

106. Client Streaming
     

     View Slide

107. Server Streaming
     

     View Slide

108. Bidirectional Streaming
     

     View Slide

109. Metadata
     

     View Slide

110. The OSS Ecosystem
     

     View Slide

111. https://github.com/grpc-ecosystem
     

     View Slide

112. gRPC enables new possibilities
     for building software.
     

     View Slide

113. gRPC + Mobile
     Applications
     

     View Slide

114. gRPC + Electron
     

     View Slide

115. gRPC Web
     

     View Slide

116. Further Reading
     • https://grpc.io - gRPC Website & Documentation
     • https://github.com/improbable-eng/grpc-web –
     Improbable’s gRPC Web implementation
     • https://developers.google.com/protocol-buffers/docs/
     proto3 - Protocol Buffers Documentation
     • https://github.com/grpc-ecosystem – GitHub
     Organization with various tools that integrate with gRPC
     

     View Slide

117. Thank you!
     @kwuchu
     

     View Slide