Writing extension libraries in Go

See also https://github.com/nurse/gohttp

Transcript

1. Writing extension libraries in Go
   ੒੉Ώ͍ !OBMTI
   
   Treasure Data
   

   View Slide

2. http://qiita.com/grj_achm/items/679b3f3af2cf377f0f02
   

   View Slide

3. Go 1.5 supports to generate C shared libraries
   (With Go 1.5, it only works on linux/{arm,amd64}, android/arm)
   

   View Slide

4. Writing Extension library in Go
   

   View Slide

5. C extension for Ruby
   #include "ruby/ruby.h"
   static VALUE
   ary_resize(VALUE ary, VALUE len)
   {
   rb_ary_resize(ary, NUM2LONG(len));
   return ary;
   }
   void
   Init_resize(void)
   {
   rb_define_method(rb_cArray, "__resize__", ary_resize, 1);
   }
   ϝιουΛCͰ࣮૷ͯ͠
   Rubyʹఆٛ͢Δ
   

   View Slide

6. % ls
   hello.go
   % cat hello.go
   package main
   // #cgo LDFLAGS: -Wl,--unresolved-symbols=ignore-all
   // #include
   // VALUE go_hello(VALUE);
   import "C"
   import "fmt"
   import "unsafe"
   func main() {}
   //export go_hello
   func go_hello(C.VALUE) C.VALUE {
   fmt.Println("Hello, Go world!")
   return C.Qnil
   }
   //export Init_hello
   func Init_hello() {
   cname := C.CString("Hello")
   cHello := C.rb_define_class(cname, C.rb_cData)
   C.free(unsafe.Pointer(cname))
   cname = C.CString("hello")
   C.rb_define_singleton_method(cHello, cname, (*[0]byte)(C.go_hello), 0)
   C.free(unsafe.Pointer(cname))
   }
   % CGO_CFLAGS=`ruby -e'$><<"-I#{RbConfig::CONFIG["rubyarchhdrdir"]} -I#{RbConfig::CONFIG["rubyhdrdir"]}"'`
   % go build -buildmode=c-shared -o hello.so hello.go
   % ruby -r./hello.so -eHello.hello
   Hello, Go world!
   Ҏ߱ɺ෦඼ͷղઆΛ͠·͢
   Goͷ৔߹
   

   View Slide

7. Cgo - package to call C code
   C and Go Types
   char --> C.char --> byte
   signed char --> C.schar --> int8
   unsigned char --> C.uchar --> uint8
   int --> C.int --> int
   unsigned int --> C.uint --> uint32
   long int --> C.long --> int32 or int64
   long unsigned int --> C.ulong --> uint32 or uint64
   double --> C.double --> float64
   

   View Slide

8. Cgo - package to call C code
   Types
   // convert Go integer to Ruby Numeric
   //
   // C.long(): Go int -> C long (cgo)
   // rb_int2inum(): C long -> Ruby Numeric (C function)
   // From Go, it can use C Macro (needs wrapper)
   func INT2NUM(n int) C.VALUE {
   return C.rb_int2inum(C.long(n))
   }
   

   View Slide

9. Cgo and Strings
   func C.CString(goString string) *C.char
   func C.GoString(cString *C.char) string
   func C.GoStringN(cString *C.char, length C.int) string
   var cmsg *C.char = C.CString("hi")
   defer C.free(unsafe.Pointer(cmsg))
   ※ C.CString() allocates memory and copy the string.

   You must call C.free().
   

   View Slide

10. Ruby strings to Go Strings
    Cgo can’t use C Macro; Write wrapper in C as Go comments
    package main
    /*
    const char * rstring_ptr(VALUE str) {
    return RSTRING_PTR(str);
    }
    int rstring_len(VALUE str) {
    return RSTRING_LENINT(str);
    }
    */
    import "C"
    func RbGoString(str C.VALUE) string {
    return C.GoStringN(C.rstring_ptr(str), C.rstring_len(str))
    }
    

    View Slide

11. Go strings to Ruby Strings
    Ago can’t use C Macro; Write wrapper in C as Go comments
    package main
    func RbString(str string) C.VALUE {
    cstr := C.CString(str)
    defer C.free(cstr)
    return C.rb_utf8_str_new(cstr, C.long(len(str)))
    }
    Allocation and Copy!
    

    View Slide

12. Go strings to Ruby Strings II
    Use unsafe.Pointer and casts to avoid copy!!!!
    func RbString(str string) C.VALUE {
    if len(str) == 0 {
    return C.rb_utf8_str_new(nil, C.long(0))
    }
    cstr := (*C.char)(unsafe.Pointer(&(*(*[]byte)(unsafe.Pointer(&str)))[0]))
    return C.rb_utf8_str_new(cstr, C.long(len(str)))
    }
    see also http://qiita.com/mattn/items/176459728ff4f854b165
    

    View Slide

13. 000000000022f3f0 :
    22f3f0: 48 8b 0d 91 0b 66 00 mov 0x660b91(%rip),%rcx # 88ff88 <_DYNAMIC+0x200>
    22f3f7: 64 48 8b 09 mov %fs:(%rcx),%rcx
    22f3fb: 48 3b 61 10 cmp 0x10(%rcx),%rsp
    22f3ff: 0f 86 ba 00 00 00 jbe 22f4bf
    22f405: 48 83 ec 20 sub $0x20,%rsp
    22f409: 48 8b 4c 24 30 mov 0x30(%rsp),%rcx
    22f40e: 48 c7 44 24 38 00 00 movq $0x0,0x38(%rsp)
    22f415: 00 00
    22f417: 48 83 f9 00 cmp $0x0,%rcx
    22f41b: 75 2b jne 22f448
    22f41d: 48 c7 04 24 00 00 00 movq $0x0,(%rsp)
    22f424: 00
    22f425: 48 c7 44 24 08 00 00 movq $0x0,0x8(%rsp)
    22f42c: 00 00
    22f42e: e8 1d cb ff ff callq 22bf50
    22f433: 48 8b 5c 24 10 mov 0x10(%rsp),%rbx
    22f438: 48 89 5c 24 38 mov %rbx,0x38(%rsp)
    22f43d: 90 nop
    22f43e: e8 7d 85 02 00 callq 2579c0
    22f443: 48 83 c4 20 add $0x20,%rsp
    22f447: c3 retq
    22f448: 48 8b 5c 24 28 mov 0x28(%rsp),%rbx
    22f44d: 48 89 1c 24 mov %rbx,(%rsp)
    22f451: 48 89 4c 24 08 mov %rcx,0x8(%rsp)
    22f456: e8 65 bb ff ff callq 22afc0
    22f45b: 48 8b 44 24 10 mov 0x10(%rsp),%rax
    22f460: 48 89 44 24 18 mov %rax,0x18(%rsp)
    22f465: 48 89 44 24 10 mov %rax,0x10(%rsp)
    22f46a: c7 04 24 08 00 00 00 movl $0x8,(%rsp)
    22f471: 48 8d 05 40 4c 38 00 lea 0x384c40(%rip),%rax # 5b40b8
    22f478: 48 89 44 24 08 mov %rax,0x8(%rsp)
    22f47d: e8 be 73 02 00 callq 256840
    22f482: 83 f8 00 cmp $0x0,%eax
    22f485: 75 2d jne 22f4b4
    22f487: 48 8b 44 24 30 mov 0x30(%rsp),%rax
    22f48c: 48 8b 5c 24 18 mov 0x18(%rsp),%rbx
    22f491: 48 89 1c 24 mov %rbx,(%rsp)
    22f495: 48 89 44 24 08 mov %rax,0x8(%rsp)
    22f49a: e8 b1 ca ff ff callq 22bf50
    22f49f: 48 8b 5c 24 10 mov 0x10(%rsp),%rbx
    22f4a4: 48 89 5c 24 38 mov %rbx,0x38(%rsp)
    22f4a9: 90 nop
    22f4aa: e8 11 85 02 00 callq 2579c0
    22f4af: 48 83 c4 20 add $0x20,%rsp
    22f4b3: c3 retq
    22f4b4: 90 nop
    22f4b5: e8 06 85 02 00 callq 2579c0
    22f4ba: 48 83 c4 20 add $0x20,%rsp
    22f4be: c3 retq
    22f4bf: e8 4c 84 05 00 callq 287910
    22f4c4: e9 27 ff ff ff jmpq 22f3f0
    C.CString()
    BEFORE
    

    View Slide

14. 000000000022afc0 :
    22afc0: 48 8b 0d c1 4f 66 00 mov 0x664fc1(%rip),%rcx # 88ff88 <_DYNAMIC+0x200>
    22afc7: 64 48 8b 09 mov %fs:(%rcx),%rcx
    22afcb: 48 3b 61 10 cmp 0x10(%rcx),%rsp
    22afcf: 0f 86 cc 00 00 00 jbe 22b0a1
    22afd5: 48 83 ec 68 sub $0x68,%rsp
    22afd9: 48 8b 5c 24 78 mov 0x78(%rsp),%rbx
    22afde: 48 ff c3 inc %rbx
    22afe1: 48 89 1c 24 mov %rbx,(%rsp)
    22afe5: e8 76 62 00 00 callq 231260
    22afea: 48 8b 74 24 08 mov 0x8(%rsp),%rsi
    22afef: 48 89 74 24 20 mov %rsi,0x20(%rsp)
    22aff4: 48 89 74 24 18 mov %rsi,0x18(%rsp)
    22aff9: 48 83 fe 00 cmp $0x0,%rsi
    22affd: 0f 84 97 00 00 00 je 22b09a
    22b003: 48 c7 c0 00 00 00 40 mov $0x40000000,%rax
    22b00a: 48 c7 c1 00 00 00 40 mov $0x40000000,%rcx
    22b011: 48 89 74 24 50 mov %rsi,0x50(%rsp)
    22b016: 48 89 44 24 58 mov %rax,0x58(%rsp)
    22b01b: 48 89 4c 24 60 mov %rcx,0x60(%rsp)
    22b020: 48 89 4c 24 48 mov %rcx,0x48(%rsp)
    22b025: 48 8b 54 24 70 mov 0x70(%rsp),%rdx
    22b02a: 48 8b 4c 24 78 mov 0x78(%rsp),%rcx
    22b02f: 48 89 44 24 40 mov %rax,0x40(%rsp)
    22b034: 48 89 4c 24 30 mov %rcx,0x30(%rsp)
    22b039: 48 39 c1 cmp %rax,%rcx
    22b03c: 7d 03 jge 22b041
    22b03e: 48 89 c8 mov %rcx,%rax
    22b041: 48 89 74 24 38 mov %rsi,0x38(%rsp)
    22b046: 48 89 34 24 mov %rsi,(%rsp)
    22b04a: 48 89 54 24 28 mov %rdx,0x28(%rsp)
    22b04f: 48 89 54 24 08 mov %rdx,0x8(%rsp)
    22b054: 48 89 44 24 10 mov %rax,0x10(%rsp)
    22b059: e8 e2 f9 05 00 callq 28aa40
    22b05e: 48 8b 44 24 78 mov 0x78(%rsp),%rax
    22b063: 48 8b 5c 24 18 mov 0x18(%rsp),%rbx
    22b068: 48 83 fb 00 cmp $0x0,%rbx
    22b06c: 74 28 je 22b096
    22b06e: 48 3d 00 00 00 40 cmp $0x40000000,%rax
    22b074: 73 19 jae 22b08f
    22b076: 48 8d 1c 03 lea (%rbx,%rax,1),%rbx
    22b07a: c6 03 00 movb $0x0,(%rbx)
    22b07d: 48 8b 5c 24 20 mov 0x20(%rsp),%rbx
    22b082: 48 89 9c 24 80 00 00 mov %rbx,0x80(%rsp)
    22b089: 00
    22b08a: 48 83 c4 68 add $0x68,%rsp
    22b08e: c3 retq
    22b08f: e8 2c b5 02 00 callq 2565c0
    22b094: 0f 0b ud2
    22b096: 89 03 mov %eax,(%rbx)
    22b098: eb d4 jmp 22b06e
    22b09a: 89 06 mov %eax,(%rsi)
    22b09c: e9 62 ff ff ff jmpq 22b003
    22b0a1: e8 6a c8 05 00 callq 287910
    22b0a6: e9 15 ff ff ff jmpq 22afc0
    memmove(3): copy C strings
    BEFORE
    

    View Slide

15. 000000000022f160 :
    22f160: 48 8b 0d 21 0e 66 00 mov 0x660e21(%rip),%rcx # 88ff88 <_DYNAMIC+0x200>
    22f167: 64 48 8b 09 mov %fs:(%rcx),%rcx
    22f16b: 48 3b 61 10 cmp 0x10(%rcx),%rsp
    22f16f: 0f 86 8f 00 00 00 jbe 22f204
    22f175: 48 83 ec 30 sub $0x30,%rsp
    22f179: 48 8b 5c 24 40 mov 0x40(%rsp),%rbx
    22f17e: 48 83 fb 00 cmp $0x0,%rbx
    22f182: 75 25 jne 22f1a9
    22f184: 48 c7 04 24 00 00 00 movq $0x0,(%rsp)
    22f18b: 00
    22f18c: 48 c7 44 24 08 00 00 movq $0x0,0x8(%rsp)
    22f193: 00 00
    22f195: e8 26 cb ff ff callq 22bcc0
    22f19a: 48 8b 5c 24 10 mov 0x10(%rsp),%rbx
    22f19f: 48 89 5c 24 48 mov %rbx,0x48(%rsp)
    22f1a4: 48 83 c4 30 add $0x30,%rsp
    22f1a8: c3 retq
    22f1a9: 48 8d 5c 24 38 lea 0x38(%rsp),%rbx
    22f1ae: 48 83 fb 00 cmp $0x0,%rbx
    22f1b2: 74 4c je 22f200
    22f1b4: 48 8b 0b mov (%rbx),%rcx
    22f1b7: 48 8b 43 08 mov 0x8(%rbx),%rax
    22f1bb: 48 8b 6b 10 mov 0x10(%rbx),%rbp
    22f1bf: 48 89 6c 24 28 mov %rbp,0x28(%rsp)
    22f1c4: 48 89 4c 24 18 mov %rcx,0x18(%rsp)
    22f1c9: 48 83 f8 00 cmp $0x0,%rax
    22f1cd: 48 89 44 24 20 mov %rax,0x20(%rsp)
    22f1d2: 76 25 jbe 22f1f9
    22f1d4: 48 89 c8 mov %rcx,%rax
    22f1d7: 48 8b 4c 24 40 mov 0x40(%rsp),%rcx
    22f1dc: 48 89 04 24 mov %rax,(%rsp)
    22f1e0: 48 89 4c 24 08 mov %rcx,0x8(%rsp)
    22f1e5: e8 d6 ca ff ff callq 22bcc0
    22f1ea: 48 8b 5c 24 10 mov 0x10(%rsp),%rbx
    22f1ef: 48 89 5c 24 48 mov %rbx,0x48(%rsp)
    22f1f4: 48 83 c4 30 add $0x30,%rsp
    22f1f8: c3 retq
    22f1f9: e8 02 71 02 00 callq 256300
    22f1fe: 0f 0b ud2
    22f200: 89 03 mov %eax,(%rbx)
    22f202: eb b0 jmp 22f1b4
    22f204: e8 47 84 05 00 callq 287650
    22f209: e9 52 ff ff ff jmpq 22f160
    zero copy!
    AFTER
    Note that rb_str_new() copies the content
    

    View Slide

16. Other wrappers
    package main
    // #include
    import "C"
    import "unsafe"
    func rb_define_method(klass C.VALUE, name string, fun unsafe.Pointer, args int) {
    cname := (*C.char)(unsafe.Pointer(&(*(*[]byte)(unsafe.Pointer(&name)))[0]))
    C.define_method(klass, cname, (*[0]byte)(fun), C.int(args))
    }
    func rb_define_singleton_method(klass C.VALUE, name string, fun unsafe.Pointer,
    args int) {
    cname := (*C.char)(unsafe.Pointer(&(*(*[]byte)(unsafe.Pointer(&name)))[0]))
    C.define_singleton_method(klass, cname, (*[0]byte)(fun), C.int(args))
    }
    

    View Slide

17. Class definition and Method declarations
    //export Init_gohttp
    func Init_gohttp() {
    sNew := "new"
    str_new := (*C.char)(unsafe.Pointer(&(*(*[]byte)(unsafe.Pointer(&sNew)))[0]))
    rb_cRequest = rb_define_class("Request", C.rb_cObject)
    C.rb_undef_alloc_func(rb_cRequest)
    C.rb_undef_method(C.rb_class_of(rb_cRequest), str_new)
    rb_define_method(rb_cRequest, "method", C.reqMethod, 0)
    //rb_define_method(rb_cRequest, "url", C.reqURL, 0)
    rb_define_method(rb_cRequest, "proto", C.reqProto, 0)
    rb_define_method(rb_cRequest, "header", C.reqHeader, 0)
    rb_define_method(rb_cRequest, "body", C.reqBody, 0)
    rb_define_method(rb_cRequest, "content_length", C.reqContentLength, 0)
    rb_define_method(rb_cRequest, "transfer_encoding", C.reqTransferEncoding, 0)
    rb_define_method(rb_cRequest, "host", C.reqHost, 0)
    //rb_define_method(rb_cRequest, "form", C.reqForm, 0)
    //rb_define_method(rb_cRequest, "post_form", C.reqForm, 0)
    //rb_define_method(rb_cRequest, "multipart_form", C.reqForm, 0)
    //rb_define_method(rb_cRequest, "trailer", C.reqForm, 0)
    rb_define_method(rb_cRequest, "remote_addr", C.reqRemoteAddr, 0)
    rb_define_method(rb_cRequest, "request_uri", C.reqRequestURI, 0)
    

    View Slide

18. Generic Go object definition for Ruby
    /*
    void goobj_retain(void *);
    void goobj_free(void *);
    static const rb_data_type_t go_type = {
    "GoStruct",
    {NULL, goobj_free, NULL},
    0, 0, RUBY_TYPED_FREE_IMMEDIATELY|RUBY_TYPED_WB_PROTECTED
    };
    */
    VALUE
    NewGoStruct(VALUE klass, void *p)
    {
    goobj_retain(p);
    return TypedData_Wrap_Struct((klass), &go_type, p);
    }
    void *
    GetGoStruct(VALUE obj)
    {
    void *val;
    return TypedData_Get_Struct((obj), void *, &go_type, (val));
    }
    

    View Slide

19. Avoid Go’s GC
    var objects = make(map[interface{}]bool)
    //export goobj_retain
    func goobj_retain(obj unsafe.Pointer) {
    objects[obj] = true
    }
    //export goobj_free
    func goobj_free(obj unsafe.Pointer) {
    delete(objects, obj)
    }
    

    View Slide

20. Method definitions
    //export gohttpGet
    func gohttpGet(dummy C.VALUE, urlstr C.VALUE) C.VALUE {
    str := RbGoString(urlstr)
    resp, err := http.Get(str)
    if err != nil {
    rb_raise(C.rb_eArgError, "'%s'", err)
    }
    return respNew(rb_cResponse, resp)
    }
    func respNew(klass C.VALUE, r *http.Response) C.VALUE {
    return C.NewGoStruct(klass, unsafe.Pointer(r))
    }
    //export respContentLength
    func respContentLength(self C.VALUE) C.VALUE {
    resp := (*http.Response)(C.GetGoStruct(self))
    return INT64toNUM(resp.ContentLength)
    }
    

    View Slide

21. Now you can call
    Go’s net/http from Ruby
    def test_get
    r = Gohttp.get('https://example.com/index.html')
    assert_kind_of Response, r
    assert_equal '200 OK', r.status
    assert_equal 200, r.status_code
    assert_equal 'HTTP/1.1', r.proto
    assert_equal 1270, r.size
    assert_equal [], r.transfer_encoding
    end
    

    View Slide

22. ·ͱΊ
    GoͰRuby༻֦ுϥΠϒϥϦΛॻ͚·͢
    C֦ுϥΠϒϥϦͷ஌͕ࣝඞཁͰ͢
    CͷϚΫϩ͸࢖͑ͳ͍ͷͰϥούʔ͕ඞཁͰ͢
    ୭͔ͱΓ·ͱΊͯཉ͍͠
    ͍͔ͭ͘ಛ༗ͷςΫχοΫ͕ඞཁͰ͢
    جຊతͳ͜ͱ͸͜ͷൃදͰΧόʔͨ͠͸ͣͰ͢
    ΈΜͳͰ஌ݟΛͨΊͯɺGoogleͷ੒Ռʹ৐͔ͬΖ͏ʂʂʂ
    

    View Slide