$30 off During Our Annual Pro Sale. View Details »

Writing extension libraries in Go

NARUSE, Yui
November 08, 2015

Writing extension libraries in Go

NARUSE, Yui

November 08, 2015
Tweet

More Decks by NARUSE, Yui

Other Decks in Technology

Transcript

  1. Writing extension libraries in Go
    ੒੉Ώ͍ !OBMTI

    Treasure Data

    View Slide

  2. http://qiita.com/grj_achm/items/679b3f3af2cf377f0f02

    View Slide

  3. Go 1.5 supports to generate C shared libraries
    (With Go 1.5, it only works on linux/{arm,amd64}, android/arm)

    View Slide

  4. Writing Extension library in Go

    View Slide

  5. C extension for Ruby
    #include "ruby/ruby.h"
    static VALUE
    ary_resize(VALUE ary, VALUE len)
    {
    rb_ary_resize(ary, NUM2LONG(len));
    return ary;
    }
    void
    Init_resize(void)
    {
    rb_define_method(rb_cArray, "__resize__", ary_resize, 1);
    }
    ϝιουΛCͰ࣮૷ͯ͠
    Rubyʹఆٛ͢Δ

    View Slide

  6. % ls
    hello.go
    % cat hello.go
    package main
    // #cgo LDFLAGS: -Wl,--unresolved-symbols=ignore-all
    // #include
    // VALUE go_hello(VALUE);
    import "C"
    import "fmt"
    import "unsafe"
    func main() {}
    //export go_hello
    func go_hello(C.VALUE) C.VALUE {
    fmt.Println("Hello, Go world!")
    return C.Qnil
    }
    //export Init_hello
    func Init_hello() {
    cname := C.CString("Hello")
    cHello := C.rb_define_class(cname, C.rb_cData)
    C.free(unsafe.Pointer(cname))
    cname = C.CString("hello")
    C.rb_define_singleton_method(cHello, cname, (*[0]byte)(C.go_hello), 0)
    C.free(unsafe.Pointer(cname))
    }
    % CGO_CFLAGS=`ruby -e'$><<"-I#{RbConfig::CONFIG["rubyarchhdrdir"]} -I#{RbConfig::CONFIG["rubyhdrdir"]}"'`
    % go build -buildmode=c-shared -o hello.so hello.go
    % ruby -r./hello.so -eHello.hello
    Hello, Go world!
    Ҏ߱ɺ෦඼ͷղઆΛ͠·͢
    Goͷ৔߹

    View Slide

  7. Cgo - package to call C code
    C and Go Types
    char --> C.char --> byte
    signed char --> C.schar --> int8
    unsigned char --> C.uchar --> uint8
    int --> C.int --> int
    unsigned int --> C.uint --> uint32
    long int --> C.long --> int32 or int64
    long unsigned int --> C.ulong --> uint32 or uint64
    double --> C.double --> float64

    View Slide

  8. Cgo - package to call C code
    Types
    // convert Go integer to Ruby Numeric
    //
    // C.long(): Go int -> C long (cgo)
    // rb_int2inum(): C long -> Ruby Numeric (C function)
    // From Go, it can use C Macro (needs wrapper)
    func INT2NUM(n int) C.VALUE {
    return C.rb_int2inum(C.long(n))
    }

    View Slide

  9. Cgo and Strings
    func C.CString(goString string) *C.char
    func C.GoString(cString *C.char) string
    func C.GoStringN(cString *C.char, length C.int) string
    var cmsg *C.char = C.CString("hi")
    defer C.free(unsafe.Pointer(cmsg))
    ※ C.CString() allocates memory and copy the string.

    You must call C.free().

    View Slide

  10. Ruby strings to Go Strings
    Cgo can’t use C Macro; Write wrapper in C as Go comments
    package main
    /*
    const char * rstring_ptr(VALUE str) {
    return RSTRING_PTR(str);
    }
    int rstring_len(VALUE str) {
    return RSTRING_LENINT(str);
    }
    */
    import "C"
    func RbGoString(str C.VALUE) string {
    return C.GoStringN(C.rstring_ptr(str), C.rstring_len(str))
    }

    View Slide

  11. Go strings to Ruby Strings
    Ago can’t use C Macro; Write wrapper in C as Go comments
    package main
    func RbString(str string) C.VALUE {
    cstr := C.CString(str)
    defer C.free(cstr)
    return C.rb_utf8_str_new(cstr, C.long(len(str)))
    }
    Allocation and Copy!

    View Slide

  12. Go strings to Ruby Strings II
    Use unsafe.Pointer and casts to avoid copy!!!!
    func RbString(str string) C.VALUE {
    if len(str) == 0 {
    return C.rb_utf8_str_new(nil, C.long(0))
    }
    cstr := (*C.char)(unsafe.Pointer(&(*(*[]byte)(unsafe.Pointer(&str)))[0]))
    return C.rb_utf8_str_new(cstr, C.long(len(str)))
    }
    see also http://qiita.com/mattn/items/176459728ff4f854b165

    View Slide

  13. 000000000022f3f0 :
    22f3f0: 48 8b 0d 91 0b 66 00 mov 0x660b91(%rip),%rcx # 88ff88 <_DYNAMIC+0x200>
    22f3f7: 64 48 8b 09 mov %fs:(%rcx),%rcx
    22f3fb: 48 3b 61 10 cmp 0x10(%rcx),%rsp
    22f3ff: 0f 86 ba 00 00 00 jbe 22f4bf
    22f405: 48 83 ec 20 sub $0x20,%rsp
    22f409: 48 8b 4c 24 30 mov 0x30(%rsp),%rcx
    22f40e: 48 c7 44 24 38 00 00 movq $0x0,0x38(%rsp)
    22f415: 00 00
    22f417: 48 83 f9 00 cmp $0x0,%rcx
    22f41b: 75 2b jne 22f448
    22f41d: 48 c7 04 24 00 00 00 movq $0x0,(%rsp)
    22f424: 00
    22f425: 48 c7 44 24 08 00 00 movq $0x0,0x8(%rsp)
    22f42c: 00 00
    22f42e: e8 1d cb ff ff callq 22bf50
    22f433: 48 8b 5c 24 10 mov 0x10(%rsp),%rbx
    22f438: 48 89 5c 24 38 mov %rbx,0x38(%rsp)
    22f43d: 90 nop
    22f43e: e8 7d 85 02 00 callq 2579c0
    22f443: 48 83 c4 20 add $0x20,%rsp
    22f447: c3 retq
    22f448: 48 8b 5c 24 28 mov 0x28(%rsp),%rbx
    22f44d: 48 89 1c 24 mov %rbx,(%rsp)
    22f451: 48 89 4c 24 08 mov %rcx,0x8(%rsp)
    22f456: e8 65 bb ff ff callq 22afc0
    22f45b: 48 8b 44 24 10 mov 0x10(%rsp),%rax
    22f460: 48 89 44 24 18 mov %rax,0x18(%rsp)
    22f465: 48 89 44 24 10 mov %rax,0x10(%rsp)
    22f46a: c7 04 24 08 00 00 00 movl $0x8,(%rsp)
    22f471: 48 8d 05 40 4c 38 00 lea 0x384c40(%rip),%rax # 5b40b8
    22f478: 48 89 44 24 08 mov %rax,0x8(%rsp)
    22f47d: e8 be 73 02 00 callq 256840
    22f482: 83 f8 00 cmp $0x0,%eax
    22f485: 75 2d jne 22f4b4
    22f487: 48 8b 44 24 30 mov 0x30(%rsp),%rax
    22f48c: 48 8b 5c 24 18 mov 0x18(%rsp),%rbx
    22f491: 48 89 1c 24 mov %rbx,(%rsp)
    22f495: 48 89 44 24 08 mov %rax,0x8(%rsp)
    22f49a: e8 b1 ca ff ff callq 22bf50
    22f49f: 48 8b 5c 24 10 mov 0x10(%rsp),%rbx
    22f4a4: 48 89 5c 24 38 mov %rbx,0x38(%rsp)
    22f4a9: 90 nop
    22f4aa: e8 11 85 02 00 callq 2579c0
    22f4af: 48 83 c4 20 add $0x20,%rsp
    22f4b3: c3 retq
    22f4b4: 90 nop
    22f4b5: e8 06 85 02 00 callq 2579c0
    22f4ba: 48 83 c4 20 add $0x20,%rsp
    22f4be: c3 retq
    22f4bf: e8 4c 84 05 00 callq 287910
    22f4c4: e9 27 ff ff ff jmpq 22f3f0
    C.CString()
    BEFORE

    View Slide

  14. 000000000022afc0 :
    22afc0: 48 8b 0d c1 4f 66 00 mov 0x664fc1(%rip),%rcx # 88ff88 <_DYNAMIC+0x200>
    22afc7: 64 48 8b 09 mov %fs:(%rcx),%rcx
    22afcb: 48 3b 61 10 cmp 0x10(%rcx),%rsp
    22afcf: 0f 86 cc 00 00 00 jbe 22b0a1
    22afd5: 48 83 ec 68 sub $0x68,%rsp
    22afd9: 48 8b 5c 24 78 mov 0x78(%rsp),%rbx
    22afde: 48 ff c3 inc %rbx
    22afe1: 48 89 1c 24 mov %rbx,(%rsp)
    22afe5: e8 76 62 00 00 callq 231260
    22afea: 48 8b 74 24 08 mov 0x8(%rsp),%rsi
    22afef: 48 89 74 24 20 mov %rsi,0x20(%rsp)
    22aff4: 48 89 74 24 18 mov %rsi,0x18(%rsp)
    22aff9: 48 83 fe 00 cmp $0x0,%rsi
    22affd: 0f 84 97 00 00 00 je 22b09a
    22b003: 48 c7 c0 00 00 00 40 mov $0x40000000,%rax
    22b00a: 48 c7 c1 00 00 00 40 mov $0x40000000,%rcx
    22b011: 48 89 74 24 50 mov %rsi,0x50(%rsp)
    22b016: 48 89 44 24 58 mov %rax,0x58(%rsp)
    22b01b: 48 89 4c 24 60 mov %rcx,0x60(%rsp)
    22b020: 48 89 4c 24 48 mov %rcx,0x48(%rsp)
    22b025: 48 8b 54 24 70 mov 0x70(%rsp),%rdx
    22b02a: 48 8b 4c 24 78 mov 0x78(%rsp),%rcx
    22b02f: 48 89 44 24 40 mov %rax,0x40(%rsp)
    22b034: 48 89 4c 24 30 mov %rcx,0x30(%rsp)
    22b039: 48 39 c1 cmp %rax,%rcx
    22b03c: 7d 03 jge 22b041
    22b03e: 48 89 c8 mov %rcx,%rax
    22b041: 48 89 74 24 38 mov %rsi,0x38(%rsp)
    22b046: 48 89 34 24 mov %rsi,(%rsp)
    22b04a: 48 89 54 24 28 mov %rdx,0x28(%rsp)
    22b04f: 48 89 54 24 08 mov %rdx,0x8(%rsp)
    22b054: 48 89 44 24 10 mov %rax,0x10(%rsp)
    22b059: e8 e2 f9 05 00 callq 28aa40
    22b05e: 48 8b 44 24 78 mov 0x78(%rsp),%rax
    22b063: 48 8b 5c 24 18 mov 0x18(%rsp),%rbx
    22b068: 48 83 fb 00 cmp $0x0,%rbx
    22b06c: 74 28 je 22b096
    22b06e: 48 3d 00 00 00 40 cmp $0x40000000,%rax
    22b074: 73 19 jae 22b08f
    22b076: 48 8d 1c 03 lea (%rbx,%rax,1),%rbx
    22b07a: c6 03 00 movb $0x0,(%rbx)
    22b07d: 48 8b 5c 24 20 mov 0x20(%rsp),%rbx
    22b082: 48 89 9c 24 80 00 00 mov %rbx,0x80(%rsp)
    22b089: 00
    22b08a: 48 83 c4 68 add $0x68,%rsp
    22b08e: c3 retq
    22b08f: e8 2c b5 02 00 callq 2565c0
    22b094: 0f 0b ud2
    22b096: 89 03 mov %eax,(%rbx)
    22b098: eb d4 jmp 22b06e
    22b09a: 89 06 mov %eax,(%rsi)
    22b09c: e9 62 ff ff ff jmpq 22b003
    22b0a1: e8 6a c8 05 00 callq 287910
    22b0a6: e9 15 ff ff ff jmpq 22afc0
    memmove(3): copy C strings
    BEFORE

    View Slide

  15. 000000000022f160 :
    22f160: 48 8b 0d 21 0e 66 00 mov 0x660e21(%rip),%rcx # 88ff88 <_DYNAMIC+0x200>
    22f167: 64 48 8b 09 mov %fs:(%rcx),%rcx
    22f16b: 48 3b 61 10 cmp 0x10(%rcx),%rsp
    22f16f: 0f 86 8f 00 00 00 jbe 22f204
    22f175: 48 83 ec 30 sub $0x30,%rsp
    22f179: 48 8b 5c 24 40 mov 0x40(%rsp),%rbx
    22f17e: 48 83 fb 00 cmp $0x0,%rbx
    22f182: 75 25 jne 22f1a9
    22f184: 48 c7 04 24 00 00 00 movq $0x0,(%rsp)
    22f18b: 00
    22f18c: 48 c7 44 24 08 00 00 movq $0x0,0x8(%rsp)
    22f193: 00 00
    22f195: e8 26 cb ff ff callq 22bcc0
    22f19a: 48 8b 5c 24 10 mov 0x10(%rsp),%rbx
    22f19f: 48 89 5c 24 48 mov %rbx,0x48(%rsp)
    22f1a4: 48 83 c4 30 add $0x30,%rsp
    22f1a8: c3 retq
    22f1a9: 48 8d 5c 24 38 lea 0x38(%rsp),%rbx
    22f1ae: 48 83 fb 00 cmp $0x0,%rbx
    22f1b2: 74 4c je 22f200
    22f1b4: 48 8b 0b mov (%rbx),%rcx
    22f1b7: 48 8b 43 08 mov 0x8(%rbx),%rax
    22f1bb: 48 8b 6b 10 mov 0x10(%rbx),%rbp
    22f1bf: 48 89 6c 24 28 mov %rbp,0x28(%rsp)
    22f1c4: 48 89 4c 24 18 mov %rcx,0x18(%rsp)
    22f1c9: 48 83 f8 00 cmp $0x0,%rax
    22f1cd: 48 89 44 24 20 mov %rax,0x20(%rsp)
    22f1d2: 76 25 jbe 22f1f9
    22f1d4: 48 89 c8 mov %rcx,%rax
    22f1d7: 48 8b 4c 24 40 mov 0x40(%rsp),%rcx
    22f1dc: 48 89 04 24 mov %rax,(%rsp)
    22f1e0: 48 89 4c 24 08 mov %rcx,0x8(%rsp)
    22f1e5: e8 d6 ca ff ff callq 22bcc0
    22f1ea: 48 8b 5c 24 10 mov 0x10(%rsp),%rbx
    22f1ef: 48 89 5c 24 48 mov %rbx,0x48(%rsp)
    22f1f4: 48 83 c4 30 add $0x30,%rsp
    22f1f8: c3 retq
    22f1f9: e8 02 71 02 00 callq 256300
    22f1fe: 0f 0b ud2
    22f200: 89 03 mov %eax,(%rbx)
    22f202: eb b0 jmp 22f1b4
    22f204: e8 47 84 05 00 callq 287650
    22f209: e9 52 ff ff ff jmpq 22f160
    zero copy!
    AFTER
    Note that rb_str_new() copies the content

    View Slide

  16. Other wrappers
    package main
    // #include
    import "C"
    import "unsafe"
    func rb_define_method(klass C.VALUE, name string, fun unsafe.Pointer, args int) {
    cname := (*C.char)(unsafe.Pointer(&(*(*[]byte)(unsafe.Pointer(&name)))[0]))
    C.define_method(klass, cname, (*[0]byte)(fun), C.int(args))
    }
    func rb_define_singleton_method(klass C.VALUE, name string, fun unsafe.Pointer,
    args int) {
    cname := (*C.char)(unsafe.Pointer(&(*(*[]byte)(unsafe.Pointer(&name)))[0]))
    C.define_singleton_method(klass, cname, (*[0]byte)(fun), C.int(args))
    }

    View Slide

  17. Class definition and Method declarations
    //export Init_gohttp
    func Init_gohttp() {
    sNew := "new"
    str_new := (*C.char)(unsafe.Pointer(&(*(*[]byte)(unsafe.Pointer(&sNew)))[0]))
    rb_cRequest = rb_define_class("Request", C.rb_cObject)
    C.rb_undef_alloc_func(rb_cRequest)
    C.rb_undef_method(C.rb_class_of(rb_cRequest), str_new)
    rb_define_method(rb_cRequest, "method", C.reqMethod, 0)
    //rb_define_method(rb_cRequest, "url", C.reqURL, 0)
    rb_define_method(rb_cRequest, "proto", C.reqProto, 0)
    rb_define_method(rb_cRequest, "header", C.reqHeader, 0)
    rb_define_method(rb_cRequest, "body", C.reqBody, 0)
    rb_define_method(rb_cRequest, "content_length", C.reqContentLength, 0)
    rb_define_method(rb_cRequest, "transfer_encoding", C.reqTransferEncoding, 0)
    rb_define_method(rb_cRequest, "host", C.reqHost, 0)
    //rb_define_method(rb_cRequest, "form", C.reqForm, 0)
    //rb_define_method(rb_cRequest, "post_form", C.reqForm, 0)
    //rb_define_method(rb_cRequest, "multipart_form", C.reqForm, 0)
    //rb_define_method(rb_cRequest, "trailer", C.reqForm, 0)
    rb_define_method(rb_cRequest, "remote_addr", C.reqRemoteAddr, 0)
    rb_define_method(rb_cRequest, "request_uri", C.reqRequestURI, 0)

    View Slide

  18. Generic Go object definition for Ruby
    /*
    void goobj_retain(void *);
    void goobj_free(void *);
    static const rb_data_type_t go_type = {
    "GoStruct",
    {NULL, goobj_free, NULL},
    0, 0, RUBY_TYPED_FREE_IMMEDIATELY|RUBY_TYPED_WB_PROTECTED
    };
    */
    VALUE
    NewGoStruct(VALUE klass, void *p)
    {
    goobj_retain(p);
    return TypedData_Wrap_Struct((klass), &go_type, p);
    }
    void *
    GetGoStruct(VALUE obj)
    {
    void *val;
    return TypedData_Get_Struct((obj), void *, &go_type, (val));
    }

    View Slide

  19. Avoid Go’s GC
    var objects = make(map[interface{}]bool)
    //export goobj_retain
    func goobj_retain(obj unsafe.Pointer) {
    objects[obj] = true
    }
    //export goobj_free
    func goobj_free(obj unsafe.Pointer) {
    delete(objects, obj)
    }

    View Slide

  20. Method definitions
    //export gohttpGet
    func gohttpGet(dummy C.VALUE, urlstr C.VALUE) C.VALUE {
    str := RbGoString(urlstr)
    resp, err := http.Get(str)
    if err != nil {
    rb_raise(C.rb_eArgError, "'%s'", err)
    }
    return respNew(rb_cResponse, resp)
    }
    func respNew(klass C.VALUE, r *http.Response) C.VALUE {
    return C.NewGoStruct(klass, unsafe.Pointer(r))
    }
    //export respContentLength
    func respContentLength(self C.VALUE) C.VALUE {
    resp := (*http.Response)(C.GetGoStruct(self))
    return INT64toNUM(resp.ContentLength)
    }

    View Slide

  21. Now you can call
    Go’s net/http from Ruby
    def test_get
    r = Gohttp.get('https://example.com/index.html')
    assert_kind_of Response, r
    assert_equal '200 OK', r.status
    assert_equal 200, r.status_code
    assert_equal 'HTTP/1.1', r.proto
    assert_equal 1270, r.size
    assert_equal [], r.transfer_encoding
    end

    View Slide

  22. ·ͱΊ
    GoͰRuby༻֦ுϥΠϒϥϦΛॻ͚·͢
    C֦ுϥΠϒϥϦͷ஌͕ࣝඞཁͰ͢
    CͷϚΫϩ͸࢖͑ͳ͍ͷͰϥούʔ͕ඞཁͰ͢
    ୭͔ͱΓ·ͱΊͯཉ͍͠
    ͍͔ͭ͘ಛ༗ͷςΫχοΫ͕ඞཁͰ͢
    جຊతͳ͜ͱ͸͜ͷൃදͰΧόʔͨ͠͸ͣͰ͢
    ΈΜͳͰ஌ݟΛͨΊͯɺGoogleͷ੒Ռʹ৐͔ͬΖ͏ʂʂʂ

    View Slide