Writing extension libraries in Go

Transcript

1. Writing extension libraries in Go ੒੉Ώ͍
   !OBMTI
   Treasure Data

2. http://qiita.com/grj_achm/items/679b3f3af2cf377f0f02

3. Go 1.5 supports to generate C shared libraries (With Go

   1.5, it only works on linux/{arm,amd64}, android/arm)

4. Writing Extension library in Go

5. C extension for Ruby #include "ruby/ruby.h" static VALUE ary_resize(VALUE ary,

   VALUE len) { rb_ary_resize(ary, NUM2LONG(len)); return ary; } void Init_resize(void) { rb_define_method(rb_cArray, "__resize__", ary_resize, 1); } ϝιουΛCͰ࣮૷ͯ͠ Rubyʹఆٛ͢Δ

6. % ls hello.go % cat hello.go package main // #cgo

   LDFLAGS: -Wl,--unresolved-symbols=ignore-all // #include <ruby/ruby.h> // VALUE go_hello(VALUE); import "C" import "fmt" import "unsafe" func main() {} //export go_hello func go_hello(C.VALUE) C.VALUE { fmt.Println("Hello, Go world!") return C.Qnil } //export Init_hello func Init_hello() { cname := C.CString("Hello") cHello := C.rb_define_class(cname, C.rb_cData) C.free(unsafe.Pointer(cname)) cname = C.CString("hello") C.rb_define_singleton_method(cHello, cname, (*[0]byte)(C.go_hello), 0) C.free(unsafe.Pointer(cname)) } % CGO_CFLAGS=`ruby -e'$><<"-I#{RbConfig::CONFIG["rubyarchhdrdir"]} -I#{RbConfig::CONFIG["rubyhdrdir"]}"'` % go build -buildmode=c-shared -o hello.so hello.go % ruby -r./hello.so -eHello.hello Hello, Go world! Ҏ߱ɺ෦඼ͷղઆΛ͠·͢ Goͷ৔߹

7. Cgo - package to call C code C and Go

   Types char --> C.char --> byte signed char --> C.schar --> int8 unsigned char --> C.uchar --> uint8 int --> C.int --> int unsigned int --> C.uint --> uint32 long int --> C.long --> int32 or int64 long unsigned int --> C.ulong --> uint32 or uint64 double --> C.double --> float64

8. Cgo - package to call C code Types // convert

   Go integer to Ruby Numeric // // C.long(): Go int -> C long (cgo) // rb_int2inum(): C long -> Ruby Numeric (C function) // From Go, it can use C Macro (needs wrapper) func INT2NUM(n int) C.VALUE { return C.rb_int2inum(C.long(n)) }

9. Cgo and Strings func C.CString(goString string) *C.char func C.GoString(cString *C.char)

   string func C.GoStringN(cString *C.char, length C.int) string var cmsg *C.char = C.CString("hi") defer C.free(unsafe.Pointer(cmsg)) ※ C.CString() allocates memory and copy the string.
 You must call C.free().

10. Ruby strings to Go Strings Cgo can’t use C Macro;

    Write wrapper in C as Go comments package main /* const char * rstring_ptr(VALUE str) { return RSTRING_PTR(str); } int rstring_len(VALUE str) { return RSTRING_LENINT(str); } */ import "C" func RbGoString(str C.VALUE) string { return C.GoStringN(C.rstring_ptr(str), C.rstring_len(str)) }

11. Go strings to Ruby Strings Ago can’t use C Macro;

    Write wrapper in C as Go comments package main func RbString(str string) C.VALUE { cstr := C.CString(str) defer C.free(cstr) return C.rb_utf8_str_new(cstr, C.long(len(str))) } Allocation and Copy!

12. Go strings to Ruby Strings II Use unsafe.Pointer and casts

    to avoid copy!!!! func RbString(str string) C.VALUE { if len(str) == 0 { return C.rb_utf8_str_new(nil, C.long(0)) } cstr := (*C.char)(unsafe.Pointer(&(*(*[]byte)(unsafe.Pointer(&str)))[0])) return C.rb_utf8_str_new(cstr, C.long(len(str))) } see also http://qiita.com/mattn/items/176459728ff4f854b165

13. 000000000022f3f0 <main.RbString>: 22f3f0: 48 8b 0d 91 0b 66 00

    mov 0x660b91(%rip),%rcx # 88ff88 <_DYNAMIC+0x200> 22f3f7: 64 48 8b 09 mov %fs:(%rcx),%rcx 22f3fb: 48 3b 61 10 cmp 0x10(%rcx),%rsp 22f3ff: 0f 86 ba 00 00 00 jbe 22f4bf <main.RbString+0xcf> 22f405: 48 83 ec 20 sub $0x20,%rsp 22f409: 48 8b 4c 24 30 mov 0x30(%rsp),%rcx 22f40e: 48 c7 44 24 38 00 00 movq $0x0,0x38(%rsp) 22f415: 00 00 22f417: 48 83 f9 00 cmp $0x0,%rcx 22f41b: 75 2b jne 22f448 <main.RbString+0x58> 22f41d: 48 c7 04 24 00 00 00 movq $0x0,(%rsp) 22f424: 00 22f425: 48 c7 44 24 08 00 00 movq $0x0,0x8(%rsp) 22f42c: 00 00 22f42e: e8 1d cb ff ff callq 22bf50 <main._Cfunc_rb_utf8_str_new> 22f433: 48 8b 5c 24 10 mov 0x10(%rsp),%rbx 22f438: 48 89 5c 24 38 mov %rbx,0x38(%rsp) 22f43d: 90 nop 22f43e: e8 7d 85 02 00 callq 2579c0 <runtime.deferreturn> 22f443: 48 83 c4 20 add $0x20,%rsp 22f447: c3 retq 22f448: 48 8b 5c 24 28 mov 0x28(%rsp),%rbx 22f44d: 48 89 1c 24 mov %rbx,(%rsp) 22f451: 48 89 4c 24 08 mov %rcx,0x8(%rsp) 22f456: e8 65 bb ff ff callq 22afc0 <main._Cfunc_CString> 22f45b: 48 8b 44 24 10 mov 0x10(%rsp),%rax 22f460: 48 89 44 24 18 mov %rax,0x18(%rsp) 22f465: 48 89 44 24 10 mov %rax,0x10(%rsp) 22f46a: c7 04 24 08 00 00 00 movl $0x8,(%rsp) 22f471: 48 8d 05 40 4c 38 00 lea 0x384c40(%rip),%rax # 5b40b8 <main._Cfunc_free.f> 22f478: 48 89 44 24 08 mov %rax,0x8(%rsp) 22f47d: e8 be 73 02 00 callq 256840 <runtime.deferproc> 22f482: 83 f8 00 cmp $0x0,%eax 22f485: 75 2d jne 22f4b4 <main.RbString+0xc4> 22f487: 48 8b 44 24 30 mov 0x30(%rsp),%rax 22f48c: 48 8b 5c 24 18 mov 0x18(%rsp),%rbx 22f491: 48 89 1c 24 mov %rbx,(%rsp) 22f495: 48 89 44 24 08 mov %rax,0x8(%rsp) 22f49a: e8 b1 ca ff ff callq 22bf50 <main._Cfunc_rb_utf8_str_new> 22f49f: 48 8b 5c 24 10 mov 0x10(%rsp),%rbx 22f4a4: 48 89 5c 24 38 mov %rbx,0x38(%rsp) 22f4a9: 90 nop 22f4aa: e8 11 85 02 00 callq 2579c0 <runtime.deferreturn> 22f4af: 48 83 c4 20 add $0x20,%rsp 22f4b3: c3 retq 22f4b4: 90 nop 22f4b5: e8 06 85 02 00 callq 2579c0 <runtime.deferreturn> 22f4ba: 48 83 c4 20 add $0x20,%rsp 22f4be: c3 retq 22f4bf: e8 4c 84 05 00 callq 287910 <runtime.morestack_noctxt> 22f4c4: e9 27 ff ff ff jmpq 22f3f0 <main.RbString> C.CString() BEFORE

14. 000000000022afc0 <main._Cfunc_CString>: 22afc0: 48 8b 0d c1 4f 66 00

    mov 0x664fc1(%rip),%rcx # 88ff88 <_DYNAMIC+0x200> 22afc7: 64 48 8b 09 mov %fs:(%rcx),%rcx 22afcb: 48 3b 61 10 cmp 0x10(%rcx),%rsp 22afcf: 0f 86 cc 00 00 00 jbe 22b0a1 <main._Cfunc_CString+0xe1> 22afd5: 48 83 ec 68 sub $0x68,%rsp 22afd9: 48 8b 5c 24 78 mov 0x78(%rsp),%rbx 22afde: 48 ff c3 inc %rbx 22afe1: 48 89 1c 24 mov %rbx,(%rsp) 22afe5: e8 76 62 00 00 callq 231260 <runtime.cmalloc> 22afea: 48 8b 74 24 08 mov 0x8(%rsp),%rsi 22afef: 48 89 74 24 20 mov %rsi,0x20(%rsp) 22aff4: 48 89 74 24 18 mov %rsi,0x18(%rsp) 22aff9: 48 83 fe 00 cmp $0x0,%rsi 22affd: 0f 84 97 00 00 00 je 22b09a <main._Cfunc_CString+0xda> 22b003: 48 c7 c0 00 00 00 40 mov $0x40000000,%rax 22b00a: 48 c7 c1 00 00 00 40 mov $0x40000000,%rcx 22b011: 48 89 74 24 50 mov %rsi,0x50(%rsp) 22b016: 48 89 44 24 58 mov %rax,0x58(%rsp) 22b01b: 48 89 4c 24 60 mov %rcx,0x60(%rsp) 22b020: 48 89 4c 24 48 mov %rcx,0x48(%rsp) 22b025: 48 8b 54 24 70 mov 0x70(%rsp),%rdx 22b02a: 48 8b 4c 24 78 mov 0x78(%rsp),%rcx 22b02f: 48 89 44 24 40 mov %rax,0x40(%rsp) 22b034: 48 89 4c 24 30 mov %rcx,0x30(%rsp) 22b039: 48 39 c1 cmp %rax,%rcx 22b03c: 7d 03 jge 22b041 <main._Cfunc_CString+0x81> 22b03e: 48 89 c8 mov %rcx,%rax 22b041: 48 89 74 24 38 mov %rsi,0x38(%rsp) 22b046: 48 89 34 24 mov %rsi,(%rsp) 22b04a: 48 89 54 24 28 mov %rdx,0x28(%rsp) 22b04f: 48 89 54 24 08 mov %rdx,0x8(%rsp) 22b054: 48 89 44 24 10 mov %rax,0x10(%rsp) 22b059: e8 e2 f9 05 00 callq 28aa40 <runtime.memmove> 22b05e: 48 8b 44 24 78 mov 0x78(%rsp),%rax 22b063: 48 8b 5c 24 18 mov 0x18(%rsp),%rbx 22b068: 48 83 fb 00 cmp $0x0,%rbx 22b06c: 74 28 je 22b096 <main._Cfunc_CString+0xd6> 22b06e: 48 3d 00 00 00 40 cmp $0x40000000,%rax 22b074: 73 19 jae 22b08f <main._Cfunc_CString+0xcf> 22b076: 48 8d 1c 03 lea (%rbx,%rax,1),%rbx 22b07a: c6 03 00 movb $0x0,(%rbx) 22b07d: 48 8b 5c 24 20 mov 0x20(%rsp),%rbx 22b082: 48 89 9c 24 80 00 00 mov %rbx,0x80(%rsp) 22b089: 00 22b08a: 48 83 c4 68 add $0x68,%rsp 22b08e: c3 retq 22b08f: e8 2c b5 02 00 callq 2565c0 <runtime.panicindex> 22b094: 0f 0b ud2 22b096: 89 03 mov %eax,(%rbx) 22b098: eb d4 jmp 22b06e <main._Cfunc_CString+0xae> 22b09a: 89 06 mov %eax,(%rsi) 22b09c: e9 62 ff ff ff jmpq 22b003 <main._Cfunc_CString+0x43> 22b0a1: e8 6a c8 05 00 callq 287910 <runtime.morestack_noctxt> 22b0a6: e9 15 ff ff ff jmpq 22afc0 <main._Cfunc_CString> memmove(3): copy C strings BEFORE

15. 000000000022f160 <main.RbString>: 22f160: 48 8b 0d 21 0e 66 00

    mov 0x660e21(%rip),%rcx # 88ff88 <_DYNAMIC+0x200> 22f167: 64 48 8b 09 mov %fs:(%rcx),%rcx 22f16b: 48 3b 61 10 cmp 0x10(%rcx),%rsp 22f16f: 0f 86 8f 00 00 00 jbe 22f204 <main.RbString+0xa4> 22f175: 48 83 ec 30 sub $0x30,%rsp 22f179: 48 8b 5c 24 40 mov 0x40(%rsp),%rbx 22f17e: 48 83 fb 00 cmp $0x0,%rbx 22f182: 75 25 jne 22f1a9 <main.RbString+0x49> 22f184: 48 c7 04 24 00 00 00 movq $0x0,(%rsp) 22f18b: 00 22f18c: 48 c7 44 24 08 00 00 movq $0x0,0x8(%rsp) 22f193: 00 00 22f195: e8 26 cb ff ff callq 22bcc0 <main._Cfunc_rb_utf8_str_new> 22f19a: 48 8b 5c 24 10 mov 0x10(%rsp),%rbx 22f19f: 48 89 5c 24 48 mov %rbx,0x48(%rsp) 22f1a4: 48 83 c4 30 add $0x30,%rsp 22f1a8: c3 retq 22f1a9: 48 8d 5c 24 38 lea 0x38(%rsp),%rbx 22f1ae: 48 83 fb 00 cmp $0x0,%rbx 22f1b2: 74 4c je 22f200 <main.RbString+0xa0> 22f1b4: 48 8b 0b mov (%rbx),%rcx 22f1b7: 48 8b 43 08 mov 0x8(%rbx),%rax 22f1bb: 48 8b 6b 10 mov 0x10(%rbx),%rbp 22f1bf: 48 89 6c 24 28 mov %rbp,0x28(%rsp) 22f1c4: 48 89 4c 24 18 mov %rcx,0x18(%rsp) 22f1c9: 48 83 f8 00 cmp $0x0,%rax 22f1cd: 48 89 44 24 20 mov %rax,0x20(%rsp) 22f1d2: 76 25 jbe 22f1f9 <main.RbString+0x99> 22f1d4: 48 89 c8 mov %rcx,%rax 22f1d7: 48 8b 4c 24 40 mov 0x40(%rsp),%rcx 22f1dc: 48 89 04 24 mov %rax,(%rsp) 22f1e0: 48 89 4c 24 08 mov %rcx,0x8(%rsp) 22f1e5: e8 d6 ca ff ff callq 22bcc0 <main._Cfunc_rb_utf8_str_new> 22f1ea: 48 8b 5c 24 10 mov 0x10(%rsp),%rbx 22f1ef: 48 89 5c 24 48 mov %rbx,0x48(%rsp) 22f1f4: 48 83 c4 30 add $0x30,%rsp 22f1f8: c3 retq 22f1f9: e8 02 71 02 00 callq 256300 <runtime.panicindex> 22f1fe: 0f 0b ud2 22f200: 89 03 mov %eax,(%rbx) 22f202: eb b0 jmp 22f1b4 <main.RbString+0x54> 22f204: e8 47 84 05 00 callq 287650 <runtime.morestack_noctxt> 22f209: e9 52 ff ff ff jmpq 22f160 <main.RbString> zero copy! AFTER Note that rb_str_new() copies the content

16. Other wrappers package main // #include <ruby/ruby.h> import "C" import

    "unsafe" func rb_define_method(klass C.VALUE, name string, fun unsafe.Pointer, args int) { cname := (*C.char)(unsafe.Pointer(&(*(*[]byte)(unsafe.Pointer(&name)))[0])) C.define_method(klass, cname, (*[0]byte)(fun), C.int(args)) } func rb_define_singleton_method(klass C.VALUE, name string, fun unsafe.Pointer, args int) { cname := (*C.char)(unsafe.Pointer(&(*(*[]byte)(unsafe.Pointer(&name)))[0])) C.define_singleton_method(klass, cname, (*[0]byte)(fun), C.int(args)) }

17. Class definition and Method declarations //export Init_gohttp func Init_gohttp() {

    sNew := "new" str_new := (*C.char)(unsafe.Pointer(&(*(*[]byte)(unsafe.Pointer(&sNew)))[0])) rb_cRequest = rb_define_class("Request", C.rb_cObject) C.rb_undef_alloc_func(rb_cRequest) C.rb_undef_method(C.rb_class_of(rb_cRequest), str_new) rb_define_method(rb_cRequest, "method", C.reqMethod, 0) //rb_define_method(rb_cRequest, "url", C.reqURL, 0) rb_define_method(rb_cRequest, "proto", C.reqProto, 0) rb_define_method(rb_cRequest, "header", C.reqHeader, 0) rb_define_method(rb_cRequest, "body", C.reqBody, 0) rb_define_method(rb_cRequest, "content_length", C.reqContentLength, 0) rb_define_method(rb_cRequest, "transfer_encoding", C.reqTransferEncoding, 0) rb_define_method(rb_cRequest, "host", C.reqHost, 0) //rb_define_method(rb_cRequest, "form", C.reqForm, 0) //rb_define_method(rb_cRequest, "post_form", C.reqForm, 0) //rb_define_method(rb_cRequest, "multipart_form", C.reqForm, 0) //rb_define_method(rb_cRequest, "trailer", C.reqForm, 0) rb_define_method(rb_cRequest, "remote_addr", C.reqRemoteAddr, 0) rb_define_method(rb_cRequest, "request_uri", C.reqRequestURI, 0)

18. Generic Go object definition for Ruby /* void goobj_retain(void *);

    void goobj_free(void *); static const rb_data_type_t go_type = { "GoStruct", {NULL, goobj_free, NULL}, 0, 0, RUBY_TYPED_FREE_IMMEDIATELY|RUBY_TYPED_WB_PROTECTED }; */ VALUE NewGoStruct(VALUE klass, void *p) { goobj_retain(p); return TypedData_Wrap_Struct((klass), &go_type, p); } void * GetGoStruct(VALUE obj) { void *val; return TypedData_Get_Struct((obj), void *, &go_type, (val)); }

19. Avoid Go’s GC var objects = make(map[interface{}]bool) //export goobj_retain func

    goobj_retain(obj unsafe.Pointer) { objects[obj] = true } //export goobj_free func goobj_free(obj unsafe.Pointer) { delete(objects, obj) }

20. Method definitions //export gohttpGet func gohttpGet(dummy C.VALUE, urlstr C.VALUE) C.VALUE

    { str := RbGoString(urlstr) resp, err := http.Get(str) if err != nil { rb_raise(C.rb_eArgError, "'%s'", err) } return respNew(rb_cResponse, resp) } func respNew(klass C.VALUE, r *http.Response) C.VALUE { return C.NewGoStruct(klass, unsafe.Pointer(r)) } //export respContentLength func respContentLength(self C.VALUE) C.VALUE { resp := (*http.Response)(C.GetGoStruct(self)) return INT64toNUM(resp.ContentLength) }

21. Now you can call Go’s net/http from Ruby def test_get

    r = Gohttp.get('https://example.com/index.html') assert_kind_of Response, r assert_equal '200 OK', r.status assert_equal 200, r.status_code assert_equal 'HTTP/1.1', r.proto assert_equal 1270, r.size assert_equal [], r.transfer_encoding end

22. ·ͱΊ GoͰRuby༻֦ுϥΠϒϥϦΛॻ͚·͢ C֦ுϥΠϒϥϦͷ஌͕ࣝඞཁͰ͢ CͷϚΫϩ͸࢖͑ͳ͍ͷͰϥούʔ͕ඞཁͰ͢ ୭͔ͱΓ·ͱΊͯཉ͍͠ ͍͔ͭ͘ಛ༗ͷςΫχοΫ͕ඞཁͰ͢ جຊతͳ͜ͱ͸͜ͷൃදͰΧόʔͨ͠͸ͣͰ͢ ΈΜͳͰ஌ݟΛͨΊͯɺGoogleͷ੒Ռʹ৐͔ͬΖ͏ʂʂʂ