Upgrade to Pro — share decks privately, control downloads, hide ads and more …

From cgo back to Go - GopherCon 2016

From cgo back to Go - GopherCon 2016

As soon as you import "C", a lot of the things we love about Go are gone.

This talk helps you walk your way back up from the bottom of that pit.

https://www.youtube.com/watch?v=lhMhApWQp2E

9fdab9d005b82612cadbfe699b541f83?s=128

Filippo Valsorda

July 12, 2016
Tweet

More Decks by Filippo Valsorda

Other Decks in Programming

Transcript

  1. From cgo back to Go Filippo Valsorda

  2. “cgo is not Go” —Dave Cheney http://dave.cheney.net/2016/01/18/cgo-is-not-go

  3. • Effortless memory management • Runtime speed • Awesome tooling

    • Fast builds • Static binaries • Reproducible builds • go get • Super-easy cross compiling
  4. Then you import “C”

  5. • Effortless memory management • Awesome tooling • Runtime speed

    • Quick compilation • Reproducible builds • Static binaries • go get • Super-easy cross compiling
  6. • Effortless memory management • Awesome tooling • Runtime speed

    • Quick compilation • Reproducible builds • Static binaries • go get • Super-easy cross compiling
  7. Go memory C memory GC

  8. Go memory C memory GC make([]byte, 1024) &MyStruct{}

  9. Go memory C memory GC []byte *MyStruct

  10. Go memory C memory GC []byte

  11. Go memory C memory GC []byte

  12. Go memory C memory GC []byte *uint8_t C.some_func()

  13. Go memory C memory GC []byte

  14. None
  15. Go memory C memory GC []byte *uint8_t C.some_func()

  16. Go memory C memory GC []byte *uint8_t

  17. Go memory C memory GC *uint8_t

  18. Go memory C memory GC *uint8_t

  19. None
  20. None
  21. // typedef struct { // int a, b, c; //

    } someStruct; // // someStruct *someReference; // void storeReference(someStruct *s) { // someReference = s; // } // int retrieveReference() { // return someReference->a; // } import "C" import "runtime" func main() { passSomeMemory() runtime.GC() println(getSomeMemory()) } func passSomeMemory() { someGoMemory := &C.someStruct{ a: 1, b: 2, c: 3, } C.storeReference(someGoMemory) } func getSomeMemory() int { return int(C.retrieveReference()) } Output: 1
  22. Output: 1 // typedef struct { // int a, b,

    c; // } someStruct; // // someStruct *someReference; // void storeReference(someStruct *s) { // someReference = s; // } // int retrieveReference() { // return someReference->a; // } import "C" import "runtime" func main() { passSomeMemory() runtime.GC() println(getSomeMemory()) } func passSomeMemory() { someGoMemory := &C.someStruct{ a: 1, b: 2, c: 3, } C.storeReference(someGoMemory) } func getSomeMemory() int { return int(C.retrieveReference()) } 537247792
  23. The cgo rules You may pass a Go pointer …

    if it doesn’t point to other pointers … and C can’t keep a reference to it
  24. The GC must see all the Go pointers.

  25. None
  26. None
  27. // #include <stdlib.h> // // typedef struct { // int

    a, b, c; // } someStruct; // // void processStruct(someStruct *s) {} import "C" func ProcessStruct() { someCMemory := C.calloc(1, C.sizeof_someStruct) defer C.free(someCMemory) s := (*C.someStruct)(someCMemory) C.processStruct(s) }
  28. // #include <stdlib.h> // // typedef struct { // int

    a, b, c; // } someStruct; // // void processStruct(someStruct *s) {} import "C" func ProcessStruct() { someCMemory := C.calloc(1, C.sizeof_someStruct) defer C.free(someCMemory) s := (*C.someStruct)(someCMemory) C.processStruct(s) }
  29. Copy. Copy. Copy.

  30. // #include <stdlib.h> // // typedef struct { // int

    a, b, c; // char *str; // } someStruct; // // void processStruct(someStruct *s) {} import "C" import "unsafe" func ProcessStruct(str string) { s := (*C.someStruct)(C.calloc(1, C.sizeof_someStruct)) defer C.free(unsafe.Pointer(s)) s.str = C.CString(str) defer C.free(unsafe.Pointer(s.str)) C.processStruct(s) }
  31. // #include <stdlib.h> // #include <stdint.h> // #include <string.h> //

    typedef struct { uint8_t *data; size_t data_len; } someStruct; // void processStruct(someStruct *s) {} import "C" import "unsafe" func ProcessStruct(data []byte) { s := (*C.someStruct)(C.calloc(1, C.sizeof_someStruct)) defer C.free(unsafe.Pointer(s)) cMem := C.calloc(1, C.size_t(len(data))) defer C.free(cMem) s.data = (*C.uint8_t)(C.memmove( cMem, unsafe.Pointer(&data[0]), C.size_t(len(data)))) s.data_len = C.size_t(len(data)) C.processStruct(s) }
  32. // #include <stdlib.h> // #include <stdint.h> // #include <string.h> //

    // typedef struct { // uint8_t *data; // size_t data_len; // } someStruct; // // void processStruct(someStruct *s) {} import "C" import "unsafe" type SomeStruct struct { Data []byte } func ProcessStruct(s *SomeStruct) { [...] }
  33. // #cgo pkg-config: sqlite3 // #include <sqlite3.h> import "C" import

    "errors" type DB struct { db *C.sqlite3 } func Open(filename string) (d *DB, err error) { d = &DB{} if C.sqlite3_open(C.CString(filename), &d.db) != C.SQLITE_OK { err = errors.New(C.GoString(C.sqlite3_errmsg(d.db))) C.sqlite3_close(d.db) } return } func (d *DB) Close() { C.sqlite3_close(d.db) } Wrapper types
  34. The arena pattern

  35. type arena []unsafe.Pointer func (a *arena) calloc(count, size int) unsafe.Pointer

    { ptr := C.calloc(C.size_t(count), C.size_t(size)) *a = append(*a, ptr) return ptr } func (a *arena) free() { for _, ptr := range *a { C.free(ptr) } }
  36. func ProcessStruct(s *SomeStruct) { ar := &arena{} defer ar.free() cs

    := (*C.someStruct)(ar.calloc(C.sizeof_someStruct, 1)) cMem := ar.calloc(1, len(s.Data)) cs.data = (*C.uint8_t)(C.memmove(cMem, unsafe.Pointer(&s.Data[0]), C.size_t(len(s.Data)))) cs.data_len = C.size_t(len(s.Data)) C.processStruct(cs) }
  37. type arena []unsafe.Pointer func (a *arena) calloc(count, size int) unsafe.Pointer

    { ptr := C.calloc(C.size_t(count), C.size_t(size)) *a = append(*a, ptr) return ptr } func (a *arena) copy(data []byte) unsafe.Pointer { ptr := a.calloc(1, len(data)) C.memmove(ptr, unsafe.Pointer(&data[0]), C.size_t(len(data))) return ptr } func (a *arena) free() { for _, ptr := range *a { C.free(ptr) } }
  38. func ProcessStruct(s *SomeStruct) { ar := &arena{} defer ar.free() cs

    := (*C.someStruct)(ar.calloc(C.sizeof_someStruct, 1)) cs.data = (*C.uint8_t)(ar.copy(s.Data)) cs.data_len = C.size_t(len(s.Data)) C.processStruct(cs) }
  39. If you really have to…

  40. // #include <stdlib.h> // #include <stdint.h> // // void processBigData(uint8_t

    *data, size_t data_len) {} import "C" func ProcessBigData(bigData []byte) { cBigData = (*C.uint8_t)(&bigData[0]) cBigDataLen = C.size_t(len(bigData)) C.processBigData(cBigData, cBigDataLen) }
  41. Go memory C memory GC []byte

  42. Go memory C memory GC []byte *uint8_t C.some_func()

  43. Go memory C memory GC []byte

  44. None
  45. // typedef struct { // int a, b, c; //

    } someStruct; // // typedef struct { // someStruct *x, *y; // } complexStruct; // // void processStruct(complexStruct *s) {} import "C" func main() { someGoMemory := &C.someStruct{ a: 1, b: 2, c: 3, } C.processStruct(&C.complexStruct{ x: someGoMemory, y: nil, }) }
  46. None
  47. None
  48. panic: runtime error: cgo argument has Go pointer to Go

    pointer
  49. GODEBUG=cgocheck=2

  50. go doc cmd/cgo

  51. Callbacks

  52. int sqlite3_exec( sqlite3*, /* An open database */ const char

    *sql, /* SQL to be evaluated */ int (*callback)(void*,int,char**,char**), /* Callback function */ void *, /* 1st argument to callback */ char **errmsg /* Error msg written here */ );
  53. var zErrMsg *C.char rc = C.sqlite3_exec(d.db, sqlStatement, ???, ???, &zErrMsg)

    if rc != C.SQLITE_OK { log.Printf("SQL error: %s\n", C.GoString(zErrMsg)) C.sqlite3_free(unsafe.Pointer(zErrMsg)) }
  54. package main import "C" import "unsafe" //export callback func callback(userData

    uintptr, _ C.int, _ **C.char, _ **C.char) C.int { return 0 }
  55. // #cgo pkg-config: sqlite3 // #include <sqlite3.h> // int callback(void*,

    int, char**, char**); import "C" [...] var zErrMsg *C.char rc = C.sqlite3_exec(d.db, sqlStatement, (*[0]byte)(C.callback), unsafe.Pointer(nil), &zErrMsg) if rc != C.SQLITE_OK { log.Printf("SQL error: %s\n", C.GoString(zErrMsg)) C.sqlite3_free(unsafe.Pointer(zErrMsg)) }
  56. // NOTE: it’s up to you to manage concurrency (i.e.

    with a RWMutex) var handles = make(map[uintptr]interface{}) // NOTE: use low numbers not to overlap with Go memory handles[uintptr(42)] = "some Go memory" rc = C.sqlite3_exec(d.db, sqlStatement, (*[0]byte)(C.callback), unsafe.Pointer(uintptr(42)), &zErrMsg) //export callback func callback(userData uintptr, _ C.int, _ **C.char, _ **C.char) C.int { goData := handles[userData] [...] } Example: https://github.com/mattn/go-sqlite3/blob/ 76e335f60bbcee20755df9864f0153af1a80ad2d/callback.go#L52
  57. • Effortless memory management • Awesome tooling • Runtime speed

    • Quick compilation • Reproducible builds • Static binaries • go get • Super-easy cross compiling
  58. package main // #cgo LDFLAGS: -lm // #include <math.h> //

    void slow_func() { // double a; int i; // for (i = 1; i < 1e8; i++) { // a = pow(a, 3); // } // } import "C" import "os" import "runtime/pprof" func main() { f, _ := os.Create("cgo.pprof") pprof.StartCPUProfile(f) C.slow_func() pprof.StopCPUProfile() }
  59. None
  60. $ apt-get install linux-tools-generic $ go version go version go1.7rc1

    linux/amd64 $ perf record ./slow $ perf report
  61. None
  62. • Effortless memory management • Awesome tooling • Runtime speed

    • Quick compilation • Reproducible builds • Static binaries • go get • Super-easy cross compiling
  63. package main // #cgo LDFLAGS: -lm // #include <math.h> import

    "C" func main() { var a C.double for i := 1; i < 1e8; i++ { a = C.pow(a, 3) } }
  64. package main // #cgo LDFLAGS: -lm // #include <math.h> //

    void powloop() { // double a; int i; // for (i = 1; i < 1e8; i++) { // a = pow(a, 3); // } // } import "C" func main() { C.powloop() }
  65. • Effortless memory management • Awesome tooling • Runtime speed

    • Quick compilation • Reproducible builds • Static binaries • go get • Super-easy cross compiling
  66. $ cat $GOPATH/src/github.com/FiloSottile/hashpass/main.go package main // #cgo pkg-config: libsodium //

    #include <sodium.h> import "C" func main() { C.sodium_init() println(hashPass("Correct Horse Battery Staple")) } func hashPass(pass string) string { cMem := C.calloc(1, C.crypto_pwhash_STRBYTES) defer C.free(cMem) res := (*C.char)(cMem) C.crypto_pwhash_str(res, C.CString(pass), (C.ulonglong)(len(pass)), C.crypto_pwhash_OPSLIMIT_SENSITIVE, C.crypto_pwhash_MEMLIMIT_SENSITIVE) return C.GoStringN(res, C.crypto_pwhash_STRBYTES) }
  67. $ (cd libsodium-1.0.10 && ./configure && make install) $ go

    build github.com/FiloSottile/hashpass $ ./hashpass $argon2i$v=19$m=524288,t=8,p=1$ISqzZv/SaISgeOJTsWIc4Q $w5PTypPasNCkQhm6mnfMhYqpkoGXn0UCO42J6Qjrnko $ file hashpass hashpass: ELF 64-bit LSB executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /lib64/ld-linux-x86-64.so.2, for GNU/Linux 2.6.32, BuildID[sha1]=dbae930638bb69e5c461a33d1823906928bb3706, not stripped
  68. $ go build -v -x github.com/FiloSottile/hashpass WORK=/tmp/go-build285911135 github.com/FiloSottile/hashpass mkdir -p

    $WORK/github.com/FiloSottile/hashpass/_obj/ mkdir -p $WORK/github.com/FiloSottile/hashpass/_obj/exe/ cd /Users/filippo/go/src/github.com/FiloSottile/hashpass pkg-config --cflags libsodium pkg-config --libs libsodium CGO_LDFLAGS="-g" "-O2" "-L/usr/local/lib" "-lsodium" /usr/local/go/pkg/tool/linux_amd64/cgo -objdir $WORK/github.com/FiloSottile/hashpass/ _obj/ -importpath github.com/FiloSottile/hashpass -- -I/usr/local/include -I $WORK/github.com/FiloSottile/hashpass/_obj/ main.go gcc -I . -fPIC -m64 -pthread -fmessage-length=0 -I/usr/local/include -I $WORK/github.com/FiloSottile/hashpass/_obj/ -g -O2 -o $WORK/ github.com/FiloSottile/hashpass/_obj/_cgo_main.o -c $WORK/github.com/FiloSottile/hashpass/_obj/_cgo_main.c gcc -I . -fPIC -m64 -pthread -fmessage-length=0 -I/usr/local/include -I $WORK/github.com/FiloSottile/hashpass/_obj/ -g -O2 -o $WORK/ github.com/FiloSottile/hashpass/_obj/_cgo_export.o -c $WORK/github.com/FiloSottile/hashpass/_obj/_cgo_export.c gcc -I . -fPIC -m64 -pthread -fmessage-length=0 -I/usr/local/include -I $WORK/github.com/FiloSottile/hashpass/_obj/ -g -O2 -o $WORK/ github.com/FiloSottile/hashpass/_obj/main.cgo2.o -c $WORK/github.com/FiloSottile/hashpass/_obj/main.cgo2.c gcc -I . -fPIC -m64 -pthread -fmessage-length=0 -o $WORK/github.com/FiloSottile/hashpass/_obj/_cgo_.o $WORK/github.com/FiloSottile/ hashpass/_obj/_cgo_main.o $WORK/github.com/FiloSottile/hashpass/_obj/_cgo_export.o $WORK/github.com/FiloSottile/hashpass/_obj/main.cgo2.o -g -O2 -L/usr/local/lib -lsodium /usr/local/go/pkg/tool/linux_amd64/cgo -objdir $WORK/github.com/FiloSottile/hashpass/_obj/ -dynpackage main -dynimport $WORK/github.com/ FiloSottile/hashpass/_obj/_cgo_.o -dynout $WORK/github.com/FiloSottile/hashpass/_obj/_cgo_import.go cd $WORK gcc -I . -fPIC -m64 -pthread -fmessage-length=0 -no-pie -c trivial.c cd /Users/filippo/go/src/github.com/FiloSottile/hashpass gcc -I . -fPIC -m64 -pthread -fmessage-length=0 -o $WORK/github.com/FiloSottile/hashpass/_obj/_all.o $WORK/github.com/FiloSottile/ hashpass/_obj/_cgo_export.o $WORK/github.com/FiloSottile/hashpass/_obj/main.cgo2.o -g -O2 -L/usr/local/lib -Wl,-r -nostdlib -Wl,--build- id=none /usr/local/go/pkg/tool/linux_amd64/compile -o $WORK/github.com/FiloSottile/hashpass.a -trimpath $WORK -p main -buildid ce5e1a0eab9fd96772b6a135be9ae6179cae3208 -D _/Users/filippo/go/src/github.com/FiloSottile/hashpass -I $WORK -pack $WORK/github.com/ FiloSottile/hashpass/_obj/_cgo_gotypes.go $WORK/github.com/FiloSottile/hashpass/_obj/main.cgo1.go $WORK/github.com/FiloSottile/hashpass/ _obj/_cgo_import.go pack r $WORK/github.com/FiloSottile/hashpass.a $WORK/github.com/FiloSottile/hashpass/_obj/_all.o # internal cd . /usr/local/go/pkg/tool/linux_amd64/link -o $WORK/github.com/FiloSottile/hashpass/_obj/exe/a.out -L $WORK -extld=gcc -buildmode=exe - buildid=ce5e1a0eab9fd96772b6a135be9ae6179cae3208 $WORK/github.com/FiloSottile/hashpass.a cp $WORK/github.com/FiloSottile/hashpass/_obj/exe/a.out hashpass
  69. $ cat $GOPATH/src/github.com/FiloSottile/hashpass/sodium/hash.go package sodium // #cgo pkg-config: libsodium //

    #include <sodium.h> import "C" func init() { C.sodium_init() } func HashPass(pass string) string { cMem := C.calloc(1, C.crypto_pwhash_STRBYTES) defer C.free(cMem) res := (*C.char)(cMem) C.crypto_pwhash_str(res, C.CString(pass), (C.ulonglong)(len(pass)), C.crypto_pwhash_OPSLIMIT_SENSITIVE, C.crypto_pwhash_MEMLIMIT_SENSITIVE) return C.GoStringN(res, C.crypto_pwhash_STRBYTES) }
  70. $ go build -v —i github.com/FiloSottile/hashpass github.com/FiloSottile/hashpass/sodium github.com/FiloSottile/hashpass $ go

    build -v -x github.com/FiloSottile/hashpass WORK=/tmp/go-build607855468 github.com/FiloSottile/hashpass mkdir -p $WORK/github.com/FiloSottile/hashpass/_obj/ mkdir -p $WORK/github.com/FiloSottile/hashpass/_obj/exe/ cd /Users/filippo/go/src/github.com/FiloSottile/hashpass /usr/local/go/pkg/tool/linux_amd64/compile -o $WORK/github.com/FiloSottile/ hashpass.a -trimpath $WORK -p main -complete -buildid 099121d2abae4b427870cba7785d01725b6dca64 -D _/Users/filippo/go/src/github.com/ FiloSottile/hashpass -I $WORK -I /Users/filippo/go/pkg/linux_amd64 -pack ./main.go cd . /usr/local/go/pkg/tool/linux_amd64/link -o $WORK/github.com/FiloSottile/hashpass/ _obj/exe/a.out -L $WORK -L /Users/filippo/go/pkg/linux_amd64 -extld=gcc - buildmode=exe -buildid=099121d2abae4b427870cba7785d01725b6dca64 $WORK/github.com/ FiloSottile/hashpass.a cp $WORK/github.com/FiloSottile/hashpass/_obj/exe/a.out hashpass
  71. • Effortless memory management • Awesome tooling • Runtime speed

    • Quick compilation • Reproducible builds • Static binaries • go get • Super-easy cross compiling
  72. $ mv libsodium-1.0.10 \ $GOPATH/src/github.com/FiloSottile/hashpass/sodium/_libsodium-1.0.10 $ head $GOPATH/src/github.com/FiloSottile/hashpass/sodium/hash.go package sodium

    // #cgo CFLAGS: -I${SRCDIR}/_libsodium-1.0.10/src/libsodium/include/ // #cgo LDFLAGS: -L${SRCDIR}/_libsodium-1.0.10/src/libsodium/.libs/ -lsodium // #include <sodium.h> import "C" $ (cd $GOPATH/src/github.com/FiloSottile/hashpass/sodium/_libsodium-1.0.10 && \ ./configure && make) $ go build github.com/FiloSottile/hashpass
  73. • Effortless memory management • Awesome tooling • Runtime speed

    • Quick compilation • Reproducible builds • Static binaries • go get • Super-easy cross compiling
  74. $ go build -ldflags="-extldflags -static" github.com/FiloSottile/hashpass $ file hashpass hashpass:

    ELF 64-bit LSB executable, x86-64, version 1 (GNU/Linux), statically linked, for GNU/Linux 2.6.32, BuildID[sha1]=b0b0ccdbb3860c12c49e8034ce8fd854b9943eb3, not stripped
  75. $ apt-get install musl-tools $ export CC=musl-gcc $ (cd $GOPATH/src/github.com/FiloSottile/hashpass/sodium/_libsodium-1.0.10

    && \ make clean && ./configure && make) $ go install -installsuffix musl github.com/FiloSottile/hashpass/sodium $ go build -installsuffix musl -ldflags="-extldflags -static" github.com/ FiloSottile/hashpass $ file hashpass hashpass: ELF 64-bit LSB executable, x86-64, version 1 (SYSV), statically linked, not stripped
  76. os/user net crypto/x509

  77. • Effortless memory management • Awesome tooling • Runtime speed

    • Quick compilation • Reproducible builds • Static binaries • go get • Super-easy cross compiling
  78. $ cd $GOPATH/src/github.com/FiloSottile/hashpass/sodium $ (cd _libsodium-1.0.10 && ./configure && make)

    $ mkdir _linux_amd64 $ cp _libsodium-1.0.10/src/libsodium/include/sodium{,.h} _linux_amd64 $ cp _libsodium-1.0.10/src/libsodium/.libs/*.a _linux_amd64 $ head $GOPATH/src/github.com/FiloSottile/hashpass/sodium/hash.go package sodium // #cgo linux,amd64 CFLAGS: -I${SRCDIR}/_linux_amd64/ // #cgo linux,amd64 LDFLAGS: -L${SRCDIR}/_linux_amd64/ // #cgo !linux !amd64 CFLAGS: -I${SRCDIR}/_libsodium-1.0.10/src/libsodium/include/ // #cgo !linux !amd64 LDFLAGS: -L${SRCDIR}/_libsodium-1.0.10/src/libsodium/.libs/ // #cgo LDFLAGS: -lsodium // #include <sodium.h> import “C" $ go get github.com/FiloSottile/hashpass
  79. • Effortless memory management • Awesome tooling • Runtime speed

    • Quick compilation • Reproducible builds • Static binaries • go get • Super-easy cross compiling
  80. musl based OS X → Linux cross-compilers brew install https://gist.github.com/FiloSottile/

    01d2bbfaf63ae1b6e373e6bc874fefc6/raw/ f74e34dbf2823e953af28c6b77b7a5139a4f2876/musl-cross.rb $ CC=x86_64-linux-musl-cc CGO_ENABLED=1 GOOS=linux go build -i \ -ldflags '-extldflags -static' github.com/FiloSottile/hashpass
  81. • Effortless memory management • Runtime speed • Awesome tooling

    • Fast builds • Static binaries • Reproducible builds • go get • Super-easy cross compiling
  82. Questions? Filippo Valsorda @FiloSottile filippo@cloudflare.com Olga Shalakhina artwork under CC

    3.0 license based on Renee French under Creative Commons 3.0 Attributions.