From cgo back to Go - GopherCon 2016

From cgo back to Go - GopherCon 2016

As soon as you import "C", a lot of the things we love about Go are gone.

This talk helps you walk your way back up from the bottom of that pit.

https://www.youtube.com/watch?v=lhMhApWQp2E

9fdab9d005b82612cadbfe699b541f83?s=128

Filippo Valsorda

July 12, 2016
Tweet

Transcript

  1. 3.

    • Effortless memory management • Runtime speed • Awesome tooling

    • Fast builds • Static binaries • Reproducible builds • go get • Super-easy cross compiling
  2. 5.

    • Effortless memory management • Awesome tooling • Runtime speed

    • Quick compilation • Reproducible builds • Static binaries • go get • Super-easy cross compiling
  3. 6.

    • Effortless memory management • Awesome tooling • Runtime speed

    • Quick compilation • Reproducible builds • Static binaries • go get • Super-easy cross compiling
  4. 14.
  5. 19.
  6. 20.
  7. 21.

    // typedef struct { // int a, b, c; //

    } someStruct; // // someStruct *someReference; // void storeReference(someStruct *s) { // someReference = s; // } // int retrieveReference() { // return someReference->a; // } import "C" import "runtime" func main() { passSomeMemory() runtime.GC() println(getSomeMemory()) } func passSomeMemory() { someGoMemory := &C.someStruct{ a: 1, b: 2, c: 3, } C.storeReference(someGoMemory) } func getSomeMemory() int { return int(C.retrieveReference()) } Output: 1
  8. 22.

    Output: 1 // typedef struct { // int a, b,

    c; // } someStruct; // // someStruct *someReference; // void storeReference(someStruct *s) { // someReference = s; // } // int retrieveReference() { // return someReference->a; // } import "C" import "runtime" func main() { passSomeMemory() runtime.GC() println(getSomeMemory()) } func passSomeMemory() { someGoMemory := &C.someStruct{ a: 1, b: 2, c: 3, } C.storeReference(someGoMemory) } func getSomeMemory() int { return int(C.retrieveReference()) } 537247792
  9. 23.

    The cgo rules You may pass a Go pointer …

    if it doesn’t point to other pointers … and C can’t keep a reference to it
  10. 25.
  11. 26.
  12. 27.

    // #include <stdlib.h> // // typedef struct { // int

    a, b, c; // } someStruct; // // void processStruct(someStruct *s) {} import "C" func ProcessStruct() { someCMemory := C.calloc(1, C.sizeof_someStruct) defer C.free(someCMemory) s := (*C.someStruct)(someCMemory) C.processStruct(s) }
  13. 28.

    // #include <stdlib.h> // // typedef struct { // int

    a, b, c; // } someStruct; // // void processStruct(someStruct *s) {} import "C" func ProcessStruct() { someCMemory := C.calloc(1, C.sizeof_someStruct) defer C.free(someCMemory) s := (*C.someStruct)(someCMemory) C.processStruct(s) }
  14. 30.

    // #include <stdlib.h> // // typedef struct { // int

    a, b, c; // char *str; // } someStruct; // // void processStruct(someStruct *s) {} import "C" import "unsafe" func ProcessStruct(str string) { s := (*C.someStruct)(C.calloc(1, C.sizeof_someStruct)) defer C.free(unsafe.Pointer(s)) s.str = C.CString(str) defer C.free(unsafe.Pointer(s.str)) C.processStruct(s) }
  15. 31.

    // #include <stdlib.h> // #include <stdint.h> // #include <string.h> //

    typedef struct { uint8_t *data; size_t data_len; } someStruct; // void processStruct(someStruct *s) {} import "C" import "unsafe" func ProcessStruct(data []byte) { s := (*C.someStruct)(C.calloc(1, C.sizeof_someStruct)) defer C.free(unsafe.Pointer(s)) cMem := C.calloc(1, C.size_t(len(data))) defer C.free(cMem) s.data = (*C.uint8_t)(C.memmove( cMem, unsafe.Pointer(&data[0]), C.size_t(len(data)))) s.data_len = C.size_t(len(data)) C.processStruct(s) }
  16. 32.

    // #include <stdlib.h> // #include <stdint.h> // #include <string.h> //

    // typedef struct { // uint8_t *data; // size_t data_len; // } someStruct; // // void processStruct(someStruct *s) {} import "C" import "unsafe" type SomeStruct struct { Data []byte } func ProcessStruct(s *SomeStruct) { [...] }
  17. 33.

    // #cgo pkg-config: sqlite3 // #include <sqlite3.h> import "C" import

    "errors" type DB struct { db *C.sqlite3 } func Open(filename string) (d *DB, err error) { d = &DB{} if C.sqlite3_open(C.CString(filename), &d.db) != C.SQLITE_OK { err = errors.New(C.GoString(C.sqlite3_errmsg(d.db))) C.sqlite3_close(d.db) } return } func (d *DB) Close() { C.sqlite3_close(d.db) } Wrapper types
  18. 35.

    type arena []unsafe.Pointer func (a *arena) calloc(count, size int) unsafe.Pointer

    { ptr := C.calloc(C.size_t(count), C.size_t(size)) *a = append(*a, ptr) return ptr } func (a *arena) free() { for _, ptr := range *a { C.free(ptr) } }
  19. 36.

    func ProcessStruct(s *SomeStruct) { ar := &arena{} defer ar.free() cs

    := (*C.someStruct)(ar.calloc(C.sizeof_someStruct, 1)) cMem := ar.calloc(1, len(s.Data)) cs.data = (*C.uint8_t)(C.memmove(cMem, unsafe.Pointer(&s.Data[0]), C.size_t(len(s.Data)))) cs.data_len = C.size_t(len(s.Data)) C.processStruct(cs) }
  20. 37.

    type arena []unsafe.Pointer func (a *arena) calloc(count, size int) unsafe.Pointer

    { ptr := C.calloc(C.size_t(count), C.size_t(size)) *a = append(*a, ptr) return ptr } func (a *arena) copy(data []byte) unsafe.Pointer { ptr := a.calloc(1, len(data)) C.memmove(ptr, unsafe.Pointer(&data[0]), C.size_t(len(data))) return ptr } func (a *arena) free() { for _, ptr := range *a { C.free(ptr) } }
  21. 38.

    func ProcessStruct(s *SomeStruct) { ar := &arena{} defer ar.free() cs

    := (*C.someStruct)(ar.calloc(C.sizeof_someStruct, 1)) cs.data = (*C.uint8_t)(ar.copy(s.Data)) cs.data_len = C.size_t(len(s.Data)) C.processStruct(cs) }
  22. 40.

    // #include <stdlib.h> // #include <stdint.h> // // void processBigData(uint8_t

    *data, size_t data_len) {} import "C" func ProcessBigData(bigData []byte) { cBigData = (*C.uint8_t)(&bigData[0]) cBigDataLen = C.size_t(len(bigData)) C.processBigData(cBigData, cBigDataLen) }
  23. 44.
  24. 45.

    // typedef struct { // int a, b, c; //

    } someStruct; // // typedef struct { // someStruct *x, *y; // } complexStruct; // // void processStruct(complexStruct *s) {} import "C" func main() { someGoMemory := &C.someStruct{ a: 1, b: 2, c: 3, } C.processStruct(&C.complexStruct{ x: someGoMemory, y: nil, }) }
  25. 46.
  26. 47.
  27. 51.
  28. 52.

    int sqlite3_exec( sqlite3*, /* An open database */ const char

    *sql, /* SQL to be evaluated */ int (*callback)(void*,int,char**,char**), /* Callback function */ void *, /* 1st argument to callback */ char **errmsg /* Error msg written here */ );
  29. 53.

    var zErrMsg *C.char rc = C.sqlite3_exec(d.db, sqlStatement, ???, ???, &zErrMsg)

    if rc != C.SQLITE_OK { log.Printf("SQL error: %s\n", C.GoString(zErrMsg)) C.sqlite3_free(unsafe.Pointer(zErrMsg)) }
  30. 54.

    package main import "C" import "unsafe" //export callback func callback(userData

    uintptr, _ C.int, _ **C.char, _ **C.char) C.int { return 0 }
  31. 55.

    // #cgo pkg-config: sqlite3 // #include <sqlite3.h> // int callback(void*,

    int, char**, char**); import "C" [...] var zErrMsg *C.char rc = C.sqlite3_exec(d.db, sqlStatement, (*[0]byte)(C.callback), unsafe.Pointer(nil), &zErrMsg) if rc != C.SQLITE_OK { log.Printf("SQL error: %s\n", C.GoString(zErrMsg)) C.sqlite3_free(unsafe.Pointer(zErrMsg)) }
  32. 56.

    // NOTE: it’s up to you to manage concurrency (i.e.

    with a RWMutex) var handles = make(map[uintptr]interface{}) // NOTE: use low numbers not to overlap with Go memory handles[uintptr(42)] = "some Go memory" rc = C.sqlite3_exec(d.db, sqlStatement, (*[0]byte)(C.callback), unsafe.Pointer(uintptr(42)), &zErrMsg) //export callback func callback(userData uintptr, _ C.int, _ **C.char, _ **C.char) C.int { goData := handles[userData] [...] } Example: https://github.com/mattn/go-sqlite3/blob/ 76e335f60bbcee20755df9864f0153af1a80ad2d/callback.go#L52
  33. 57.

    • Effortless memory management • Awesome tooling • Runtime speed

    • Quick compilation • Reproducible builds • Static binaries • go get • Super-easy cross compiling
  34. 58.

    package main // #cgo LDFLAGS: -lm // #include <math.h> //

    void slow_func() { // double a; int i; // for (i = 1; i < 1e8; i++) { // a = pow(a, 3); // } // } import "C" import "os" import "runtime/pprof" func main() { f, _ := os.Create("cgo.pprof") pprof.StartCPUProfile(f) C.slow_func() pprof.StopCPUProfile() }
  35. 59.
  36. 60.

    $ apt-get install linux-tools-generic $ go version go version go1.7rc1

    linux/amd64 $ perf record ./slow $ perf report
  37. 61.
  38. 62.

    • Effortless memory management • Awesome tooling • Runtime speed

    • Quick compilation • Reproducible builds • Static binaries • go get • Super-easy cross compiling
  39. 63.

    package main // #cgo LDFLAGS: -lm // #include <math.h> import

    "C" func main() { var a C.double for i := 1; i < 1e8; i++ { a = C.pow(a, 3) } }
  40. 64.

    package main // #cgo LDFLAGS: -lm // #include <math.h> //

    void powloop() { // double a; int i; // for (i = 1; i < 1e8; i++) { // a = pow(a, 3); // } // } import "C" func main() { C.powloop() }
  41. 65.

    • Effortless memory management • Awesome tooling • Runtime speed

    • Quick compilation • Reproducible builds • Static binaries • go get • Super-easy cross compiling
  42. 66.

    $ cat $GOPATH/src/github.com/FiloSottile/hashpass/main.go package main // #cgo pkg-config: libsodium //

    #include <sodium.h> import "C" func main() { C.sodium_init() println(hashPass("Correct Horse Battery Staple")) } func hashPass(pass string) string { cMem := C.calloc(1, C.crypto_pwhash_STRBYTES) defer C.free(cMem) res := (*C.char)(cMem) C.crypto_pwhash_str(res, C.CString(pass), (C.ulonglong)(len(pass)), C.crypto_pwhash_OPSLIMIT_SENSITIVE, C.crypto_pwhash_MEMLIMIT_SENSITIVE) return C.GoStringN(res, C.crypto_pwhash_STRBYTES) }
  43. 67.

    $ (cd libsodium-1.0.10 && ./configure && make install) $ go

    build github.com/FiloSottile/hashpass $ ./hashpass $argon2i$v=19$m=524288,t=8,p=1$ISqzZv/SaISgeOJTsWIc4Q $w5PTypPasNCkQhm6mnfMhYqpkoGXn0UCO42J6Qjrnko $ file hashpass hashpass: ELF 64-bit LSB executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /lib64/ld-linux-x86-64.so.2, for GNU/Linux 2.6.32, BuildID[sha1]=dbae930638bb69e5c461a33d1823906928bb3706, not stripped
  44. 68.

    $ go build -v -x github.com/FiloSottile/hashpass WORK=/tmp/go-build285911135 github.com/FiloSottile/hashpass mkdir -p

    $WORK/github.com/FiloSottile/hashpass/_obj/ mkdir -p $WORK/github.com/FiloSottile/hashpass/_obj/exe/ cd /Users/filippo/go/src/github.com/FiloSottile/hashpass pkg-config --cflags libsodium pkg-config --libs libsodium CGO_LDFLAGS="-g" "-O2" "-L/usr/local/lib" "-lsodium" /usr/local/go/pkg/tool/linux_amd64/cgo -objdir $WORK/github.com/FiloSottile/hashpass/ _obj/ -importpath github.com/FiloSottile/hashpass -- -I/usr/local/include -I $WORK/github.com/FiloSottile/hashpass/_obj/ main.go gcc -I . -fPIC -m64 -pthread -fmessage-length=0 -I/usr/local/include -I $WORK/github.com/FiloSottile/hashpass/_obj/ -g -O2 -o $WORK/ github.com/FiloSottile/hashpass/_obj/_cgo_main.o -c $WORK/github.com/FiloSottile/hashpass/_obj/_cgo_main.c gcc -I . -fPIC -m64 -pthread -fmessage-length=0 -I/usr/local/include -I $WORK/github.com/FiloSottile/hashpass/_obj/ -g -O2 -o $WORK/ github.com/FiloSottile/hashpass/_obj/_cgo_export.o -c $WORK/github.com/FiloSottile/hashpass/_obj/_cgo_export.c gcc -I . -fPIC -m64 -pthread -fmessage-length=0 -I/usr/local/include -I $WORK/github.com/FiloSottile/hashpass/_obj/ -g -O2 -o $WORK/ github.com/FiloSottile/hashpass/_obj/main.cgo2.o -c $WORK/github.com/FiloSottile/hashpass/_obj/main.cgo2.c gcc -I . -fPIC -m64 -pthread -fmessage-length=0 -o $WORK/github.com/FiloSottile/hashpass/_obj/_cgo_.o $WORK/github.com/FiloSottile/ hashpass/_obj/_cgo_main.o $WORK/github.com/FiloSottile/hashpass/_obj/_cgo_export.o $WORK/github.com/FiloSottile/hashpass/_obj/main.cgo2.o -g -O2 -L/usr/local/lib -lsodium /usr/local/go/pkg/tool/linux_amd64/cgo -objdir $WORK/github.com/FiloSottile/hashpass/_obj/ -dynpackage main -dynimport $WORK/github.com/ FiloSottile/hashpass/_obj/_cgo_.o -dynout $WORK/github.com/FiloSottile/hashpass/_obj/_cgo_import.go cd $WORK gcc -I . -fPIC -m64 -pthread -fmessage-length=0 -no-pie -c trivial.c cd /Users/filippo/go/src/github.com/FiloSottile/hashpass gcc -I . -fPIC -m64 -pthread -fmessage-length=0 -o $WORK/github.com/FiloSottile/hashpass/_obj/_all.o $WORK/github.com/FiloSottile/ hashpass/_obj/_cgo_export.o $WORK/github.com/FiloSottile/hashpass/_obj/main.cgo2.o -g -O2 -L/usr/local/lib -Wl,-r -nostdlib -Wl,--build- id=none /usr/local/go/pkg/tool/linux_amd64/compile -o $WORK/github.com/FiloSottile/hashpass.a -trimpath $WORK -p main -buildid ce5e1a0eab9fd96772b6a135be9ae6179cae3208 -D _/Users/filippo/go/src/github.com/FiloSottile/hashpass -I $WORK -pack $WORK/github.com/ FiloSottile/hashpass/_obj/_cgo_gotypes.go $WORK/github.com/FiloSottile/hashpass/_obj/main.cgo1.go $WORK/github.com/FiloSottile/hashpass/ _obj/_cgo_import.go pack r $WORK/github.com/FiloSottile/hashpass.a $WORK/github.com/FiloSottile/hashpass/_obj/_all.o # internal cd . /usr/local/go/pkg/tool/linux_amd64/link -o $WORK/github.com/FiloSottile/hashpass/_obj/exe/a.out -L $WORK -extld=gcc -buildmode=exe - buildid=ce5e1a0eab9fd96772b6a135be9ae6179cae3208 $WORK/github.com/FiloSottile/hashpass.a cp $WORK/github.com/FiloSottile/hashpass/_obj/exe/a.out hashpass
  45. 69.

    $ cat $GOPATH/src/github.com/FiloSottile/hashpass/sodium/hash.go package sodium // #cgo pkg-config: libsodium //

    #include <sodium.h> import "C" func init() { C.sodium_init() } func HashPass(pass string) string { cMem := C.calloc(1, C.crypto_pwhash_STRBYTES) defer C.free(cMem) res := (*C.char)(cMem) C.crypto_pwhash_str(res, C.CString(pass), (C.ulonglong)(len(pass)), C.crypto_pwhash_OPSLIMIT_SENSITIVE, C.crypto_pwhash_MEMLIMIT_SENSITIVE) return C.GoStringN(res, C.crypto_pwhash_STRBYTES) }
  46. 70.

    $ go build -v —i github.com/FiloSottile/hashpass github.com/FiloSottile/hashpass/sodium github.com/FiloSottile/hashpass $ go

    build -v -x github.com/FiloSottile/hashpass WORK=/tmp/go-build607855468 github.com/FiloSottile/hashpass mkdir -p $WORK/github.com/FiloSottile/hashpass/_obj/ mkdir -p $WORK/github.com/FiloSottile/hashpass/_obj/exe/ cd /Users/filippo/go/src/github.com/FiloSottile/hashpass /usr/local/go/pkg/tool/linux_amd64/compile -o $WORK/github.com/FiloSottile/ hashpass.a -trimpath $WORK -p main -complete -buildid 099121d2abae4b427870cba7785d01725b6dca64 -D _/Users/filippo/go/src/github.com/ FiloSottile/hashpass -I $WORK -I /Users/filippo/go/pkg/linux_amd64 -pack ./main.go cd . /usr/local/go/pkg/tool/linux_amd64/link -o $WORK/github.com/FiloSottile/hashpass/ _obj/exe/a.out -L $WORK -L /Users/filippo/go/pkg/linux_amd64 -extld=gcc - buildmode=exe -buildid=099121d2abae4b427870cba7785d01725b6dca64 $WORK/github.com/ FiloSottile/hashpass.a cp $WORK/github.com/FiloSottile/hashpass/_obj/exe/a.out hashpass
  47. 71.

    • Effortless memory management • Awesome tooling • Runtime speed

    • Quick compilation • Reproducible builds • Static binaries • go get • Super-easy cross compiling
  48. 72.

    $ mv libsodium-1.0.10 \ $GOPATH/src/github.com/FiloSottile/hashpass/sodium/_libsodium-1.0.10 $ head $GOPATH/src/github.com/FiloSottile/hashpass/sodium/hash.go package sodium

    // #cgo CFLAGS: -I${SRCDIR}/_libsodium-1.0.10/src/libsodium/include/ // #cgo LDFLAGS: -L${SRCDIR}/_libsodium-1.0.10/src/libsodium/.libs/ -lsodium // #include <sodium.h> import "C" $ (cd $GOPATH/src/github.com/FiloSottile/hashpass/sodium/_libsodium-1.0.10 && \ ./configure && make) $ go build github.com/FiloSottile/hashpass
  49. 73.

    • Effortless memory management • Awesome tooling • Runtime speed

    • Quick compilation • Reproducible builds • Static binaries • go get • Super-easy cross compiling
  50. 74.

    $ go build -ldflags="-extldflags -static" github.com/FiloSottile/hashpass $ file hashpass hashpass:

    ELF 64-bit LSB executable, x86-64, version 1 (GNU/Linux), statically linked, for GNU/Linux 2.6.32, BuildID[sha1]=b0b0ccdbb3860c12c49e8034ce8fd854b9943eb3, not stripped
  51. 75.

    $ apt-get install musl-tools $ export CC=musl-gcc $ (cd $GOPATH/src/github.com/FiloSottile/hashpass/sodium/_libsodium-1.0.10

    && \ make clean && ./configure && make) $ go install -installsuffix musl github.com/FiloSottile/hashpass/sodium $ go build -installsuffix musl -ldflags="-extldflags -static" github.com/ FiloSottile/hashpass $ file hashpass hashpass: ELF 64-bit LSB executable, x86-64, version 1 (SYSV), statically linked, not stripped
  52. 77.

    • Effortless memory management • Awesome tooling • Runtime speed

    • Quick compilation • Reproducible builds • Static binaries • go get • Super-easy cross compiling
  53. 78.

    $ cd $GOPATH/src/github.com/FiloSottile/hashpass/sodium $ (cd _libsodium-1.0.10 && ./configure && make)

    $ mkdir _linux_amd64 $ cp _libsodium-1.0.10/src/libsodium/include/sodium{,.h} _linux_amd64 $ cp _libsodium-1.0.10/src/libsodium/.libs/*.a _linux_amd64 $ head $GOPATH/src/github.com/FiloSottile/hashpass/sodium/hash.go package sodium // #cgo linux,amd64 CFLAGS: -I${SRCDIR}/_linux_amd64/ // #cgo linux,amd64 LDFLAGS: -L${SRCDIR}/_linux_amd64/ // #cgo !linux !amd64 CFLAGS: -I${SRCDIR}/_libsodium-1.0.10/src/libsodium/include/ // #cgo !linux !amd64 LDFLAGS: -L${SRCDIR}/_libsodium-1.0.10/src/libsodium/.libs/ // #cgo LDFLAGS: -lsodium // #include <sodium.h> import “C" $ go get github.com/FiloSottile/hashpass
  54. 79.

    • Effortless memory management • Awesome tooling • Runtime speed

    • Quick compilation • Reproducible builds • Static binaries • go get • Super-easy cross compiling
  55. 80.

    musl based OS X → Linux cross-compilers brew install https://gist.github.com/FiloSottile/

    01d2bbfaf63ae1b6e373e6bc874fefc6/raw/ f74e34dbf2823e953af28c6b77b7a5139a4f2876/musl-cross.rb $ CC=x86_64-linux-musl-cc CGO_ENABLED=1 GOOS=linux go build -i \ -ldflags '-extldflags -static' github.com/FiloSottile/hashpass
  56. 81.

    • Effortless memory management • Runtime speed • Awesome tooling

    • Fast builds • Static binaries • Reproducible builds • go get • Super-easy cross compiling
  57. 82.

    Questions? Filippo Valsorda @FiloSottile filippo@cloudflare.com Olga Shalakhina artwork under CC

    3.0 license based on Renee French under Creative Commons 3.0 Attributions.