From cgo back to Go - GopherCon 2016

As soon as you import "C", a lot of the things we love about Go are gone.

This talk helps you walk your way back up from the bottom of that pit.

https://www.youtube.com/watch?v=lhMhApWQp2E

Transcript

1. From cgo back to Go Filippo Valsorda
2. “cgo is not Go” —Dave Cheney http://dave.cheney.net/2016/01/18/cgo-is-not-go
3. • Effortless memory management • Runtime speed • Awesome tooling • Fast builds • Static binaries • Reproducible builds • go get • Super-easy cross compiling
4. Then you import “C”
5. • Effortless memory management • Awesome tooling • Runtime speed • Quick compilation • Reproducible builds • Static binaries • go get • Super-easy cross compiling
6. • Effortless memory management • Awesome tooling • Runtime speed • Quick compilation • Reproducible builds • Static binaries • go get • Super-easy cross compiling
7. Go memory C memory GC
8. Go memory C memory GC make([]byte, 1024) &MyStruct{}
9. Go memory C memory GC []byte *MyStruct
10. Go memory C memory GC []byte
11. Go memory C memory GC []byte
12. Go memory C memory GC []byte *uint8_t C.some_func()
13. Go memory C memory GC []byte
15. Go memory C memory GC []byte *uint8_t C.some_func()
16. Go memory C memory GC []byte *uint8_t
17. Go memory C memory GC *uint8_t
18. Go memory C memory GC *uint8_t
21. // typedef struct { // int a, b, c; // } someStruct; // // someStruct *someReference; // void storeReference(someStruct *s) { // someReference = s; // } // int retrieveReference() { // return someReference->a; // } import "C" import "runtime" func main() { passSomeMemory() runtime.GC() println(getSomeMemory()) } func passSomeMemory() { someGoMemory := &C.someStruct{ a: 1, b: 2, c: 3, } C.storeReference(someGoMemory) } func getSomeMemory() int { return int(C.retrieveReference()) } Output: 1
22. Output: 1 // typedef struct { // int a, b, c; // } someStruct; // // someStruct *someReference; // void storeReference(someStruct *s) { // someReference = s; // } // int retrieveReference() { // return someReference->a; // } import "C" import "runtime" func main() { passSomeMemory() runtime.GC() println(getSomeMemory()) } func passSomeMemory() { someGoMemory := &C.someStruct{ a: 1, b: 2, c: 3, } C.storeReference(someGoMemory) } func getSomeMemory() int { return int(C.retrieveReference()) } 537247792
23. The cgo rules You may pass a Go pointer … if it doesn’t point to other pointers … and C can’t keep a reference to it
24. The GC must see all the Go pointers.
27. // #include <stdlib.h> // // typedef struct { // int a, b, c; // } someStruct; // // void processStruct(someStruct *s) {} import "C" func ProcessStruct() { someCMemory := C.calloc(1, C.sizeof_someStruct) defer C.free(someCMemory) s := (*C.someStruct)(someCMemory) C.processStruct(s) }
28. // #include <stdlib.h> // // typedef struct { // int a, b, c; // } someStruct; // // void processStruct(someStruct *s) {} import "C" func ProcessStruct() { someCMemory := C.calloc(1, C.sizeof_someStruct) defer C.free(someCMemory) s := (*C.someStruct)(someCMemory) C.processStruct(s) }
29. Copy. Copy. Copy.
30. // #include <stdlib.h> // // typedef struct { // int a, b, c; // char *str; // } someStruct; // // void processStruct(someStruct *s) {} import "C" import "unsafe" func ProcessStruct(str string) { s := (*C.someStruct)(C.calloc(1, C.sizeof_someStruct)) defer C.free(unsafe.Pointer(s)) s.str = C.CString(str) defer C.free(unsafe.Pointer(s.str)) C.processStruct(s) }
31. // #include <stdlib.h> // #include <stdint.h> // #include <string.h> // typedef struct { uint8_t *data; size_t data_len; } someStruct; // void processStruct(someStruct *s) {} import "C" import "unsafe" func ProcessStruct(data []byte) { s := (*C.someStruct)(C.calloc(1, C.sizeof_someStruct)) defer C.free(unsafe.Pointer(s)) cMem := C.calloc(1, C.size_t(len(data))) defer C.free(cMem) s.data = (*C.uint8_t)(C.memmove( cMem, unsafe.Pointer(&data[0]), C.size_t(len(data)))) s.data_len = C.size_t(len(data)) C.processStruct(s) }
32. // #include <stdlib.h> // #include <stdint.h> // #include <string.h> // // typedef struct { // uint8_t *data; // size_t data_len; // } someStruct; // // void processStruct(someStruct *s) {} import "C" import "unsafe" type SomeStruct struct { Data []byte } func ProcessStruct(s *SomeStruct) { [...] }
33. // #cgo pkg-config: sqlite3 // #include <sqlite3.h> import "C" import "errors" type DB struct { db *C.sqlite3 } func Open(filename string) (d *DB, err error) { d = &DB{} if C.sqlite3_open(C.CString(filename), &d.db) != C.SQLITE_OK { err = errors.New(C.GoString(C.sqlite3_errmsg(d.db))) C.sqlite3_close(d.db) } return } func (d *DB) Close() { C.sqlite3_close(d.db) } Wrapper types
34. The arena pattern
35. type arena []unsafe.Pointer func (a *arena) calloc(count, size int) unsafe.Pointer { ptr := C.calloc(C.size_t(count), C.size_t(size)) *a = append(*a, ptr) return ptr } func (a *arena) free() { for _, ptr := range *a { C.free(ptr) } }
36. func ProcessStruct(s *SomeStruct) { ar := &arena{} defer ar.free() cs := (*C.someStruct)(ar.calloc(C.sizeof_someStruct, 1)) cMem := ar.calloc(1, len(s.Data)) cs.data = (*C.uint8_t)(C.memmove(cMem, unsafe.Pointer(&s.Data[0]), C.size_t(len(s.Data)))) cs.data_len = C.size_t(len(s.Data)) C.processStruct(cs) }
37. type arena []unsafe.Pointer func (a *arena) calloc(count, size int) unsafe.Pointer { ptr := C.calloc(C.size_t(count), C.size_t(size)) *a = append(*a, ptr) return ptr } func (a *arena) copy(data []byte) unsafe.Pointer { ptr := a.calloc(1, len(data)) C.memmove(ptr, unsafe.Pointer(&data[0]), C.size_t(len(data))) return ptr } func (a *arena) free() { for _, ptr := range *a { C.free(ptr) } }
38. func ProcessStruct(s *SomeStruct) { ar := &arena{} defer ar.free() cs := (*C.someStruct)(ar.calloc(C.sizeof_someStruct, 1)) cs.data = (*C.uint8_t)(ar.copy(s.Data)) cs.data_len = C.size_t(len(s.Data)) C.processStruct(cs) }
39. If you really have to…
40. // #include <stdlib.h> // #include <stdint.h> // // void processBigData(uint8_t *data, size_t data_len) {} import "C" func ProcessBigData(bigData []byte) { cBigData = (*C.uint8_t)(&bigData[0]) cBigDataLen = C.size_t(len(bigData)) C.processBigData(cBigData, cBigDataLen) }
41. Go memory C memory GC []byte
42. Go memory C memory GC []byte *uint8_t C.some_func()
43. Go memory C memory GC []byte
45. // typedef struct { // int a, b, c; // } someStruct; // // typedef struct { // someStruct *x, *y; // } complexStruct; // // void processStruct(complexStruct *s) {} import "C" func main() { someGoMemory := &C.someStruct{ a: 1, b: 2, c: 3, } C.processStruct(&C.complexStruct{ x: someGoMemory, y: nil, }) }
48. panic: runtime error: cgo argument has Go pointer to Go pointer
49. GODEBUG=cgocheck=2
50. go doc cmd/cgo
51. Callbacks
52. int sqlite3_exec( sqlite3*, /* An open database */ const char *sql, /* SQL to be evaluated */ int (*callback)(void*,int,char**,char**), /* Callback function */ void *, /* 1st argument to callback */ char **errmsg /* Error msg written here */ );
53. var zErrMsg *C.char rc = C.sqlite3_exec(d.db, sqlStatement, ???, ???, &zErrMsg) if rc != C.SQLITE_OK { log.Printf("SQL error: %s\n", C.GoString(zErrMsg)) C.sqlite3_free(unsafe.Pointer(zErrMsg)) }
54. package main import "C" import "unsafe" //export callback func callback(userData uintptr, _ C.int, _ **C.char, _ **C.char) C.int { return 0 }
55. // #cgo pkg-config: sqlite3 // #include <sqlite3.h> // int callback(void*, int, char**, char**); import "C" [...] var zErrMsg *C.char rc = C.sqlite3_exec(d.db, sqlStatement, (*[0]byte)(C.callback), unsafe.Pointer(nil), &zErrMsg) if rc != C.SQLITE_OK { log.Printf("SQL error: %s\n", C.GoString(zErrMsg)) C.sqlite3_free(unsafe.Pointer(zErrMsg)) }
56. // NOTE: it’s up to you to manage concurrency (i.e. with a RWMutex) var handles = make(map[uintptr]interface{}) // NOTE: use low numbers not to overlap with Go memory handles[uintptr(42)] = "some Go memory" rc = C.sqlite3_exec(d.db, sqlStatement, (*[0]byte)(C.callback), unsafe.Pointer(uintptr(42)), &zErrMsg) //export callback func callback(userData uintptr, _ C.int, _ **C.char, _ **C.char) C.int { goData := handles[userData] [...] } Example: https://github.com/mattn/go-sqlite3/blob/ 76e335f60bbcee20755df9864f0153af1a80ad2d/callback.go#L52
57. • Effortless memory management • Awesome tooling • Runtime speed • Quick compilation • Reproducible builds • Static binaries • go get • Super-easy cross compiling
58. package main // #cgo LDFLAGS: -lm // #include <math.h> // void slow_func() { // double a; int i; // for (i = 1; i < 1e8; i++) { // a = pow(a, 3); // } // } import "C" import "os" import "runtime/pprof" func main() { f, _ := os.Create("cgo.pprof") pprof.StartCPUProfile(f) C.slow_func() pprof.StopCPUProfile() }
60. $ apt-get install linux-tools-generic $ go version go version go1.7rc1 linux/amd64 $ perf record ./slow $ perf report
62. • Effortless memory management • Awesome tooling • Runtime speed • Quick compilation • Reproducible builds • Static binaries • go get • Super-easy cross compiling
63. package main // #cgo LDFLAGS: -lm // #include <math.h> import "C" func main() { var a C.double for i := 1; i < 1e8; i++ { a = C.pow(a, 3) } }
64. package main // #cgo LDFLAGS: -lm // #include <math.h> // void powloop() { // double a; int i; // for (i = 1; i < 1e8; i++) { // a = pow(a, 3); // } // } import "C" func main() { C.powloop() }
65. • Effortless memory management • Awesome tooling • Runtime speed • Quick compilation • Reproducible builds • Static binaries • go get • Super-easy cross compiling
66. $ cat $GOPATH/src/github.com/FiloSottile/hashpass/main.go package main // #cgo pkg-config: libsodium // #include <sodium.h> import "C" func main() { C.sodium_init() println(hashPass("Correct Horse Battery Staple")) } func hashPass(pass string) string { cMem := C.calloc(1, C.crypto_pwhash_STRBYTES) defer C.free(cMem) res := (*C.char)(cMem) C.crypto_pwhash_str(res, C.CString(pass), (C.ulonglong)(len(pass)), C.crypto_pwhash_OPSLIMIT_SENSITIVE, C.crypto_pwhash_MEMLIMIT_SENSITIVE) return C.GoStringN(res, C.crypto_pwhash_STRBYTES) }
67. $ (cd libsodium-1.0.10 && ./configure && make install) $ go build github.com/FiloSottile/hashpass $ ./hashpass $argon2i$v=19$m=524288,t=8,p=1$ISqzZv/SaISgeOJTsWIc4Q $w5PTypPasNCkQhm6mnfMhYqpkoGXn0UCO42J6Qjrnko $ file hashpass hashpass: ELF 64-bit LSB executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /lib64/ld-linux-x86-64.so.2, for GNU/Linux 2.6.32, BuildID[sha1]=dbae930638bb69e5c461a33d1823906928bb3706, not stripped
68. $ go build -v -x github.com/FiloSottile/hashpass WORK=/tmp/go-build285911135 github.com/FiloSottile/hashpass mkdir -p $WORK/github.com/FiloSottile/hashpass/_obj/ mkdir -p $WORK/github.com/FiloSottile/hashpass/_obj/exe/ cd /Users/filippo/go/src/github.com/FiloSottile/hashpass pkg-config --cflags libsodium pkg-config --libs libsodium CGO_LDFLAGS="-g" "-O2" "-L/usr/local/lib" "-lsodium" /usr/local/go/pkg/tool/linux_amd64/cgo -objdir $WORK/github.com/FiloSottile/hashpass/ _obj/ -importpath github.com/FiloSottile/hashpass -- -I/usr/local/include -I $WORK/github.com/FiloSottile/hashpass/_obj/ main.go gcc -I . -fPIC -m64 -pthread -fmessage-length=0 -I/usr/local/include -I $WORK/github.com/FiloSottile/hashpass/_obj/ -g -O2 -o $WORK/ github.com/FiloSottile/hashpass/_obj/_cgo_main.o -c $WORK/github.com/FiloSottile/hashpass/_obj/_cgo_main.c gcc -I . -fPIC -m64 -pthread -fmessage-length=0 -I/usr/local/include -I $WORK/github.com/FiloSottile/hashpass/_obj/ -g -O2 -o $WORK/ github.com/FiloSottile/hashpass/_obj/_cgo_export.o -c $WORK/github.com/FiloSottile/hashpass/_obj/_cgo_export.c gcc -I . -fPIC -m64 -pthread -fmessage-length=0 -I/usr/local/include -I $WORK/github.com/FiloSottile/hashpass/_obj/ -g -O2 -o $WORK/ github.com/FiloSottile/hashpass/_obj/main.cgo2.o -c $WORK/github.com/FiloSottile/hashpass/_obj/main.cgo2.c gcc -I . -fPIC -m64 -pthread -fmessage-length=0 -o $WORK/github.com/FiloSottile/hashpass/_obj/_cgo_.o $WORK/github.com/FiloSottile/ hashpass/_obj/_cgo_main.o $WORK/github.com/FiloSottile/hashpass/_obj/_cgo_export.o $WORK/github.com/FiloSottile/hashpass/_obj/main.cgo2.o -g -O2 -L/usr/local/lib -lsodium /usr/local/go/pkg/tool/linux_amd64/cgo -objdir $WORK/github.com/FiloSottile/hashpass/_obj/ -dynpackage main -dynimport $WORK/github.com/ FiloSottile/hashpass/_obj/_cgo_.o -dynout $WORK/github.com/FiloSottile/hashpass/_obj/_cgo_import.go cd $WORK gcc -I . -fPIC -m64 -pthread -fmessage-length=0 -no-pie -c trivial.c cd /Users/filippo/go/src/github.com/FiloSottile/hashpass gcc -I . -fPIC -m64 -pthread -fmessage-length=0 -o $WORK/github.com/FiloSottile/hashpass/_obj/_all.o $WORK/github.com/FiloSottile/ hashpass/_obj/_cgo_export.o $WORK/github.com/FiloSottile/hashpass/_obj/main.cgo2.o -g -O2 -L/usr/local/lib -Wl,-r -nostdlib -Wl,--build- id=none /usr/local/go/pkg/tool/linux_amd64/compile -o $WORK/github.com/FiloSottile/hashpass.a -trimpath $WORK -p main -buildid ce5e1a0eab9fd96772b6a135be9ae6179cae3208 -D _/Users/filippo/go/src/github.com/FiloSottile/hashpass -I $WORK -pack $WORK/github.com/ FiloSottile/hashpass/_obj/_cgo_gotypes.go $WORK/github.com/FiloSottile/hashpass/_obj/main.cgo1.go $WORK/github.com/FiloSottile/hashpass/ _obj/_cgo_import.go pack r $WORK/github.com/FiloSottile/hashpass.a $WORK/github.com/FiloSottile/hashpass/_obj/_all.o # internal cd . /usr/local/go/pkg/tool/linux_amd64/link -o $WORK/github.com/FiloSottile/hashpass/_obj/exe/a.out -L $WORK -extld=gcc -buildmode=exe - buildid=ce5e1a0eab9fd96772b6a135be9ae6179cae3208 $WORK/github.com/FiloSottile/hashpass.a cp $WORK/github.com/FiloSottile/hashpass/_obj/exe/a.out hashpass
69. $ cat $GOPATH/src/github.com/FiloSottile/hashpass/sodium/hash.go package sodium // #cgo pkg-config: libsodium // #include <sodium.h> import "C" func init() { C.sodium_init() } func HashPass(pass string) string { cMem := C.calloc(1, C.crypto_pwhash_STRBYTES) defer C.free(cMem) res := (*C.char)(cMem) C.crypto_pwhash_str(res, C.CString(pass), (C.ulonglong)(len(pass)), C.crypto_pwhash_OPSLIMIT_SENSITIVE, C.crypto_pwhash_MEMLIMIT_SENSITIVE) return C.GoStringN(res, C.crypto_pwhash_STRBYTES) }
70. $ go build -v —i github.com/FiloSottile/hashpass github.com/FiloSottile/hashpass/sodium github.com/FiloSottile/hashpass $ go build -v -x github.com/FiloSottile/hashpass WORK=/tmp/go-build607855468 github.com/FiloSottile/hashpass mkdir -p $WORK/github.com/FiloSottile/hashpass/_obj/ mkdir -p $WORK/github.com/FiloSottile/hashpass/_obj/exe/ cd /Users/filippo/go/src/github.com/FiloSottile/hashpass /usr/local/go/pkg/tool/linux_amd64/compile -o $WORK/github.com/FiloSottile/ hashpass.a -trimpath $WORK -p main -complete -buildid 099121d2abae4b427870cba7785d01725b6dca64 -D _/Users/filippo/go/src/github.com/ FiloSottile/hashpass -I $WORK -I /Users/filippo/go/pkg/linux_amd64 -pack ./main.go cd . /usr/local/go/pkg/tool/linux_amd64/link -o $WORK/github.com/FiloSottile/hashpass/ _obj/exe/a.out -L $WORK -L /Users/filippo/go/pkg/linux_amd64 -extld=gcc - buildmode=exe -buildid=099121d2abae4b427870cba7785d01725b6dca64 $WORK/github.com/ FiloSottile/hashpass.a cp $WORK/github.com/FiloSottile/hashpass/_obj/exe/a.out hashpass
71. • Effortless memory management • Awesome tooling • Runtime speed • Quick compilation • Reproducible builds • Static binaries • go get • Super-easy cross compiling
72. $ mv libsodium-1.0.10 \ $GOPATH/src/github.com/FiloSottile/hashpass/sodium/_libsodium-1.0.10 $ head $GOPATH/src/github.com/FiloSottile/hashpass/sodium/hash.go package sodium // #cgo CFLAGS: -I${SRCDIR}/_libsodium-1.0.10/src/libsodium/include/ // #cgo LDFLAGS: -L${SRCDIR}/_libsodium-1.0.10/src/libsodium/.libs/ -lsodium // #include <sodium.h> import "C" $ (cd $GOPATH/src/github.com/FiloSottile/hashpass/sodium/_libsodium-1.0.10 && \ ./configure && make) $ go build github.com/FiloSottile/hashpass
73. • Effortless memory management • Awesome tooling • Runtime speed • Quick compilation • Reproducible builds • Static binaries • go get • Super-easy cross compiling
74. $ go build -ldflags="-extldflags -static" github.com/FiloSottile/hashpass $ file hashpass hashpass: ELF 64-bit LSB executable, x86-64, version 1 (GNU/Linux), statically linked, for GNU/Linux 2.6.32, BuildID[sha1]=b0b0ccdbb3860c12c49e8034ce8fd854b9943eb3, not stripped
75. $ apt-get install musl-tools $ export CC=musl-gcc $ (cd $GOPATH/src/github.com/FiloSottile/hashpass/sodium/_libsodium-1.0.10 && \ make clean && ./configure && make) $ go install -installsuffix musl github.com/FiloSottile/hashpass/sodium $ go build -installsuffix musl -ldflags="-extldflags -static" github.com/ FiloSottile/hashpass $ file hashpass hashpass: ELF 64-bit LSB executable, x86-64, version 1 (SYSV), statically linked, not stripped
76. os/user net crypto/x509
77. • Effortless memory management • Awesome tooling • Runtime speed • Quick compilation • Reproducible builds • Static binaries • go get • Super-easy cross compiling
78. $ cd $GOPATH/src/github.com/FiloSottile/hashpass/sodium $ (cd _libsodium-1.0.10 && ./configure && make) $ mkdir _linux_amd64 $ cp _libsodium-1.0.10/src/libsodium/include/sodium{,.h} _linux_amd64 $ cp _libsodium-1.0.10/src/libsodium/.libs/*.a _linux_amd64 $ head $GOPATH/src/github.com/FiloSottile/hashpass/sodium/hash.go package sodium // #cgo linux,amd64 CFLAGS: -I${SRCDIR}/_linux_amd64/ // #cgo linux,amd64 LDFLAGS: -L${SRCDIR}/_linux_amd64/ // #cgo !linux !amd64 CFLAGS: -I${SRCDIR}/_libsodium-1.0.10/src/libsodium/include/ // #cgo !linux !amd64 LDFLAGS: -L${SRCDIR}/_libsodium-1.0.10/src/libsodium/.libs/ // #cgo LDFLAGS: -lsodium // #include <sodium.h> import “C" $ go get github.com/FiloSottile/hashpass
79. • Effortless memory management • Awesome tooling • Runtime speed • Quick compilation • Reproducible builds • Static binaries • go get • Super-easy cross compiling
80. musl based OS X → Linux cross-compilers brew install https://gist.github.com/FiloSottile/ 01d2bbfaf63ae1b6e373e6bc874fefc6/raw/ f74e34dbf2823e953af28c6b77b7a5139a4f2876/musl-cross.rb $ CC=x86_64-linux-musl-cc CGO_ENABLED=1 GOOS=linux go build -i \ -ldflags '-extldflags -static' github.com/FiloSottile/hashpass
81. • Effortless memory management • Runtime speed • Awesome tooling • Fast builds • Static binaries • Reproducible builds • go get • Super-easy cross compiling
82. Questions? Filippo Valsorda @FiloSottile filippo@cloudflare.com Olga Shalakhina artwork under CC 3.0 license based on Renee French under Creative Commons 3.0 Attributions.