Tweet
Tweet

Slide 1

Slide 1 text

CHANNELING FAILURE @mattheath

Slide 2

Slide 2 text

S TA R L I N G

Slide 3

Slide 3 text

No content

Slide 4

Slide 4 text

No content

Slide 5

Slide 5 text

No content

Slide 6

Slide 6 text

No content

Slide 7

Slide 7 text

if err != nil { // save me }

Slide 8

Slide 8 text

err := pleaseWork() if err != nil { // save me }

Slide 9

Slide 9 text

if err := pleaseWork(); err != nil { // save me }

Slide 10

Slide 10 text

if err := pleaseWork(); err != nil { // pass this on? return err }

Slide 11

Slide 11 text

if err := pleaseWork(); err != nil { // maybe this? return nil, err }

Slide 12

Slide 12 text

if err := pleaseWork(); err != nil { // or this? return "", err }

Slide 13

Slide 13 text

return fmt.Errorf("failed to load url %s", url)

Slide 14

Slide 14 text

return fmt.Errorf("failed to load url %s: %s", url, err)

Slide 15

Slide 15 text

url := "http://beesbeesbees.com" resp, err := http.Get(url) if err != nil { return fmt.Errorf("failed to load url %s: %s", url, err) }

Slide 16

Slide 16 text

return fmt.Errorf("failed to load url %s: %s", url, err)

Slide 17

Slide 17 text

return fmt.Errorf("timeout")

Slide 18

Slide 18 text

return errors.New("timeout")

Slide 19

Slide 19 text

var ErrTimeout = errors.New("timeout") var ErrNotFound = errors.New("not found") var ErrTooManyCooks = errors.New("spoil the broth") return errors.New("timeout")

Slide 20

Slide 20 text

var ErrTimeout = errors.New("timeout") var ErrNotFound = errors.New("not found") var ErrTooManyCooks = errors.New("spoil the broth") return ErrTimeout

Slide 21

Slide 21 text

if err := loadBees(); err != nil { // maybe this is ok? return err }

Slide 22

Slide 22 text

if err := loadBees(); err != nil { switch err { case ErrTimeout: // retry? case ErrNotFound: // abort! } }

Slide 23

Slide 23 text

if err := loadBees(); err != nil { switch err { case ErrTimeout: // retry? case ErrNotFound: return err } }

Slide 24

Slide 24 text

if err := loadBees(); err != nil { switch err { case ErrTimeout: // retry? case ErrNotFound: return err default: return err } }

Slide 25

Slide 25 text

if err := loadBees(); err != nil { switch err { case ErrTimeout: // retry? default: return err } }

Slide 26

Slide 26 text

for { if err := loadBees(); err != nil { switch err { case ErrTimeout: continue // retry...? default: return err } } return nil }

Slide 27

Slide 27 text

for { if err := loadBees(); err != nil { switch err { case ErrTimeout: continue // retry...? default: return err } } return nil }

Slide 28

Slide 28 text

for { if err := loadBees(); err != nil { switch err { case ErrTimeout: // retry...? // backoff // increasing timeouts // limits on this default: return err

Slide 29

Slide 29 text

ErrTimeout?

Slide 30

Slide 30 text

package httpclient import ( "net" "net/http" "time" ) func TimeoutDialer(cTimeout time.Duration, rwTimeout time.Duration) func(netw, addr string)
 (c net.Conn, err error) { return func(netw, addr string) (net.Conn, error) { conn, err := net.DialTimeout(netw, addr, cTimeout) if err != nil { return nil, err } conn.SetDeadline(time.Now().Add(rwTimeout)) return conn, nil } } func NewTimeoutClient(connectTimeout time.Duration, readWriteTimeout time.Duration) *http.Client { return &http.Client{ Transport: &http.Transport{ Dial: TimeoutDialer(connectTimeout, readWriteTimeout), }, } }

Slide 31

Slide 31 text

package httpclient import ( "net" "net/http" "time" ) func TimeoutDialer(cTimeout time.Duration, rwTimeout time.Duration) func(netw, addr string)
 (c net.Conn, err error) { return func(netw, addr string) (net.Conn, error) { conn, err := net.DialTimeout(netw, addr, cTimeout) if err != nil { return nil, err } conn.SetDeadline(time.Now().Add(rwTimeout)) return conn, nil } } func NewTimeoutClient(connectTimeout time.Duration, readWriteTimeout time.Duration) *http.Client { return &http.Client{ Transport: &http.Transport{ Dial: TimeoutDialer(connectTimeout, readWriteTimeout), }, } }

Slide 32

Slide 32 text

package httpclient import ( "net" "net/http" "time" ) func TimeoutDialer(cTimeout time.Duration, rwTimeout time.Duration) func(netw, addr string)
 (c net.Conn, err error) { return func(netw, addr string) (net.Conn, error) { conn, err := net.DialTimeout(netw, addr, cTimeout) if err != nil { return nil, err } conn.SetDeadline(time.Now().Add(rwTimeout)) return conn, nil } } func NewTimeoutClient(connectTimeout time.Duration, readWriteTimeout time.Duration) *http.Client { return &http.Client{ Transport: &http.Transport{ Dial: TimeoutDialer(connectTimeout, readWriteTimeout), }, } }

Slide 33

Slide 33 text

package httpclient import ( "net" "net/http" "time" ) func TimeoutDialer(cTimeout time.Duration, rwTimeout time.Duration) func(netw, addr string)
 (c net.Conn, err error) { return func(netw, addr string) (net.Conn, error) { conn, err := net.DialTimeout(netw, addr, cTimeout) if err != nil { return nil, err } conn.SetDeadline(time.Now().Add(rwTimeout)) return conn, nil } } func NewTimeoutClient(connectTimeout time.Duration, readWriteTimeout time.Duration) *http.Client { return &http.Client{ Transport: &http.Transport{ Dial: TimeoutDialer(connectTimeout, readWriteTimeout), }, } }

Slide 34

Slide 34 text

package httpclient import ( "net" "net/http" "time" ) func TimeoutDialer(cTimeout time.Duration, rwTimeout time.Duration) func(netw, addr string)
 (c net.Conn, err error) { return func(netw, addr string) (net.Conn, error) { conn, err := net.DialTimeout(netw, addr, cTimeout) if err != nil { return nil, err } conn.SetDeadline(time.Now().Add(rwTimeout)) return conn, nil } } func NewTimeoutClient(connectTimeout time.Duration, readWriteTimeout time.Duration) *http.Client { return &http.Client{ Transport: &http.Transport{ Dial: TimeoutDialer(connectTimeout, readWriteTimeout), }, } }

Slide 35

Slide 35 text

ErrTimeout?

Slide 36

Slide 36 text

func loadBees() []bee {} // Blocking bees := loadBees()

Slide 37

Slide 37 text

func loadBees(chan []bee) {} beeChan := make(chan []bee) go loadBees(beeChan) // Still blocking bees := <-beeChan

Slide 38

Slide 38 text

func loadBees(chan []bee) {} beeChan := make(chan []bee) go loadBees(beeChan) // Blocks until timeout select { case bees := <-beeChan: case <-time.After(1 * time.Second): }

Slide 39

Slide 39 text

func loadBees(chan []bee) {} beeChan := make(chan []bee) go loadBees(beeChan) // Blocks until timeout select { case bees := <-beeChan: case <-time.After(1 * time.Second): return ErrTimeout }

Slide 40

Slide 40 text

func loadBees(chan []bee) {} beeChan := make(chan []bee) go loadBees(beeChan) // Blocks until timeout select { case bees := <-beeChan: case <-time.After(1 * time.Second): return ErrTimeout }

Slide 41

Slide 41 text

func loadBees(chan []bee) {} beeChan := make(chan []bee) go loadBees(beeChan) // Blocks until timeout select { case bees := <-beeChan: // errors? case <-time.After(1 * time.Second): return ErrTimeout }

Slide 42

Slide 42 text

func loadBees(chan []bee) {} beeChan := make(chan []bee) go loadBees(beeChan) // On timeout, writer is blocked select { case bees := <-beeChan: case <-time.After(1 * time.Second): return ErrTimeout }

Slide 43

Slide 43 text

func loadBees(chan []bee) {} beeChan := make(chan []bee, 1) go loadBees(beeChan) // On timeout, writer not blocked select { case bees := <-beeChan: case <-time.After(1 * time.Second): return ErrTimeout }

Slide 44

Slide 44 text

connect()

Slide 45

Slide 45 text

if err := connect(); err != nil { // retry }

Slide 46

Slide 46 text

for { if err := connect(); err != nil { // retry } }

Slide 47

Slide 47 text

for { if err := connect(); err != nil { // retry, with backoff } }

Slide 48

Slide 48 text

for { if err := connect(); err != nil { // retry, with backoff } err := <-Conn.NotifyClose() }

Slide 49

Slide 49 text

for { if err := connect(); err != nil { // retry, with backoff } select { case err := <-Conn.NotifyClose(): } }

Slide 50

Slide 50 text

for { if err := connect(); err != nil { // retry, with backoff } select { case err := <-Conn.NotifyClose(): case <-configChange: // disconnect } }

Slide 51

Slide 51 text

for { if err := connect(); err != nil { // retry, with backoff } select { case err := <-Conn.NotifyClose(): case <-quit: return } }

Slide 52

Slide 52 text

quit := make(chan struct{}) close(quit) select { case err := <-Conn.NotifyClose(): case <-quit: return // channel closed }

Slide 53

Slide 53 text

select { case <-configChange: // teardown case <-quit: return }

Slide 54

Slide 54 text

select { case <-lockExpired: // oh crap, teardown! case <-configChange: // teardown case <-quit: return }

Slide 55

Slide 55 text

select { case <-lockExpired: // oh crap, teardown! case <-configChange: // teardown case <-tomb.Dying(): return }

Slide 56

Slide 56 text

notify := make(chan bool) ... // Blocking write into a channel notify <- true

Slide 57

Slide 57 text

notify := make(chan struct{}) ... // Blocking write into a channel notify <- struct{}{}

Slide 58

Slide 58 text

notify := make(chan struct{}, 1) ... // Non-blocking write? notify <- struct{}{}

Slide 59

Slide 59 text

notify := make(chan struct{}) ... // Non-blocking write select { case notify <- struct{}{}: default: }

Slide 60

Slide 60 text

var observers []chan struct{} ... // Non-blocking write select { case notify <- struct{}{}: default: }

Slide 61

Slide 61 text

var observers []chan struct{} ... // Non-blocking writes for o := range observers { select { case o <- struct{}{}: default: } }

Slide 62

Slide 62 text

var observers []chan struct{} ... // Notify all observers, if possible for o := range observers { select { case o <- struct{}{}: default: } }

Slide 63

Slide 63 text

var traces chan []byte func init() { // Use a buffered channel traces = make(chan []byte, 200) // Fire off a background worker defaultClient = NewClient(traces) go defaultClient.Publisher() } // Send, drops trace if the backend is at capacity func Send(trace []byte) { select { case traces <- trace: // Success default: // Default case fired if channel is full // Ensures this is non blocking } }

Slide 64

Slide 64 text

var traces chan []byte func init() { // Use a buffered channel traces = make(chan []byte, 200) // Fire off a background worker defaultClient = NewClient(traces) go defaultClient.Publisher() } // Send, drops trace if the backend is at capacity func Send(trace []byte) { select { case traces <- trace: // Success default: // Default case fired if channel is full // Ensures this is non blocking } }

Slide 65

Slide 65 text

var traces chan []byte func init() { // Use a buffered channel traces = make(chan []byte, 200) // Fire off a background worker defaultClient = NewClient(traces) go defaultClient.Publisher() } // Send, drops trace if the backend is at capacity func Send(trace []byte) { select { case traces <- trace: // Success default: // Default case fired if channel is full // Ensures this is non blocking } }

Slide 66

Slide 66 text

deliveries, err := rabbit.Connect() for d := range deliveries { go handleDelivery(d) } func handleDelivery(d *rabbit.Delivery) { // execute request, send response }

Slide 67

Slide 67 text

func handleDelivery(d *rabbit.Delivery) { // what if we’re heavily loaded? }

Slide 68

Slide 68 text

var tokens chan struct{} tokens = make(chan struct{}, 1000) // push 1000 structs into channel func handleDelivery(d *rabbit.Delivery) { // what if we’re heavily loaded? }

Slide 69

Slide 69 text

var tokens chan struct{} tokens = make(chan struct{}, 1000) // push 1000 structs into channel func handleDelivery(d *rabbit.Delivery) { select { case <-tokens: // block executeRequest(d) tokens <- struct{}{} // replace } }

Slide 70

Slide 70 text

var tokens chan struct{} tokens = make(chan struct{}, 1000) // push 1000 structs into channel func handleDelivery(d *rabbit.Delivery) { select { case <-tokens: executeRequest(d) tokens <- struct{}{} // replace default: return ErrBackendAtCapacity }

Slide 71

Slide 71 text

var tokens chan struct{} tokens = make(chan struct{}, 1000) // push 1000 structs into channel func handleDelivery(d *rabbit.Delivery) { select { case <-tokens: executeRequest(d) tokens <- struct{}{} // replace case <-time.After(100 * time.Millisecond) return ErrBackendAtCapacity }

Slide 72

Slide 72 text

func handleDelivery(d *rabbit.Delivery) { // server is still receiving requests }

Slide 73

Slide 73 text

No content

Slide 74

Slide 74 text

Failures Timeouts Error Rates

Slide 75

Slide 75 text

client := circuit.NewHTTPClient( time.Second * 5, // timeout 10, // threshold nil // http.Client ) resp, err := client.Get("http://beesbeesbees.com/")

Slide 76

Slide 76 text

No content