Upgrade to Pro — share decks privately, control downloads, hide ads and more …

Channeling Failure

mattheath
February 03, 2015

Channeling Failure

Go’s concurrency primitives make it easy to build highly concurrent systems, however any distributed system operating at scale will experience failures. This is especially important in microservice architectures where requests may traverse many systems initiating remote calls, and failures of a single component may cascade through several more.

In this talk we look at a number of common patterns from simple usage of channels to control and throttle concurrency, to more complex patterns such as the CircuitBreaker which can be used to prevent cascading failures; increasing the reliability of our systems.

Images:
ATM - Thomas Hawk - https://www.flickr.com/photos/thomashawk/5650906605/
Network - Norlando Pobre - https://www.flickr.com/photos/npobre/8437956869/

mattheath

February 03, 2015
Tweet

More Decks by mattheath

Other Decks in Programming

Transcript

  1. url := "http://beesbeesbees.com" resp, err := http.Get(url) if err !=

    nil { return fmt.Errorf("failed to load url %s: %s", url, err) }
  2. var ErrTimeout = errors.New("timeout") var ErrNotFound = errors.New("not found") var

    ErrTooManyCooks = errors.New("spoil the broth") return errors.New("timeout")
  3. var ErrTimeout = errors.New("timeout") var ErrNotFound = errors.New("not found") var

    ErrTooManyCooks = errors.New("spoil the broth") return ErrTimeout
  4. if err := loadBees(); err != nil { switch err

    { case ErrTimeout: // retry? case ErrNotFound: // abort! } }
  5. if err := loadBees(); err != nil { switch err

    { case ErrTimeout: // retry? case ErrNotFound: return err } }
  6. if err := loadBees(); err != nil { switch err

    { case ErrTimeout: // retry? case ErrNotFound: return err default: return err } }
  7. if err := loadBees(); err != nil { switch err

    { case ErrTimeout: // retry? default: return err } }
  8. for { if err := loadBees(); err != nil {

    switch err { case ErrTimeout: continue // retry...? default: return err } } return nil }
  9. for { if err := loadBees(); err != nil {

    switch err { case ErrTimeout: continue // retry...? default: return err } } return nil }
  10. for { if err := loadBees(); err != nil {

    switch err { case ErrTimeout: // retry...? // backoff // increasing timeouts // limits on this default: return err
  11. package httpclient import ( "net" "net/http" "time" ) func TimeoutDialer(cTimeout

    time.Duration, rwTimeout time.Duration) func(netw, addr string)
 (c net.Conn, err error) { return func(netw, addr string) (net.Conn, error) { conn, err := net.DialTimeout(netw, addr, cTimeout) if err != nil { return nil, err } conn.SetDeadline(time.Now().Add(rwTimeout)) return conn, nil } } func NewTimeoutClient(connectTimeout time.Duration, readWriteTimeout time.Duration) *http.Client { return &http.Client{ Transport: &http.Transport{ Dial: TimeoutDialer(connectTimeout, readWriteTimeout), }, } }
  12. package httpclient import ( "net" "net/http" "time" ) func TimeoutDialer(cTimeout

    time.Duration, rwTimeout time.Duration) func(netw, addr string)
 (c net.Conn, err error) { return func(netw, addr string) (net.Conn, error) { conn, err := net.DialTimeout(netw, addr, cTimeout) if err != nil { return nil, err } conn.SetDeadline(time.Now().Add(rwTimeout)) return conn, nil } } func NewTimeoutClient(connectTimeout time.Duration, readWriteTimeout time.Duration) *http.Client { return &http.Client{ Transport: &http.Transport{ Dial: TimeoutDialer(connectTimeout, readWriteTimeout), }, } }
  13. package httpclient import ( "net" "net/http" "time" ) func TimeoutDialer(cTimeout

    time.Duration, rwTimeout time.Duration) func(netw, addr string)
 (c net.Conn, err error) { return func(netw, addr string) (net.Conn, error) { conn, err := net.DialTimeout(netw, addr, cTimeout) if err != nil { return nil, err } conn.SetDeadline(time.Now().Add(rwTimeout)) return conn, nil } } func NewTimeoutClient(connectTimeout time.Duration, readWriteTimeout time.Duration) *http.Client { return &http.Client{ Transport: &http.Transport{ Dial: TimeoutDialer(connectTimeout, readWriteTimeout), }, } }
  14. package httpclient import ( "net" "net/http" "time" ) func TimeoutDialer(cTimeout

    time.Duration, rwTimeout time.Duration) func(netw, addr string)
 (c net.Conn, err error) { return func(netw, addr string) (net.Conn, error) { conn, err := net.DialTimeout(netw, addr, cTimeout) if err != nil { return nil, err } conn.SetDeadline(time.Now().Add(rwTimeout)) return conn, nil } } func NewTimeoutClient(connectTimeout time.Duration, readWriteTimeout time.Duration) *http.Client { return &http.Client{ Transport: &http.Transport{ Dial: TimeoutDialer(connectTimeout, readWriteTimeout), }, } }
  15. package httpclient import ( "net" "net/http" "time" ) func TimeoutDialer(cTimeout

    time.Duration, rwTimeout time.Duration) func(netw, addr string)
 (c net.Conn, err error) { return func(netw, addr string) (net.Conn, error) { conn, err := net.DialTimeout(netw, addr, cTimeout) if err != nil { return nil, err } conn.SetDeadline(time.Now().Add(rwTimeout)) return conn, nil } } func NewTimeoutClient(connectTimeout time.Duration, readWriteTimeout time.Duration) *http.Client { return &http.Client{ Transport: &http.Transport{ Dial: TimeoutDialer(connectTimeout, readWriteTimeout), }, } }
  16. func loadBees(chan []bee) {} beeChan := make(chan []bee) go loadBees(beeChan)

    // Blocks until timeout select { case bees := <-beeChan: case <-time.After(1 * time.Second): }
  17. func loadBees(chan []bee) {} beeChan := make(chan []bee) go loadBees(beeChan)

    // Blocks until timeout select { case bees := <-beeChan: case <-time.After(1 * time.Second): return ErrTimeout }
  18. func loadBees(chan []bee) {} beeChan := make(chan []bee) go loadBees(beeChan)

    // Blocks until timeout select { case bees := <-beeChan: case <-time.After(1 * time.Second): return ErrTimeout }
  19. func loadBees(chan []bee) {} beeChan := make(chan []bee) go loadBees(beeChan)

    // Blocks until timeout select { case bees := <-beeChan: // errors? case <-time.After(1 * time.Second): return ErrTimeout }
  20. func loadBees(chan []bee) {} beeChan := make(chan []bee) go loadBees(beeChan)

    // On timeout, writer is blocked select { case bees := <-beeChan: case <-time.After(1 * time.Second): return ErrTimeout }
  21. func loadBees(chan []bee) {} beeChan := make(chan []bee, 1) go

    loadBees(beeChan) // On timeout, writer not blocked select { case bees := <-beeChan: case <-time.After(1 * time.Second): return ErrTimeout }
  22. for { if err := connect(); err != nil {

    // retry, with backoff } }
  23. for { if err := connect(); err != nil {

    // retry, with backoff } err := <-Conn.NotifyClose() }
  24. for { if err := connect(); err != nil {

    // retry, with backoff } select { case err := <-Conn.NotifyClose(): } }
  25. for { if err := connect(); err != nil {

    // retry, with backoff } select { case err := <-Conn.NotifyClose(): case <-configChange: // disconnect } }
  26. for { if err := connect(); err != nil {

    // retry, with backoff } select { case err := <-Conn.NotifyClose(): case <-quit: return } }
  27. quit := make(chan struct{}) close(quit) select { case err :=

    <-Conn.NotifyClose(): case <-quit: return // channel closed }
  28. var observers []chan struct{} ... // Non-blocking writes for o

    := range observers { select { case o <- struct{}{}: default: } }
  29. var observers []chan struct{} ... // Notify all observers, if

    possible for o := range observers { select { case o <- struct{}{}: default: } }
  30. var traces chan []byte func init() { // Use a

    buffered channel traces = make(chan []byte, 200) // Fire off a background worker defaultClient = NewClient(traces) go defaultClient.Publisher() } // Send, drops trace if the backend is at capacity func Send(trace []byte) { select { case traces <- trace: // Success default: // Default case fired if channel is full // Ensures this is non blocking } }
  31. var traces chan []byte func init() { // Use a

    buffered channel traces = make(chan []byte, 200) // Fire off a background worker defaultClient = NewClient(traces) go defaultClient.Publisher() } // Send, drops trace if the backend is at capacity func Send(trace []byte) { select { case traces <- trace: // Success default: // Default case fired if channel is full // Ensures this is non blocking } }
  32. var traces chan []byte func init() { // Use a

    buffered channel traces = make(chan []byte, 200) // Fire off a background worker defaultClient = NewClient(traces) go defaultClient.Publisher() } // Send, drops trace if the backend is at capacity func Send(trace []byte) { select { case traces <- trace: // Success default: // Default case fired if channel is full // Ensures this is non blocking } }
  33. deliveries, err := rabbit.Connect() for d := range deliveries {

    go handleDelivery(d) } func handleDelivery(d *rabbit.Delivery) { // execute request, send response }
  34. var tokens chan struct{} tokens = make(chan struct{}, 1000) //

    push 1000 structs into channel func handleDelivery(d *rabbit.Delivery) { // what if we’re heavily loaded? }
  35. var tokens chan struct{} tokens = make(chan struct{}, 1000) //

    push 1000 structs into channel func handleDelivery(d *rabbit.Delivery) { select { case <-tokens: // block executeRequest(d) tokens <- struct{}{} // replace } }
  36. var tokens chan struct{} tokens = make(chan struct{}, 1000) //

    push 1000 structs into channel func handleDelivery(d *rabbit.Delivery) { select { case <-tokens: executeRequest(d) tokens <- struct{}{} // replace default: return ErrBackendAtCapacity }
  37. var tokens chan struct{} tokens = make(chan struct{}, 1000) //

    push 1000 structs into channel func handleDelivery(d *rabbit.Delivery) { select { case <-tokens: executeRequest(d) tokens <- struct{}{} // replace case <-time.After(100 * time.Millisecond) return ErrBackendAtCapacity }
  38. client := circuit.NewHTTPClient( time.Second * 5, // timeout 10, //

    threshold nil // http.Client ) resp, err := client.Get("http://beesbeesbees.com/")