Distributed Tracing: From Chaos to Clarity

Distributed Tracing: From Chaos to Clarity

Have you ever lost a request? You see it going into your system but can't see where it's gone or what happened to it? In this talk I am going to show you how to fix the problem of tangled systems that are hard to see through using distributed tracing, and also give you an insight into the wider context and future of the tooling. Some go knowledge assumed.

About Billie (https://twitter.com/purplebooth):
Billie Thompson (she/her) is a cloud-native consultant working at Armakuni. I help organisations move towards, fast-moving, sustainable, modern development practices. Every team is different, but usually, I'm helping teach Agile, DevOps and DDD practices at all skill levels.

33a8eda64dec30551fd0b474443e4b35?s=128

Armakuni

July 31, 2019
Tweet

Transcript

  1. 2.

    @PurpleBooth About me ▪ Quite tall ▪ Help people use

    best practices ▪ Capybaras are my favourite animal
  2. 35.

    @PurpleBooth • Slightly more common • Rapid creating new functionality

    • Publications talk about how to build it yourself
  3. 39.

    @PurpleBooth • It’s unusual if you don’t do it •

    You know it, and you know how to use it • Publications talk about how to use it
  4. 42.
  5. 95.

    # traefik.toml [tracing] backend = "jaeger" [tracing.jaeger] samplingServerURL = "http:/

    /jaeger-agent:5778/sampling" localAgentHostPort = "jaeger-agent:6831" @PurpleBooth
  6. 96.

    helm install stable/traefik --name my-release \ --set tracing.enabled=true \ --set

    tracing.backend=jaeger \ --set tracing.jaeger.localAgentHostPort=jaeger-agent:6831 \ --set tracing.jaeger.samplingServerURL=http:/ /jaeger-agent:5778/sampling \ --namespace kube-system @PurpleBooth
  7. 102.

    func getTracer() (opentracing.Tracer, io.Closer, error) { jaegerHostPort := ":6831" cfg

    := config.Configuration{ ServiceName: "account-service", Sampler: &config.SamplerConfig{ Type: "const", Param: 1, }, Reporter: &config.ReporterConfig{ LogSpans: false, BufferFlushInterval: 1 * time.Second, LocalAgentHostPort: jaegerHostPort, }, } return cfg.NewTracer() } @PurpleBooth
  8. 103.

    func getTracer() (opentracing.Tracer, io.Closer, error) { jaegerHostPort := ":6831" cfg

    := config.Configuration{ // Snipped } return cfg.NewTracer() } @PurpleBooth
  9. 106.

    @PurpleBooth tracer, _, err := getTracer() opentracing.InitGlobalTracer(tracer) if err !=

    nil { log.Fatal("cannot initialize Tracer", err) } _ = r.Walk(func(route *mux.Route, router *mux.Router, ancestors []*mux.Route) error { route.Handler( gorilla.Middleware(tracer, route.GetHandler())) return nil })
  10. 108.

    @PurpleBooth func HomeHandler(writer http.ResponseWriter, request *http.Request) { client := redis.NewClient(&redis.Options{

    Addr: "localhost:6379", Password: "", DB: 0, }) ctx := request.Context() // Pass me to other opentracing libraries client = otredis.WrapRedisClient(ctx, client) // Do other stuff
  11. 113.
  12. 117.

    @PurpleBooth type RabbitBroker interface { Produce(*Event, map[string]string) error } type

    TracedRabbitBroker struct { RabbitBroker RabbitBroker context context.Context hostname string exchange string } func (t TracedRabbitBroker) Produce(event *Event, headers map[string]string) error { span, _ := opentracing.StartSpanFromContext(t.context, "emit") defer span.Finish() ext.MessageBusDestination.Set(span, t.exchange) ext.PeerHostname.Set(span, t.hostname) ext.SpanKindConsumer.Set(span) tracer := opentracing.GlobalTracer() if err := tracer.Inject(span.Context(), opentracing.TextMap, opentracing.TextMapCarrier(headers)); err != nil { ext.Error.Set(span, true) return err } err := t.RabbitBroker.Produce(event, headers) ext.Error.Set(span, err != nil) return err }
  13. 118.

    @PurpleBooth type RabbitBroker interface { Produce(*Event, map[string]string) error } func

    (t TracedRabbitBroker) Produce(event *Event, headers map[string]string) error { }
  14. 120.

    @PurpleBooth tracer := opentracing.GlobalTracer() if err := tracer.Inject(span.Context(), opentracing.TextMap, opentracing.TextMapCarrier(headers));

    err != nil { ext.Error.Set(span, true) return err } err := t.RabbitBroker.Produce(event, headers)
  15. 122.

    @PurpleBooth func TraceConsume(hostname string, exchange string, callback func(msg string, headers

    map[string]string) error) func(msg string, headers map[string]string) error { return func(msg string, headers map[string]string) error { tracer := opentracing.GlobalTracer() spanCtx, err := tracer.Extract(opentracing.TextMap, opentracing.TextMapCarrier(headers)) if err == nil { return err } span := opentracing.StartSpan("consume", opentracing.ChildOf(spanCtx)) defer span.Finish() ext.MessageBusDestination.Set(span, exchange) ext.PeerHostname.Set(span, hostname) ext.SpanKindConsumer.Set(span) return callback(msg, headers) } }