Slide 1
Slide 1 text
OpenCensus/OpenTelemetry meetup vol.2
@takashabe
OpenCensusͰcustom context
propagationͱexporterΛॻ͍
ͨ
Slide 2
Slide 2 text
whoami
• Takashi Abe (@takashabe)
• גࣜձࣾαΠόʔΤʔδΣϯτ ΞυςΫελδΦ
• GoͰࠂ৴ϓϩμΫτΛ࡞͍ͬͯ·͢
• KubernetesͰMicroservicesతͳͷ
• ਪ͠ΩʔϘʔυirisͰ͢
• Corne Cherry࡞த…
Slide 3
Slide 3 text
͡Ίʹ
• Go + OpenCensus + Stackdriver Λલఏͱ͍ͯ͠·͢
Slide 4
Slide 4 text
࣍
• ࢄτϨʔγϯάͷ֓ཁ
• context propagation
• exporter
• ·ͱΊ
Slide 5
Slide 5 text
ࢄτϨʔγϯά
Slide 6
Slide 6 text
ࢄτϨʔγϯά
https://github.com/census-instrumentation/opencensus-go
• 1 traceʹରͯ͠ɺෳͷspan͕ඥͮ͘
• spanಉҰΞϓϦέʔγϣϯͰෳൃߦͯ͠ྑ͍͠ɺҟͳΔΞϓϦέʔγϣϯͰ
τϨʔεID͕ಉҰͰ͋Εඥͮ͘
Slide 7
Slide 7 text
context propagation
Slide 8
Slide 8 text
context propagation is …
Slide 9
Slide 9 text
context propagation is …
• τϨʔεIDΛൖ͢ΔͨΊͷϓϩτίϧతͳͷ
• opencensus-goͰocgrpcochttpͰ࣮͞Ε͍ͯΔ
• SpanContextΛհͯ͠σʔλΛൖͤ͞Δ
• TraceID, SpanID, TraceOptions(αϯϓϦϯάର͔Ͳ
͏͔)
type SpanContext struct {
TraceID TraceID // [16]byte
SpanID SpanID // [8]byte
TraceOptions TraceOptions // uint32
Tracestate *tracestate.Tracestate // Ұ෦ͷpropagation༻
}
Slide 10
Slide 10 text
ocgrpc, ochttp
ocgrpc
ochttp
Slide 11
Slide 11 text
HTTPFormatͷ࣮
• ocgrpc࣮͕1ͭ
• ochttpͰHTTPFormatΛ࣮ͨ͠b3ͱtracecontextͱ͍͏2
ͭͷ࣮͕ଘࡏ͢Δ
• b3: ZipkinͰఆٛ͞Εͨpropagation(σϑΥϧτ)
• https://github.com/openzipkin/b3-propagation
• tracecontext: w3cͰఆٛ͞Εͨpropagation
• https://github.com/w3c/trace-context
type HTTPFormat interface {
SpanContextFromRequest(req *http.Request) (sc trace.SpanContext, ok bool)
SpanContextToRequest(sc trace.SpanContext, req *http.Request)
}
Slide 12
Slide 12 text
B3 Propagation
https://github.com/openzipkin/b3-propagation
Slide 13
Slide 13 text
B3 Propagation
https://github.com/openzipkin/b3-propagation
4QBO$POUFYU'SPN3FRVFTU
4QBO$POUFYU5P3FRVFTU
Slide 14
Slide 14 text
OpenCensusͰͷ࣮(ToRequest)
const (
TraceIDHeader = "X-B3-TraceId"
SpanIDHeader = "X-B3-SpanId"
SampledHeader = "X-B3-Sampled"
)
func (f *HTTPFormat) SpanContextToRequest(sc trace.SpanContext, req
*http.Request) {
req.Header.Set(TraceIDHeader, hex.EncodeToString(sc.TraceID[:]))
req.Header.Set(SpanIDHeader, hex.EncodeToString(sc.SpanID[:]))
var sampled string
if sc.IsSampled() {
sampled = "1"
} else {
sampled = "0"
}
req.Header.Set(SampledHeader, sampled)
}
https://github.com/census-instrumentation/opencensus-go/blob/master/plugin/ochttp/propagation/b3/b3.go
Slide 15
Slide 15 text
OpenCensusͰͷ࣮(FromRequest)
const (
TraceIDHeader = "X-B3-TraceId"
SpanIDHeader = "X-B3-SpanId"
SampledHeader = "X-B3-Sampled"
)
func (f *HTTPFormat) SpanContextFromRequest(req *http.Request) (sc
trace.SpanContext, ok bool) {
tid, ok := ParseTraceID(req.Header.Get(TraceIDHeader))
if !ok {
return trace.SpanContext{}, false
}
sid, ok := ParseSpanID(req.Header.Get(SpanIDHeader))
if !ok {
return trace.SpanContext{}, false
}
sampled, _ := ParseSampled(req.Header.Get(SampledHeader))
return trace.SpanContext{
TraceID: tid,
SpanID: sid,
TraceOptions: sampled,
}, true
}
https://github.com/census-instrumentation/opencensus-go/blob/master/plugin/ochttp/propagation/b3/b3.go
Slide 16
Slide 16 text
ochttpΛར༻͢Δ
// import "go.opencensus.io/plugin/ochttp"
mux := http.NewServeMux()
mux.Handle("/users", ochttp.WithRouteTag(usersHandler, "/users"))
log.Fatal(http.ListenAndServe("localhost:8080", &ochttp.Handler{
Handler: mux,
Propagation: &b3.HTTPFormat{},
}))
https://github.com/census-instrumentation/opencensus-go/blob/master/plugin/ochttp/example_test.go
• ochttp.Handler͕ϛυϧΣΞͱͯ͠τϨʔγϯάपΓ
Λ໘ݟͯ͘ΕΔ
• context propagationҙࣝ͠ͳͯ͘ྑ͍
Slide 17
Slide 17 text
custom context propagation
Slide 18
Slide 18 text
Cloud Pub/Sub ʹద༻ͨ͠
• GCPͷϚωʔδυPub/Sub
• B3 PropagationΛϕʔεʹͯ͠ɺϝοηʔδͷ
AttributesʹσʔλΛ٧Ίͯൖͤ͞Δ
• https://github.com/takashabe/oc-propagation-demo
• ΄΅ಉ͡ͷΛϓϩμΫγϣϯʹಋೖࡁΈ͕ͩɺڥ
ґଘ͕͋ΔͷͰαϯϓϧ࣮Λ༻ҙ
• αʔό༻ҙ͢Δͷ໘ͰCloud Run༻ʹHTTPαʔό
ͱ࣮͍ͯͯ͠͠ΔͷͰएׯݟͮΒ͍
Slide 19
Slide 19 text
Pub/Sub൛ͷ࣮(ToRequest)
const (
TraceIDField = "X-Pubsub-TraceId"
SpanIDField = "X-Pubsub-SpanId"
SampledField = "X-Pubsub-Sampled"
)
func WrapMessage(ctx context.Context, m *pubsub.Message) *pubsub.Message {
if m.Attributes != nil {
if m.Attributes[TraceIDField] != "" || m.Attributes[SpanIDField] != "" {
return m
}
} else {
m.Attributes = make(map[string]string, 3)
}
sc := trace.FromContext(ctx).SpanContext()
m.Attributes[TraceIDField] = sc.TraceID.String()
m.Attributes[SpanIDField] = sc.SpanID.String()
m.Attributes[SampledField] = fmt.Sprintf("%t", sc.IsSampled())
return m
}
Slide 20
Slide 20 text
Pub/Sub൛ͷ࣮(FromRequest)
const (
TraceIDField = "X-Pubsub-TraceId"
SpanIDField = "X-Pubsub-SpanId"
SampledField = "X-Pubsub-Sampled"
)
func SpanContextFromMessage(m *pubsub.Message) trace.SpanContext {
if m.Attributes == nil {
return trace.SpanContext{}
}
tid, ok := parseTraceID(m.Attributes[TraceIDField])
if !ok {
return trace.SpanContext{}
}
sid, ok := parseSpanID(m.Attributes[SpanIDField])
if !ok {
return trace.SpanContext{}
}
sampled := parseSampled(m.Attributes[SampledField])
return trace.SpanContext{
TraceID: tid,
SpanID: sid,
TraceOptions: sampled,
}
}
Slide 21
Slide 21 text
ར༻ଆ
// import “github.com/takashabe/oc-propagation-demo/internal/propagation"
...
ctx, span := trace.StartSpan(context.Background(), "publish")
defer span.End()
msg := &pubsub.Message{
Data: []byte("foo"),
}
topic.Publish(ctx, propagation.WrapMessage(ctx, msg)).Get(ctx)
// import “github.com/takashabe/oc-propagation-demo/internal/propagation"
...
subscription.Receive(ctx, func(ctx context.Context, msg *pubsub.Message) {
sc := propagation.SpanContextFromMessage(msg)
_, span := trace.StartSpanWithRemoteParent(ctx, "receive", sc)
defer span.End()
publisher(client)
subscriber(server)
Slide 22
Slide 22 text
Stackdriver Trace্ͷදࣔ
publisher subscriber
Pub/Sub
Slide 23
Slide 23 text
exporter
Slide 24
Slide 24 text
exporter is …
Slide 25
Slide 25 text
exporter is …
• StackdriverDatadogͳͲͷόοΫΤϯυαʔϏεʹ
τϨʔεɺ͋Δ͍ϝτϦΫεΛૹΔͨΊͷΈ
• σʔλऩू෦ͱexporterʹΑΔόοΫΤϯυαʔϏε
͕͞Ε͍ͯΔͷ͕OpenCensusͷ1ͭͷಛ
Slide 26
Slide 26 text
exporterͷ࣮(Go)
type Exporter interface {
ExportSpan(s *SpanData)
}
• ExporterΠϯλϑΣʔεΛ࣮͢Δ͚ͩͰྑ͍
• ҙͷόοΫΤϯυαʔϏεAPIͳͲΛୟ͍ͯσʔλΛ
ૹ͍ͬͯΔ
• ࣮ࡍʹύϑΥʔϚϯεͰ͔ͳΓؾΛ͍ͬͯΔ(ͣ)
Slide 27
Slide 27 text
custom exporter
Slide 28
Slide 28 text
Tail LatencyΛิ͢ΔͨΊͷexporter
• Tail Latency
• 99ύʔηϯλΠϧͷΑ͏ͳͷ
• ௨ৗɺτϥϑΟοΫͷଟ͍αʔϏεͩͱαϯϓϦϯά
ͷඞཁ͕͋Δ͕ɺͦ͏͢ΔͱTail Latency͕શ͘ه͞
Εͳ͍
• Tail LatencyͰΜͰ͍Δͱ͖मਖ਼ɺܭଌͷαΠΫ
ϧΛճ͍͕ͨ͠ɺ؍ଌ͢ΔͨΊʹ࣌ؒҎ্͔͔Δ
͜ͱ͋Δ…
Slide 29
Slide 29 text
edge exporter
• https://github.com/takashabe/edge-exporter
• ϓϩμΫγϣϯೖग़དྷͯͳ͍ͷͰࢀߟ࣮ͱͯ͠…
• ύϑΥʔϚϯεվળͷ༨͕େ͍ʹͬͯΔ
• ߏΛ࿅͍ͬͯΔ࣌ʹϓϩδΣΫτҠಈʹͳͬͨ
• ଞexporterͷલஈͰproxyͱͯ͠ಈ࡞ͤ͞Δ
• શ݅αϯϓϦϯάͭͭ͠ɺҰఆ࣌ؒ͝ͱʹ࠷ϨΠςϯ
γͷߴ͍τϨʔε͚ͩΛ࠾༻͢Δ
• Φʔόʔϔου͋Δ
Slide 30
Slide 30 text
spanϥΠϑαΠΫϧ
• ࠓճͷΑ͏ʹTail LatencyΛั·͑ΔͨΊʹશ݅αϯ
ϓϦϯά͢Δඞཁ͕ग़ͯ͘Δ
• αϯϓϦϯάରͰͳ͍spanʹର͢Δૢ࡞ૣظϦ
λʔϯ͢ΔΑ͏ʹͳ͍ͬͯΔ
• αϯϓϦϯάରʹͳ͍ͬͯΔ͚ͩͰएׯͷΦʔόʔ
ϔου͕͋Δ
_, span := trace.StartSpan(ctx, name) // αϯϓϦϯάܾఆ. SpanDatasੜ
span.AddAttributes(...) // αϯϓϦϯάର֎ͳΒεΩοϓ
...
span.End() // ExportSpan͕ݺΕΔ
Slide 31
Slide 31 text
edge exporter࣮
type EdgeExporter struct {
// Stackdriver exporterͳͲͷଞexporterΛอ࣋͢Δ
exporters exportersList
// Tail LatencyͷܭଌִؒͱRateLimit
interval time.Duration
limiter *rate.Limiter
// interval͝ͱͷTail LatencyΛอ࣋͢Δ
tail *trace.SpanData
tailLatency time.Duration
tailMu sync.Mutex
}
• EdgeExporterߏମ
Slide 32
Slide 32 text
edge exporter࣮
• ExportSpan()
func (e *EdgeExporter) ExportSpan(sd *trace.SpanData) {
e.storeTailLatencySpan(sd)
if !e.limiter.Allow() {
return
}
e.tailMu.Lock()
defer e.tailMu.Unlock()
if e.tail == nil {
return
}
for _, exp := range e.exporters.Load() {
exp.ExportSpan(e.tail)
}
e.tail = nil
e.tailLatency = 0
}
Slide 33
Slide 33 text
example
func main() {
sd, _ := stackdriver.NewExporter(stackdriver.Options{
ProjectID: os.Getenv("PROJECT_ID"),
})
edge := edgeexporter.New(edgeexporter.WithExportInterval(10*time.Second))
edge.RegisterExporter(sd)
trace.RegisterExporter(edge)
trace.ApplyConfig(trace.Config{
DefaultSampler: trace.AlwaysSample(),
})
http.HandleFunc("/", handler)
http.ListenAndServe(":8080", nil)
}
var cnt int64
func handler(w http.ResponseWriter, req *http.Request) {
_, span := trace.StartSpan(context.Background(), "handler")
c := atomic.LoadInt64(&cnt)
if c%5 == 0 {
time.Sleep(50 * time.Millisecond)
}
span.End()
atomic.AddInt64(&cnt, 1)
}
https://github.com/takashabe/edge-exporter/blob/master/example/main.go
Slide 34
Slide 34 text
example
func main() {
sd, _ := stackdriver.NewExporter(stackdriver.Options{
ProjectID: os.Getenv("PROJECT_ID"),
})
edge := edgeexporter.New(edgeexporter.WithExportInterval(10*time.Second))
edge.RegisterExporter(sd)
trace.RegisterExporter(edge)
trace.ApplyConfig(trace.Config{
DefaultSampler: trace.AlwaysSample(),
})
http.HandleFunc("/", handler)
http.ListenAndServe(":8080", nil)
}
var cnt int64
func handler(w http.ResponseWriter, req *http.Request) {
_, span := trace.StartSpan(context.Background(), "handler")
c := atomic.LoadInt64(&cnt)
if c%5 == 0 {
time.Sleep(50 * time.Millisecond)
}
span.End()
atomic.AddInt64(&cnt, 1)
}
https://github.com/takashabe/edge-exporter/blob/master/example/main.go
10ඵ͝ͱʹTail Latency SpanΛ
Stackdriver exporterʹྲྀ͢
Slide 35
Slide 35 text
Stackdriver Trace্ͷදࣔ
$ echo "GET https://…” | vegeta attack -rate=50 -duration=1m
Slide 36
Slide 36 text
custom exporterͷεεϝ
• ࠓճTail LatencyΛରͱ͕ͨ͠ɺྫ͑Τϥʔ͕ൃ
ੜͨ͠ͱ͖͚ͩɺಛఆͷϦΫΤετ͚ͩͳͲԠ༻ޮ
͖ͦ͏
• ܭଌ։࢝࣌(StartSpan)Ͱஅग़དྷͳ͍ͷΛܭଌ͢
ΔͨΊʹࠓճͷΑ͏ʹύϑΥʔϚϯεͱͷτϨʔυ
ΦϑʹͳΓͦ͏
Slide 37
Slide 37 text
Conclusion
Slide 38
Slide 38 text
Conclusion
• Cloud Pub/Sub༻ͷcontext propagationͱɺTail
Latency༻ͷexporterΛ࡞ͬͯΈͨ
• ಠ࣮ࣗͷϋʔυϧͦΕ΄Ͳߴ͘ͳ͍
• ؍ଌ͍ͨ͠ͷΛ؍ଌग़དྷΔΑ͏ʹ͠Α͏