Go Conference'19 Summer in Fukuokaでの登壇資料です https://fukuoka.gocon.jp/ja/
ϚΠΫϩαʔϏε࣌ͷHTTPΫϥΠΞϯτͷ࡞Γํ@dxhuy Observability Team / LINE Corp
View Slide
@dxhuyhttps://github.com/huydx
Developer
• LINEͷObservabilityνʔϜʹ͍ͭͯ• ϚΠΫϩαʔϏεͷམͱ݀͠• ෦ͰΧελϜͨ͠HTTPΫϥΠΞϯτͷઃܭͱհࠓͷ
EngineeringLINE’S OBSERVABILITYTEAM
EngineeringOBSERVABILITYIS NEW WAVE
ΠϝʔδతʹࣾDATADOGΛ࡞͍ͬͯΔ
EngineeringLogج൫Metricج൫ࢄτϨʔγϯάج൫ZipkinͷOSSίϯτϦϏϡʔτ
EngineeringGO͍ͬͯΔͷʁMetric Agent Time Series StorageMetric API Server͜Ε͔Β࡞ΔͷͲΜͲΜGO࠾༻ʂCLI toolMetric Alert Engine
> 1.000.000metrics per secεέʔϧνϟϨϯδ
> 2.000read per secεέʔϧνϟϨϯδ
EngineeringMetrics૿ՃϏδωεεέʔϧ
EngineeringGOPHERੵۃ࠾༻தʂιϑτΣΞΤϯδχΞ (Software EngineerɺEngineering Efficiency)ʲLINEϓϥοτϑΥʔϜʳ
Engineeringࠓͷຊ
EngineeringϚΠΫϩαʔϏε͍ͬͯ·͔͢
Engineeringίϛϡχέʔγϣϯ(RPC)͕େࣄ
Engineeringtwitch/TwirpGRPC ThriftRESTHttpRPCख๏͕༷ʑࠓޙ૿͑ͦ͏
EngineeringGRPC, THRIFTͳͲશͯͷγεςϜΛม͑ͳ͍ͱߦ͚ͳ͍
Engineering͢Ͱʹଘࡏ͍ͯ͠ΔAPIαʔϏε͕ࠞࡏnet/http͕ఆ൪
EngineeringNET/HTTPͪΐͬͱΓͳ͍
EngineeringϚΠΫϩαʔϏε௨৴ͷΫϥΠΞϯτඞཁͳੑ࣭
EngineeringαʔϏεͷελοΫωοτϫʔΫԆϦΫΤετࣦഊαʔϏεಥવؾઈhttps://www.flickr.com/photos/chrish_99/8526019374/
Engineeringhttps://www.flickr.com/photos/chrish_99/8526019374/
EngineeringϦϞʔτίʔϧ͕ࣦഊͯ͠ճ෮Ͱ͖ΔΑ͏ʹͳΔ࣮ύλʔϯ- Retry- CircuitBreakerճ෮ੑίʔϧઌ͕ࢄͰ͖ΔΑ͏ʹͳΔ࣮ύλʔϯ- Load Balancing- Service DiscoveryߴՄ༻ੑࢲ͕ཉ͍͠ HTTPΫϥΠΞϯτ
Engineering● CircuitBreaker, RetryͳͲ͕ἧ͏ύοέʔδͳͲଘࡏ͢Δ͕● Ұͭͷػೳ͔͍࣋ͬͯ͠ͳ͍ͷ͕ଟ͍● ඪ४ͷnet/httpͷΠϯλʔϑΣʔεʹ૬ੑѱ͍ɺίʔυϕʔεͷมߋ͕ଟ͘ͳΔ● BoilerPlate͕ଟ͍● ύϑΥʔϚϯεྑ͘ͳ͍● ֎෦dependencies͕ଟ͍● ྫɿgojek/heimdall, go-kitܥͷϥΠϒϥΠͳͲͭ·ΓΧελϜNET/HTTPΫϥΠΞϯτཉ͍͠
EngineeringࣾͰΧελϜNET/HTTP࡞Γ·ͨ͠
EngineeringXHTTPύοέʔδઃܭͱ͍ํͷհ
Engineering● ඪ४ͷnet/httpͷΠϯλʔϑΣʔεͰ͖Δ͚ͩै͏● ͏ଆʢϢʔβʣͷมߋΛ࠷খݶʹ͢Δ● net/httpͷϥούϥΠϒϥϦͱͯ͠● ඞཁͳͷશ෦ἧ͏● ճ෮ੑɺߴՄ༻ੑͷػೳΛἧ͏● ϝτϦΫεɺTracingͷػೳΛἧ͏● ଥͳσϑΥϧτʢλΠϜΞτͳͲʣઃܭࢥߟ
Engineeringpackage xhttptype ClientConfig struct {…}func NewClient(config *ClientConfig) *Client {}func NewRequestX(method string, addr *AddressGroup, path string, bodyio.Reader) (*Request, error)func (c *Client) DoX(req *Request, option ...CallOption) (*Response, error)func (c *Client) PostX(addr *AddressGroup, path string, contentType string,body io.Reader, options ...CallOption) (resp *Response, err error) {func (c *Client) PostFormX(addr *AddressGroup, path string, data url.Values,options ...CallOption) (resp *Response, err error)func (c *Client) GetX(addr *AddressGroup, path string, options ...CallOption)(resp *Response, err error) {NET/HTTPͱಉ͡ΠϯλʔϑΣΠε
EngineeringaddrGroup := xhttp.MustNewAddressGroup( //AddressGroup͋ͱ΄Ͳհ͠·͢“dns: //txt:your-domain.com:8080”,xhttp.WithLoadBalancer(xhttp.RoundRobin),)reqP, _ := xhttp.NewRequestX("POST", addrGroup, "/", nil)res, err = client.DoX(reqG)res, err = client.PostFormX(addrGroup, "/", url.Values{})res, err = client.GetX(addrGroup, "/")͍ํͷΠϝʔδ
EngineeringߴՄ༻ੑͷ
Upstream 2 diexhttp clientUpstream 1αʔό͍ͭͰࢮ͵ՄೳੑUpstream 3
Engineering● ීஈ Reverse Proxy (VIP)ͳͲར༻͕ଟ͍● ৽͍͠αʔϏεՃΛ͢Δͨͼʹ৽͍͠ Reverse Proxyಋೖ͢Δͷ͕໘● Solution● Client Side Load Balancing● ΫϥΠΞϯτίʔϧઌͷαʔόάϧʔϓΛࣗͰίϯτϩʔϧ͢Δ͜ͱ● ΫϥΠΞϯτίʔϧઌͷαʔόάϧʔϓɺͲΕΛݺͼग़͢ͷ͔ͷϩʔυόϥϯγϯάϩδοΫΛ࣮͢Δ͜ͱαʔόάϧʔϓ͕ඞཁ
Engineeringtype Address struct {Port stringHost string}type AddressGroup struct {scheme Schemename stringresolver ResolverloadBalancer LoadBalancercacheAddress []AddresscloseCh chan interface{}}AddressGroup ʢαʔόάϧʔϓͷநԽʣ
Engineeringtype Resolver interface {Scheme() SchemeInit(address string) errorUpdateCh(context.Context) Close()}type LoadBalancer interface {Pick() (Address, error)HandleStateChange(list []Address) error}Resolver + LoadBalancer
Engineeringtype DnsResolver struct {domain stringupdateCh chan []AddressdoneCh chan interface{}}func (d *DnsResolver) Scheme() Scheme {return Scheme("dns")}func (d *DnsResolver) Init(name string) error {d.resolve()}func (d *DnsResolver) resolve() {go func() {// resolve d.domaind.updateCh time.Sleep(TTL)}}Example: DnsResolve
Engineeringtype roundRobinLB struct {list []Addressnext uint32mu sync.RWMutex}func (rr *roundRobinLB) Pick() (Address, error) {l := len(rr.list)if l == 0 {return Address{}, fmt.Errorf("empty list of address")}v := atomic.AddUint32(&rr.next, 1)return rr.list[int(v)%len(rr.list)], nil}func (rr *roundRobinLB) HandleStateChange(list []Address) error {rr.mu.Lock()defer rr.mu.Unlock()rr.list = listreturn nil}Example: Round Robin LB
Engineering͏ଆBOILER PLATE͕΄ͱΜͲͳ͍addrGroup := xhttp.MustNewAddressGroup( //AddressGroup͋ͱ΄Ͳհ͠·͢“dns: //txt:your-domain.com:8080”,xhttp.WithLoadBalancer(xhttp.RoundRobin),)reqP, _ := xhttp.NewRequestX("POST", addrGroup, "/", nil)ϥΠϒϥϦଆͰschemeύʔεͯ͠ResolverΛΞαΠϯ͢Δ
Engineering● Load Balancing is impossible● https://www.youtube.com/watch?v=kpvbOzHUakA● Round Robin Upstreamϗετͷঢ়ଶʢLatency, Φʔόϩʔυঢ়گʣͳͲΛҙࣝͰ͖ͣɺͣͬͱಉ͡ѱ͍ঢ়ଶͷϗετʹಉ͡ϦΫΤετΛ͛Δɹ● P2C ΞϧΰϦζϜ● Power of 2 Random Choice● Upstreamͷঢ়ଶΛୡඞཁ͋Δ● https://www.eecs.harvard.edu/~michaelm/postscripts/mythesis.pdf༨ஊϩʔυόϥγϯάΞϧΰϦζϜ͍͠
Engineeringճ෮ੑͷ
EngineeringUpstream 2 diexhttp clientUpstream 1αʔό͍ͭͰࢮ͵ՄೳੑUpstream 3
EngineeringUpstream 2 diexhttp clientUpstream 1 die(શʣαʔό͍ͭͰࢮ͵ՄೳੑUpstream 3 die
Engineering● શʹࢮ͵͡Όͳͯ͘ɺ(GCͳͲͷཧ༝ͰʣLatency͕Ͷ্͕Δ● దͳσϑΥϧτλΠϜΞτ͕ඞཁ● ଈ࣌తͳϦΫΤετࣦഊ (500ΤϥʔͳͲʣ● దͳϦτϥΠ● ܧଓతͳϦΫΤετࣦഊ (ωοτϫʔΫࣦഊͳͲʣ● Ͱ͖Δ͚ͩૣ͍ஈ֊ͰϦΫΤετΛυϩοϓ͢Δ● CircuitBreakerύʔλϯɿFail Fast Errorαʔό͕ࢮ͵ͱ͖Ͳ͏͠·͢ʁ
Engineering● net/httpσϑΥϧτλΠϜΞτ͋Γ·ͤΜʂ● ແݶʹϦΫΤετͭ● xhttpͰσϑΥϧτͰ30sઃఆՄೳɺ֎͔Β͢ͷՄೳλΠϜΞτʹ͍ͭͯfunc NewClient(config *ClientConfig) *Client {client.client = &http.Client{Timeout: config.RequestTimeout,Transport: newRoundTripper(config.RoundTripper,config.MetricRegistry),}…}
EngineeringDial TLS Handshake Request Resp Headers Resp Body IdleClient.Dohttp.Client.Timeoutnet.Dialer.Timeouthttp.Transport.TLSHandshake.Timeouthttp.Transport.ResponseHeader.Timeouthttp.Transport.IdleConn.Timeouthttps://blog.cloudflare.com/the-complete-guide-to-golang-net-http-timeouts/߹ʹΑΓࡉ͔͘ίϯτϩʔϧՄೳ
Engineering● Լखʹretry͢Δ͜ͱʹࣾαʔϏεΛDDOS͢Δ͜ͱ● ඞͣ Backoff ( Exponential )͢Δ͜ͱ● RetryϦΫΤετείʔϓͷOption (ClientείʔϓͰ͋Γ·ͤΜʣRETRYʹ͍ͭͯres, err = client.DoX(reqG,xhttp.WithRetry(&xhttp.RetryOptions{MaxAttempts: 2,RetryOn: func(response *http.Response) bool {return response.StatusCode != 200},TimeoutPerAttempt: time.Second,}))
Engineering● ૣ͍ஈ֊Ͱυϩοϓ͢Δख๏ͷҰͭͱͯ͠ CircuitBreakerύλʔϯɹ● https://engineering.linecorp.com/ja/blog/circuit-breakers-for-distributed-services/● ϗετͷঢ়ଶΛʮࣦഊ͢ΔϦΫΤετͷʯͰஅ● ʮࣦഊʯఆٛࣗମϢʔβʹͤΔ● ঢ়ଶʹΑΓʮϦΫΤετՄೳʯͱʮϦΫΤετෆՄೳʯͰ͚Δ● ϦΫΤετෆՄೳͷ߹୯७ʹ FailFastErr ฦ͢FAIL FASTରࡦʹ͍ͭͯ
Engineering● ׂͱύϥϝʔλ͕ଟ͍ͷͰɺΞϧΰϦζϜࣗମΛཧղ͢Δ্ઃఆ͢Δඞཁ͕͋ΔCIRCUIT BREAKERtype CircuitBreakerConfig struct {noop boolFailureRateThreshold float64MinimumRequestThreshold int64TrialRequestInterval time.DurationCircuitOpenWindow time.DurationCounterSlidingWindow time.DurationCounterUpdateInterval time.DurationIsFailed FailedJudgerKeyBuilder KeyBuilder}client := xhttp.NewClient(&xhttp.ClientConfig{CircuitBreakerConfig: &xhttp.CircuitBreakerConfig{},})
Engineering● > ϗετͷঢ়ଶΛʮࣦഊ͢ΔϦΫΤετͷʯͰஅ● Ͳ͏ͬͯૣ͑͘Δʁ༨ஊɹHIGH PERFORMANCECIRCUITBREAKERͷͨΊʹvar c int64c ++var c atomic.Int64c.Add(1)● AtomicͬͯcontentionมΘΒͳ͍ʢಉ͡มʹࢀর͢Δ͜ͱʣ● https://github.com/linxGnu/go-adder● Java LongAdderͷΞΠσΟΞʢatomicΛsharding)● ϚϧνϧʔνϯڥͰatomic.Int64ΑΓ2~4ഒૣ͍
EngineeringMETRICS/TRACINGͷ
Engineering● ObservabilityνʔϜͳͷͰͦͷॏཁੑΛ͘͢͝ཧղ͍ͯ͠Δ● ͑ͳ͍ͱ͍͚ͳ͍ͷ● ϦΫΤετLatencyߴ͍Ͱ͔͢ʁ● Latencyߴ͍࣌ͳͥʁOBSERVABILITYͷॏཁੑ● Latencyߴ͍͔Ͳ͏͔MetricsͰ͑Δ● Metrics Instrumentation prometheus client ͏● ͳͥʁΛ͑Δͷ Metrics + TracingͰ͑Δ● Tracing Instrumentation zipkin-go
EngineeringROUNDTRIPPERར༻ͯ͠INSTRUMENTtype ClientConfig struct {MetricRegistry *prometheus.RegistryTracer *zipkin.Tracer……}func newRoundTripper(originRt http.RoundTripper, mr *prometheus.Registry, tc*zipkin.Tracer) *roundtripper {… ..}client.client = &http.Client{Timeout: config.RequestTimeout,Transport: newRoundTripper(config.RoundTripper, config.MetricRegistry,config.Tracer),}
Engineering● ΧελϚΠζ͢ΔͨΊͷϑοΫ● ϦΫΤετૹΔॲཧΛΧελϚΠζɿ RoundTripperΛ͢● ίωΫγϣϯΛ࡞ΔॲཧΛΧελϚΠζɿDialContextΛ͢༨ஊɹNET/HTTPࢁϑοΫఏڙͯ͘͠ΕͯΔ● ϝτϦΫεΛऔಘ͢ΔͨΊͷϑοΫ● http/ClientTraceेɹ● https://blog.golang.org/http-tracing
Engineering࠷ޙ
Engineering● Ͱ͖Εࣾɺ͋Δ͍νʔϜڞ௨ͷϥΠϒϥϦἧͬͨ΄͏͕ྑ͍● http client, log package, metrics instrument…● ϚΠΫϩαʔϏε͍͠ɺͰ͖Εආ͚ͨ΄͏͕ྑ͍● ͍ۙ͏ͪʹxhttpΛOSS͢Δ● go͖ͰɺେنγεςϜΛ࡞Γ͍ͨਓ͝࿈བྷ͓ͪͯ͠Γ·͢
Engineering͝੩ௌ͋Γ͕ͱ͏͍͟͝·͢
line_developers
shimashima35
masashi_sutou
yoshinagaiwnl
tomohisa
aiji42
visional_engineering_and_design
yayoi_dd
stefafafan
akiomik
ybrliiu
megabitsenmzq
regonn
schacon
reverentgeek
rocio
jnunemaker
lauravandoore
garrettdimon
myddelton
chriscoyier
mthomps
cherdarchuk
keathley
tanoku