堅牢なTCPサーバを作るために - katsubushiの知見から/kamakura.go#5

kamakura.go #5

Transcript

1. ݎ࿚ͳTCPαʔόΛ࡞ΔͨΊʹ
   ʙkatsubushiͷ஌ݟ͔Βʙ
   2019-06-22 Kamakura.go #5
   @fujiwara
   

   View Slide

2. @fujiwara
   SRE(૯຿෦)
   github.com/fujiwara
   sfujiwara.hatenablog.com
   WEB+DB Press vol.111 SREνʔϜ࿈ࡌ
   Perl Hackers HubͰAWS X-Rayͷ࿩Λॻ
   ͖·ͨ͠
   

   View Slide

3. katsubushi - ෼ࢄϢχʔΫID࠾൪ػ
   • ϢχʔΫͳ࣌ܥྻॱͷInt64 IDΛൃߦ͢Δ
   • ෼ࢄ؀ڥ(ෳ਺୆)ͰඃΒͣಈ͘
   • Memcached ProtocolΛ࣮૷͍ͯ͠Δ
   • Go੡ͷಠ࣮ࣗ૷ϛυϧ΢ΣΞ
   github.com/kayac/go-katsubushi
   

   View Slide

4. ݎ࿚ͳαʔόʁ
   ݎ࿚ is ...
   • ϦιʔεϦʔΫ͠ͳ͍
   • མͪͳ͍
   • (ϓϩμΫτͷੑ্࣭) ID͕ܾͯ͠ඃΒͳ͍
   • →γϟʔσΟϯά͞ΕͨผDBʹಉҰID͕ൃߦ͞ΕͨΒࢮ
   

   View Slide

5. memcached protocol ͷ࠾༻ཧ༝
   • ܰྔ
   • HTTPͷΑ͏ʹ༨ܭͳϔομ͕ͳ͍
   • Int64 Λ1ݸฦ͢ͷʹϔομ͕Կඦbyteͱ͔ͪΐͬͱ…
   • ΫϥΠΞϯτ͕ߴ଎
   • ౰࣌ओʹ࢖༻͍ͯͨ͠Perlʹ͸Cache::Memcached::Fastͱ
   ͍͏ߴ଎ͳ࣮૷
   ID͸େྔʹൃߦ͢Δ͜ͱ͕͋ΔͷͰ͍ܰͷ͕Α͍
   

   View Slide

6. HTTPͳΒͱ΋͔͘memcached protocolΛॲཧ͢Δαʔό͸
   طଘͷϑϨʔϜϫʔΫ͕ͳ͍
   ॻ͚͹͍͍ΑͶɺ؆୯ͩ͠
   

   View Slide

7. Go Ͱ TCP αʔό
   net package Ͱ؆୯ʹͰ͖Δ
   Listen, Acceptͯ͠ConnΛgoroutineͰॲཧ͢Δ͚ͩ
   ln, err := net.Listen("tcp", ":8080")
   if err != nil {
   // handle error
   }
   for {
   conn, err := ln.Accept()
   if err != nil {
   // handle error
   }
   go handleConnection(conn)
   }
   

   View Slide

8. Go Ͱ TCP Echo αʔόɺ௒؆୯
   func handleConnection(conn net.Conn) {
   defer conn.Close() // ൈ͚Δͱ͖ʹ੾அ
   b := bufio.NewReader(conn)
   for {
   line, err := b.ReadString('\n') // 1ߦಡΉ
   if err != nil {
   return
   }
   _, err = io.WriteString(conn, line) // Φ΢Ϝฦ͠
   if err != nil {
   return
   }
   }
   }
   

   View Slide

9. ͔͠͠ݱ࣮͸໘౗
   

   View Slide

10. ϦιʔεϦʔΫ͠ͳ͍
    

    View Slide

11. ແ௨৴λΠϜΞ΢τ
    • ܨ͗ͬͺͳ͠Ͱ͍ͳ͘ͳΔΫϥΠΞϯτରࡦ
    • ΫϥΠΞϯτ͕TCPΛ͖Ε͍ʹ੾அ͢Δͱ͸ݶΒͳ͍
    • αʔόଆͰ੾Βͳ͍ͱίωΫγϣϯ(goroutine)͕ϦʔΫ͢Δ
    

    View Slide

12. net/Conn.SetDeadline
    ίωΫγϣϯʹࢦఆͨ࣌͠ࠁ·Ͱ௨৴͕ͳ͔ͬͨΒ
    ΤϥʔʹͳΔΑ͏ઃఆ͢Δ
    ௨৴ཱ֬௚ޙɺಡΈॻ͖Ͱ͖ͨλΠϛϯάͰ࠶౓ઃఆ
    → ແ௨৴λΠϜΞ΢τ͕࣮૷Ͱ͖Δ
    type Conn interface {
    SetDeadline(t time.Time) error
    }
    

    View Slide

13. for {
    conn, err := ln.Accept()
    if err != nil {
    // handle error
    }
    conn.SetDeadline(time.Now().Add(idleTimeout))
    go handleConnection(conn)
    }
    func handleConnection(conn net.Conn) {
    defer conn.Close()
    buf := bufio.NewReader(conn)
    for {
    line, _ := buf.ReadString('\n') // 1ߦಡΉ
    conn.SetDeadline(time.Now().Add(idleTimeout))
    // ^λΠϜΞ΢τԆ௕
    

    View Slide

14. ϓϩηεऴྃ࣌ʹ੾அ
    αʔό΋ͪΌΜͱ੾அ͠ͳ͍ͱߦّ͕ѱ͍ΑͶ?
    func (app *App) handleConn(ctx context.Context, conn net.Conn) {
    defer conn.Close() // ͜ͷίωΫγϣϯͷॲཧΛࣗൃతʹ੾Δͱ͖
    go func() {
    conn.Close() // ͪΌΜͱ੾Δʂ
    }()
    // ...
    

    View Slide

15. ͜͜Ͱ໰୊Ͱ͢
    ͜ͷίʔυͰ͸ͳʹ͔͕ϦʔΫ͠·͢
    func (app *App) handleConn(ctx context.Context, conn net.Conn) {
    defer conn.Close() // ͜ͷίωΫγϣϯͷॲཧΛࣗൃతʹ੾Δͱ͖
    go func() {
    conn.Close() // ͪΌΜͱ੾Δʂ
    }()
    // ...
    

    View Slide

16. goroutineϦʔΫ
    handleConn͕returnͯ͠΋goroutine͕ࢮͳͳ͍
    func (app *App) handleConn(ctx context.Context, conn net.Conn) {
    defer conn.Close() // ͜ͷίωΫγϣϯͷॲཧΛࣗൃతʹ੾Δͱ͖
    go func() {
    conn.Close() // goroutine ͕ੜ͖࢒Δ
    }()
    // ...
    

    View Slide

17. मਖ਼ํ๏1
    ࢠ context Λ࡞ͬͯ defer cancel()
    ࣗൃతऴྃ࣌ɺ਌ऴྃ࣌ͲͪΒͰ΋ίωΫγϣϯ੾அͱ
    goroutineऴ͕ྃͰ͖ΔͷͰϦʔΫ͠ͳ͍
    func (app *App) handleConn(ctx context.Context, conn net.Conn) {
    ctx2, cancel := context.WithCancel(ctx)
    defer cancel()
    go func() {
    conn.Close()
    }()
    // ...
    1 Fix goroutine leak. #28 https://github.com/kayac/go-katsubushi/pull/28/files
    

    View Slide

18. མͪͳ͍
    

    View Slide

19. ʮ͜ͷ katsubushi ಈ͍ͯΔ͔ͳ?ʯ
    $ curl localhost:11212
    VALUE / 0 18
    591644124189298688
    VALUE HTTP/1.1 0 18
    591644124189298689
    END
    ERROR
    ERROR
    ERROR
    ʮID͸ฦ͖͚ͬͯͨͲͳʹ͜Ε??ʯ
    ʮ͋ɺcurlͰୟ͍ͯͨɻɻɻʯ
    

    View Slide

20. Կ͕ى͖͍͔ͯͨ
    curl ͸ katsubushi ʹ HTTP ͰϦΫΤετͨ͠(౰વ)
    $ curl -v 127.0.0.1:11212
    > GET / HTTP/1.1
    > Host: 127.0.0.1:11212
    > User-Agent: curl/7.65.1
    > Accept: */*
    > (ۭߦ)
    HTTPͷϦΫΤετ͸ memcached protocol Ͱ΋ղऍͰ͖Δʂ
    memcached protocol Ͱ஋ͷऔಘ: GET key1 key2
    HTTP Ͱ GET ϦΫΤετͷ1ߦ໨: GET / HTTP/1.1
    

    View Slide

21. ͭ·Γ͜͏͍͏Ϩεϙϯε͕ฦΔ
    VALUE / 0 18 # / ͱ͍͏keyͷ஋(flag=0, 18byte)
    591644124189298688
    VALUE HTTP/1.1 0 18 # HTTP/1.1 ͱ͍͏keyͷ஋(flag=0, 18byte)
    591644124189298689
    END # HTTPͷ1ߦ໨ʹ͸Ϩεϙϯε͕ฦͤͨΑʂ
    ERROR # Host: ͱ͔஌Βͳ͍!
    ERROR # User-Agnet: ͱ͔஌Βͳ͍!!
    ERROR # Accept: ͱ͔஌Βͳ͍!!!
    

    View Slide

22. Ұํͦͷ͜Ζ HTTP Λ৯΂ͨ katsubushi ͸
    panic: runtime error: invalid memory address or nil pointer dereference
    [signal SIGSEGV: segmentation violation code=0x1 addr=0x18 pc=0x134807f]
    goroutine 20 [running]:
    github.com/kayac/go-katsubushi.(*App).handleConn(0xc00012a000, 0x14a3de0, 0xc0000ac038)
    /Users/fujiwara/src/github.com/kayac/go-katsubushi/app.go:160 +0x2cf
    created by github.com/kayac/go-katsubushi.(*App).Listen
    /Users/fujiwara/src/github.com/kayac/go-katsubushi/app.go:131 +0x2ed
    ʮ͠ɺࢮΜͰΔ…ʯ
    

    View Slide

23. Կ͕ى͖͍͔ͯͨ
    ಡΈऔͬͨϦΫΤετߦ([]byte)ΛίϚϯυʹ࣮ͯ͠ߦ
    HTTPϔομ͓ΘΓͷۭߦΛಡΜͩͱ͜ΖͰ…
    cmd, err := app.BytesToCmd(scanner.Bytes())
    if err != nil {
    return
    }
    err = cmd.Execute(app, conn) // ͜͜Ͱࢮ
    

    View Slide

24. ൜ਓ
    func (app *App) BytesToCmd(data []byte) (cmd MemdCmd, err error) {
    if len(data) == 0 { // ۭߦͳͷͰ 0 byte
    return nil, nil // ← ͍ͭ͜
    }
    foo, err := Foo() ͱ͍͏ΠϯλʔϑΣʔεΛΈͨ
    ී௨ͷGopher͕ߟ͑Δ͜ͱ
    ʮerr != nil ͳΒ foo == nil ͔΋͔ͩΒؾΛ͚ͭΑ͏ʯ
    ʮerr == nil ͳΒ foo != nil ͩΑͶʂʯ
    

    View Slide

25. nil, nil Λฦ͢ͷ͕ѱ͍
    ॻ͍ͨͷ͸Ͳͬͪ΋ࣗ෼͚ͩͲ݀ʹམͪͨ
    ௨ৗmemcached protocolͰϦΫΤετʹۭߦ͸དྷͳ͍
    ຊ൪ӡ༻தͷϓϩηε͕ଈࢮͯ͠ͼͬ͘Γͨ͠
    katsubushiΫϥΠΞϯτϥΠϒϥϦ͸failoverΛ࣮૷͍ͯͨ͠
    daemontools Ͱ͙͢ʹ࠶ىಈ͞Εͨ
    ͷͰக໋ইʹ͸ͳΒͳ͔ͬͨ
    γεςϜશମͰݎ࿚ʹ͍͖ͯ͠·͠ΐ͏…
    

    View Slide

26. ରԠ2
    func (app *App) BytesToCmd(data []byte) (cmd MemdCmd, err error) {
    if len(data) == 0 {
    - return nil, nil
    + return nil, errors.New("No command")
    ஌Βͳ͍ίϚϯυ͕དྷͨΒଈ࠲ʹ઀ଓΛ੾ͬͨ΄͏͕͍͍͔΋
    2 Never die #8 https://github.com/kayac/go-katsubushi/pull/8
    

    View Slide

27. ༨ஊ1
    ࣮͸ curl ͸ telnet Ͱ͖·͢
    $ curl telnet://127.0.0.1:11212
    GET id
    VALUE id 0 18
    591655476635111424
    END
    quit
    

    View Slide

28. ༨ஊ2
    ʮ1ߦ໨Λղऍͯ͠key͕ / HTTP/1.[01] ͳΒHTTPɻɻɻʯ
    ʮͭ·Γಉ͡ϙʔτͰ memcached ͱ HTTP ྆ରԠ
    αʔό͕ॻ͚ΔͷͰ͸!??ʯ
    ్த·Ͱॻ͍ͯਖ਼ؾʹ໭ͬͯ຅ʹ
    

    View Slide

29. (ϓϩμΫτͷੑ্࣭)
    ID͕ܾͯ͠ඃΒͳ͍
    

    View Slide

30. katsubushi ID ͸࣌ࠁϕʔε
    +-+-----------------------------------------+----------+------------+
    | | Timestamp | WorkerID | Sequence |
    +-+-----------------------------------------+----------+------------+
    |0|00001001001001010100110010111011011100100|0000000001|000000000000|
    +-+-----------------------------------------+----------+------------+
    | | 78,560,982,756ms since epoch | 1| 0|
    +-+-----------------------------------------+----------+------------+
    = 2017-06-28T06:29:42.756Z
    ࣌ࠁ͕໭Δͱಉ͡ID͕ൃߦ͞ΕΔՄೳੑ͕͋Δ
    

    View Slide

31. monotonic time
    https://golang.org/doc/go1.9#monotonic-time
    Go 1.9ͰOS࣌ࠁ͕໭ͬͯ΋ time.Now() ͸໭Βͳ͍
    monotonic time͕࣮૷͞Εͨ
    2017−01−01ͷӞඵൃੜ࣌
    Cloudflare͕࣌ࠁͷר͖໭ΓͰো֐3
    ࣌ࠁ͕໭ͬͨ݁Ռ math/rand.Int63n ʹෛͷ஋͕౉Γpanic
    3 How and why the leap second affected Cloudflare DNS https://blog.cloudflare.com/how-and-why-the-leap-second-affected-
    cloudflare-dns/
    

    View Slide

32. monotonic time Ҏલͷ katsubushi
    ts := g.timestamp()
    // for rewind of server clock
    if ts < g.lastTimestamp {
    return 0, errors.New("system clock was rollbacked")
    }
    // ུ
    g.lastTimestamp = ts
    ࣌ࠁ͕ר͖໭ͬͨΒΤϥʔʹͯ͠๷ޚ
    ID͕ඃΔ͙Β͍ͳΒൃߦͰ͖ͳ͍ํ͕Ϛγ
    

    View Slide

33. monotonic time ରԠ4
    katsubushi ID ͷ࣌ࠁ෦෼͸ 2015-01-01T00:00:00 UTC
    Λىݯͱͨ͠ܦաඵ਺
    var Epoch = time.Date(2015, 1, 1, 0, 0, 0, 0, time.UTC)
    d := time.Now().Sub(Epoch)
    ͜ͷΑ͏ʹɺ೔࣌ࢦఆͰ࡞ͬͨ஋͔Βܭࢉ͢Δͱmonotonicʹ
    ͳΒͳ͍
    4 use Monotonic time #21 https://github.com/kayac/go-katsubushi/pull/21
    

    View Slide

34. monotonic time ରԠ
    now := time.Now()
    startedAt := now
    offset := now.Sub(Epoch) // ىಈ࣌ͷ࣌ࠁͱepochͷࠩΛܭࢉ͓ͯ͘͠
    // ...
    d := time.Now().Sub(startedAt) + offset
    time.Now() Ͱ࡞ͬͨ஋ಉ࢜Λൺֱͯ͠offsetΛௐ੔͢Ε͹
    ר͖໭Βͳ͍
    

    View Slide

35. ࣌ࠁͷଞʹ΋ඃΔՄೳੑ͕͋Δ෦෼͕…
    +-+-----------------------------------------+----------+------------+
    | | Timestamp | WorkerID | Sequence |
    +-+-----------------------------------------+----------+------------+
    |0|00001001001001010100110010111011011100100|0000000001|000000000000|
    +-+-----------------------------------------+----------+------------+
    | | 78,560,982,756ms since epoch | 1| 0|
    +-+-----------------------------------------+----------+------------+
    = 2017-06-28T06:29:42.756Z
    WorkerID = ಉ࣌ʹಈ࡞͍ͯ͠ΔϓϩηεؒͰҰҙͷID
    

    View Slide

36. RedisΛ࢖ͬͯҰҙੑΛ୲อ͢Δ
    Ҏલͷൃදʹ͋ΔͷͰͲ͏ͧ
    https://speakerdeck.com/fujiwara3/katsubushi?slide=79
    

    View Slide

37. ؂ࢹ΋େࣄ
    • github.com/fukata/golang-stats-api-handler
    • GoͷϝτϦΫεΛऔΔ
    • mackerel-plugin-gostats ΋͋ΔΑ
    • net/http/pprof ͰϓϩϑΝΠϧऔಘͷޱΛ։͚Δ
    

    View Slide

38. ·ͱΊ
    • GoͰTCPαʔόΛ࡞Δͷ͸؆୯
    • Ͱ΋࣮ӡ༻Ͱݎ࿚ͳ΋ͷΛ࡞Δʹ͸͍Ζ͍Ζ͋Δ
    • Ұݟ؆୯ʹݟ͑Δ΋ͷͰ΋ɺࣗલ࣮૷͸প΁ͷୈҰา
    

    View Slide