How we monitor microservices at Mercari microservices platform team

Talk at SRE session #1
https://sre-lounge.connpass.com/event/123701/

Explained the overview of our microservices monitoring.

Transcript

1. )PX
   XF
   NPOJUPS NJDSPTFSWJDFT BU
   .FSDBSJ
   .JDSPTFSWJDFT
   1MBUGPSN
   UFBN
2. !TQFTOPWB
3. $VSSFOU
   4UBUVT 
   FOHJOFFST 
   NJDSPTFSWJDFT 
   DIBOHFTXFFL
4. ,VCFSOFUFT
   $MVTUFS
   (,& $VSSFOU
   4UBDL %#
   4QBOOFS
   FUD .JDSPTFSWJDFT
   (P
   BQQ
   FUD
5. "SDIJUFDUVSF
   0WFSWJFX ($-# DMJFOU "1*
   HBUFXBZ #''
   NJDSPTFSWJDFT NJDSPTFSWJDFT %#T
6. )PX
   XF
   NPOJUPS NJDSPTFSWJDFT
   
7. 4UBSUJOH
   GSPN
   
   
   l8IZ
   XF
   NPOJUPS z
8. 
   
   8F
   XBOU
   UP
   LFFQ
   TZTUFNT
   XPSL
   QSPQFSMZ 8IZ
9. 8IFO
   B
   TZTUFN
   JT
   OPU
   XPSLJOH
   QSPQFSMZ
   XF
   OFFE
   UP EFUFDU
   JU
   JOWFTUJHBUF
   JU
   BOE
   pY
   JU
10. 4P
    IPX
    DBO
    XF
    EFUFDU
    B
    TZTUFN
    JTOU
    XPSLJOH
11. $PMMFDU
    NFUSJDT
    
    
    
    
    
    
    
    UP
    LOPX
    lXIBUz
12. .FUSJDT
13. ,VCFSOFUFT
    /PEF .JDSPTFSWJDF
    QPET .POJUPSJOH
    BHFOU
    QPE $PMMFDUJOH
    BSDIJUFDUVSF
    
    PWFSWJFX .FUSJDT
    TUPSBHF
14. .FUSJDT
    UFMM
    ZPV
    XIFUIFS
    B
    TZTUFN
    JT
    XPSLJOH
    QSPQFSMZ
    PS
    OPU
15. 5IJT
    NFUSJD
    UFMM
    ZPV
    B
    TZTUFN
    XBT
    TMPX
    UIFO
16. 5IJT
    NFUSJD
    UFMM
    ZPV
    B
    TZTUFN
    SFUVSOFE
    NBOZ
    FSSPST
17. :PV
    DBO
    LOPX
    B
    TZTUFN
    JT
    XPSLJOH
    CZ
    DIFDLJOH
    NFUSJDT
18. .FUSJDT
    EPO`U
    UFMM
    ZPV
    IPX
    TMPX
    BOE
    XIZ
    TMPX )PXFWFS
19. $PMMFDU
    USBDFT
    
    
    
    
    
    
    
    UP
    LOPX
    lIPXz
20. 5SBDFT
21. ,VCFSOFUFT
    /PEF .JDSPTFSWJDF
    QPET .POJUPSJOH
    BHFOU
    QPE $PMMFDUJOH
    BSDIJUFDUVSF
    PWFSWJFX 5SBDFT
    TUPSBHF
22. 5SBDFT
    UFMM
    ZPV
    IPX
    TMPX
    IPX
    CSPLFO
23. 5IJT
    USBDF
    UFMM
    ZPV
    UIF
    SFE
    QBSU
    UPPL
    NPTU
    PG
    UJNF
24. 5IJT
    USBDF
    UFMM
    ZPV
    XIFSF
    BO
    FSSPS
    IBQQFOFE
25. 1SPQBHBUJOH
    ASFRVFTU@JEA
    GPS
    EJTUSJCVUFE
    USBDJOH
26. 5PQPMPHZ TFSWJDF
    NBQ
27. 5SBDFT
    EPO`U
    UFMM
    ZPV
    XIZ )PXFWFS
28. $PMMFDU
    MPHT
    
    
    
    
    
    
    
    UP
    LOPX
    lXIZz
29. -PHT
30. ,VCFSOFUFT
    /PEF .JDSPTFSWJDF
    QPET .POJUPSJOH
    BHFOU
    QPE $PMMFDUJOH
    BSDIJUFDUVSF
    PWFSWJFX -PHT
    TUPSBHF
31. -PHT
    UFMM
    ZPV
    XIZ
    JU
    IBQQFOFE
32. 1SPQBHBUJOH
    ASFRVFTU@JEA
    GPS
    EJTUSJCVUFE
    MPHHJOH
33. 4FBSDIJOH
    MPHT
    XJUI
    lSFRVFTU@JE
    YYYYz
    BDSPTT
    NJDSPTFSWJDFT
34. %FpOJOH
    UIF
    MPH
    GPSNBU
    TUBOEBSE
    UP
    UP
    BOBMZ[F
    BDSPTT
    TFSWJDFT
36. &BDI
    NJDSPTFSWJDF
    TIPVME
    CF
    BCMF
    UP
    EFpOF
    JUT
    PXO
    MPH
    GPSNBU
    TP
    UIBU
    UIF
    TUBOEBSE
    EFpOFT
    POMZ
    DPNNPO
    BUUSJCVUF
    OBNFT
37. &OGPSDJOH
    UIF
    TUBOEBSE
    UP
    IVOESFET
    EFWFMPQFST
    UBLFT
    UJNF
    
    #F
    QBUJFOU
    BOE
    IBWF
    B
    MPOHUFSN
    WJTJPO
38. 5IF
    UISFF
    QJMMBS
    
    
    
    
    
    
    
    
    
    
    
    
    
    
    
    
    
    
    PG
    PCTFSWBCJMJUZ 8IBU
    EBUB
    EP
    XF
    DPMMFDU
39. 5IF
    UISFF
    QJMMBS
    
    
    
    
    
    
    
    
    
    
    
    
    
    
    
    
    PG
    PCTFSWBCJMJUZ .FUSJDT
    5SBDFT
    -PHT
40. 5IF
    3&%64&
    .FUIPE PS
    8PSL
    BOE
    3FTPVSDF
    .FUIPE )PX
    EP
    XF
    VTF
    UIF
    EBUB
41. 3FRVFTUT
    &SSPST
    %VSBUJPO XPSL
    NFUSJDT
42. 6UJMJ[BUJPO
    4BUVSBUJPO
    &SSPST SFTPVSDF
    NFUSJDT
43. IUUQTXXXEBUBEPHIRDPNCMPHNPOJUPSJOHJOWFTUJHBUJPO *OWFTUJHBUJPO
    'MPX  $IFDL
    3&% XPSL
    NFUSJDT
    UP
    LOPX
    8)"5
     $IFDL
    USBDFT
    UP
    LOPX
    )08
     $IFDL
    64& SFTPVSDF
    NFUSJDT
    UP
    LOPX
    8):
     $IFDL
    MPHT
    UP
    LOPX
    8):
44. IUUQTXXXEBUBEPHIRDPNCMPHNPOJUPSJOHJOWFTUJHBUJPO *OWFTUJHBUJPO
    'MPX
45. "MFSUJOH
46. 6TF
    3&%
    NFUSJDT
    GPS
    BMFSUJOH
47. 8F
    EPO`U
    VTF
    64&
    NFUSJDT
    GPS
    BMFSUJOH
    
    
    
    
    
    
    
    
    UP
    BWPJE
    GBMTF
    BMFSU
    JO
    NPTU
    DBTFT
48. .POJUPSJOH
    TZTUFN .JDSPTFSWJDF $IBU
    OPUJpDBUJPO 4.4
    QVTI
    OPUJpDBUJPO
    1IPOF
    $BMM $SJUJDBM
    BMFSU 8BSOJOH
    BMFSU
49. IUUQTNPO[PDPNCMPHPODBMM 8BSOJOH "
    IVNBO
    SFTQPOTF
    JT
    SFRVJSFE
    CVU
    OPU
    VSHFOUMZ
    
    5IFTF
    TIPVMEO`U
    XBLF
    ZPV
    BU
    OJHIU
    
    CVU
    TIPVME
    CF
    MPPLFE
    BU
    XJUIJO
    FJHIU
    IPVST
50. IUUQTNPO[PDPNCMPHPODBMM $SJUJDBM "O
    VSHFOU
    SFTQPOTF
    JT
    SFRVJSFE
    
    BOE
    TIPVME
    HFU
    UIF
    BUUFOUJPO
    PG
    BO
    PODBMMFS
    JNNFEJBUFMZ
51. 4-*T
    BOE
    4-0T
52. 6TF
    3&%
    NFUSJDT
    GPS
    4-*T
53. &TQFDJBMMZ
    XF
    VTF
    &SSPST
    BOE
    %VSBUJPO
    GPS
    4-*T
56. %FDJEJOH
    4-0
    JT
    OPU
    FBTZ
57. 8F
    QSPWJEF
    B
    SPVHIFYQFSJNFOUBM
    HVJEFMJOF
60. 0O$BMM
61. 8F`SF
    BEPQUJOH
    lZPV
    CVJME
    JU
    ZPV
    SVO
    JUz
    DVMUVSF
62. 
    1BHFS%VUZ
    VTFST
63. .BOZ
    FOHJOFFST
    OPX
    CFDPNF
    0O$BMM
    
    
    1MBUGPSN
    UFBN
    OFFET
    UP
    QSFQBSF
    UIF
    0O$BMM
    USBJOJOH
64. 5ISFF
    MBZFST
    FTDBMBUFT
    BGUFS
    NJOT
65. IUUQTSFTQPOTFQBHFSEVUZDPNPODBMMCFJOH@PODBMM
67. %PO`U
    XBLF
    VQ
    FOHJOFFST
    JO
    NJEOJHIU
    GPS
    XBSOJOH
    BMFSUT
68. .POJUPSJOH
    TZTUFN .JDSPTFSWJDF $IBU
    OPUJpDBUJPO 4.4
    QVTI
    OPUJpDBUJPO
    1IPOF
    $BMM $SJUJDBM
    BMFSU 8BSOJOH
    BMFSU
69. /FYU
    USZ
    JT
    JOUSPEVDJOH
    *ODJEFOU
    $PNNBOEFS
    SPMF
70. 3FTQPOTJCJMJUZ
    %FTJHO
71. ,VCFSOFUFT
    /PEF .JDSPTFSWJDF
    QPET .POJUPSJOH
    BHFOU
    QPE 5SBDFT
    TUPSBHF %FWFMPQFST
    NPOJUPS
    UIFJS
    NJDSPTFSWJDFT
72. ,VCFSOFUFT
    /PEF .JDSPTFSWJDF
    QPET .POJUPSJOH
    BHFOU
    QPE 5SBDFT
    TUPSBHF 1MBUGPSN
    UFBN
    NPOJUPST
    DPNNPO
    QBSUT
73. ,VCFSOFUFT
    /PEF .JDSPTFSWJDF
    QPET .POJUPSJOH
    BHFOU
    QPE 5SBDFT
    TUPSBHF 1MBUGPSN
    UFBN
    LFFQT
    NPOJUPSJOH
    FOWJSPONFOU
    XPSLT
74. 1MBZCPPL
75. 8F`WF
    TUBSUFE
    XSJUJOH
    QMBZCPPLT
    BT
    FWFO
    JG
    JU
    JT
    WFSZ
    SPVHI
    JU`T
    CFUUFS
    UIBO
    OPUIJOH
76. 8F
    DBOOPU
    QSFQBSF
    XFMMEFTDSJCFE
    QMBZCPPLT
    JO
    PWFSOJHIU
77. $POUJOVPVT
    QMBZCPPLT
    JNQSPWFNFOU
    XJUI
    HJU
    
80. 7JTVBMJ[BUJPO
81. )BWF
    B
    TJOHMF
    QVSQPTF
    GPS
    B
    HSBQI
82. .VMUJQVSQPTF
    HSBQI
    JT
    IBSE
    UP
    FWBMVBUF
84. *G
    ZPV
    IBWF
    BOPUIFS
    QVSQPTF
    
    
    
    
    
    
    
    
    
    BEE
    POF
    NPSF
    HSBQI
    UP
    TIPX
    JU
86. 4FF
    B
    TBNF
    NFUSJD
    GSPN
    EJ⒎FSFOU
    BOHMFT
87. 4FF
    B
    TBNF
    NFUSJD
    GSPN
    EJ⒎FSFOU
    BOHMFT
88. 4FF
    B
    TBNF
    NFUSJD
    GSPN
    EJ⒎FSFOU
    BOHMFT
89. 3FDBQ "MXBZT
    TUBSU
    GSPN
    l8IZ
    XF
    NPOJUPS z
    $PMMFDU
    NFUSJDT
    USBDFT
    BOE
    MPHT
    *OWFTUJHBUF
    XJUI
    3&%64&
    NFUIPE
    6TF
    3&%
    GPS
    BMFSUJOH
    %FpOF
    4-0
    XJUI
    3&%
    &WFSZ
    EFWFMPQFS
    EPFT
    NPOJUPSJOH
    )BWF
    B
    TJOHMF
    QVSQPTF
    GPS
    B
    HSBQI
90. 5IBOL
    ZPV
    GPS
    TIBSJOH
    CFBVUJGVM
    QJDUVSFT
    
    0MFH
    .BHOJ
    4IVZV
    ,JN
    ,BU
    +BZOF
    +BWPO
    4XBCZ
    "EPOZJ
    (ÂCPS