Pushsphere: LINE's Secret Recipe for Reliable, Prompt and High-Volume Push Notifications

ユーザーは、LINEのメッセージング・プラットフォームがメッセージや音声通話を確実で迅速に通知してくれることを期待しています。プッシュ通知が間に合わず、重要なメッセージや通話を見逃してしまうことは、LINEとしては絶対に避けたい事態です。しかし、APNsやFCMのようなプッシュ通知サービス自体が、特にトラフィックが多い場合、必ずしも信頼できるとは限らない中で、どうすれば信頼性の高い迅速な配信を実現できるのでしょうか？

本講演では、インスタント・メッセージに最適化されたLINEのプッシュ通知配信メカニズム、Pushsphereをオープンソースで公開します。1月1日に世界中の新年の挨拶を一度も滞りなく配信することを可能にした、洗練されたテクニックをご覧ください！

Keywords: Client-side load-balancing, Unreliable third-party services, Circuit breakers and automatic retries, High-volume service on Kubernetes, Armeria

Transcript

1. 1VTITQIFSF
   -*/&`T
   4FDSFU
   3FDJQF
   GPS 3FMJBCMF
   1SPNQU
   BOE
   )JHIWPMVNF
   1VTI
   /PUJpDBUJPOT %FWFMPQFS
   F9QFSJFODF
   -*/&
   1MVT
   $PSQPSBUJPO 5SVTUJO
   -FF *LIVO
   6N
2. "HFOEB • 0VS
   $IBMMFOHFT
   • )JHIMFWFM
   %FTJHO • *NQMFNFOUBUJPO
   %FUBJMT • 3FDBQ
   
   'VUVSF
   8PSL
3. .JTTFE
   EJTDPVOU
   QSPNP .BZCF
   OFYU
   UJNFʜ .JTTFE
   XJGF`T
   DBMM *`N
   EFBE
   CZ
   UPOJHIUʜ .JTTFE
   EJTBTUFS
   XBSOJOH 8IZʜ 8F
   CFMJFWFE
   JO
   ZPV .BUUFS
   PG
   -JGF
   BOE
   %FBUI *OTUBOU
   NFTTBHJOH
   QPTFT
   DIBMMFOHFT
   XJUI
   QVTI
   OPUJpDBUJPOT
4. /FX
   :FBS
   WT
   &BSUIRVBLF
   WT
   .JTTJMF
   8IJDI
   JT
   UIF
   CJHHFS
   UISFBU
   UP
   PVS
   TFSWFST
5. • .PTUMZ
   TUBCMF • ʜ
   XJUI
   TP
   NBOZ
   #65T
   
   • .JTCFIBWJOH
   JOTUBODFT • 4MPX
   SFTQPOTFT
   
   UJNFPVUT • 4VEEFO
   EJTDPOOFDUJPOT "1/T
   WT
   'JSFCBTF #FMJFWF
   NF
   5IFZ
   BSF
   OPU
   CVJMU
   GPS
   UIJT
6. • 3FUSZJOH
   B
   QVTI
   OPUJpDBUJPO • )PX
   NBOZ
   UJNFT • )PX
   NVDI
   EFMBZ • (JWJOH
   VQ
   B
   TFSWFS • 8IFO
   UP
   BCBOEPO
   B
   CBE
   TFSWFS
   • 8IJDI
   TFSWFS
   TIPVME
   XF
   TXJUDI
   UP
   0VU
   PG
   T • 8JMM
   JU
   HFU
   IFBMUIZ
   BHBJO %FDJTJPOT
   %FDJTJPOT
   %FDJTJPOT 3FBDIJOH
   BU
   UIF
   TDBMF
   XIFSF
   OBÏWF
   SFUSJFT
   XPO`U
   XPSL
   BOZNPSF
7. .JOE
   UIF
   2VPUB :FT
   SFUSJFT
   EP
   DPVOU 429 Too Many Requests 1IPUP
   CZ
   &IUFTIBNVM
   )BRVF
   "EJU
   PO
   6OTQMBTI
8. "HFOEB • 0VS
   $IBMMFOHFT • )JHIMFWFM
   %FTJHO
   • *NQMFNFOUBUJPO
   %FUBJMT • 3FDBQ
   
   'VUVSF
   8PSL
9. $MJFOU
   -JCSBSZ
   "SNFSJB
   DVTUPNJ[BUJPOT • -PBE#BMBODFS • 3FUSZ
   3VMFT (BUFXBZ
   4FSWFS /FUUZ +BWB
   7JSUVBM
   .BDIJOF 1VTITQIFSF )551
   .JDSPTFSWJDF
   'SBNFXPSL "1/T -*/& .FTTBHJOH 4FSWFST
10. %$ 1VTITQIFSF
    (BUFXBZ
    4FSWFS %$ 1VTITQIFSF
    (BUFXBZ
    4FSWFS ʜ Y%4
    ;POFBXBSF 3PVUJOH 0VUMJFSEFUFDUJOH -PBE
    #BMBODFS
11. "HFOEB • 0VS
    $IBMMFOHFT • )JHIMFWFM
    %FTJHO • *NQMFNFOUBUJPO
    %FUBJMT
    • 3FDBQ
    
    'VUVSF
    8PSL
12. *OTJEF
    1VTITQIFSF
    $MJFOU &OEQPJOU
    .BOBHFS -PBE
    #BMBODFS $JSDVJU
    #SFBLFS "VUIFOUJDBUJPO
    -BZFS N5-4
    
    0"VUI
     5SBOTQPSU
    -BZFS )JHIQFSGPSNBODF
    OPOCMPDLJOH
    DPNNVOJDBUJPO
    WJB
    
    
    
    
    
    
    
    
    
    
    
    
    
    
    
    
    
    
    
    
    
    
    
    /FUUZ .FTTBHF
    5SBOTGPSNBUJPO $POWFSUT
    GPS
    '$."1/4
    DPNQBUJCJMJUZ 6OJGJFE
    *OUFSGBDF "VUPNBUJD
    3FUSZ 'FFECBDL 0O
    'BJMVSF *OUFSOBM
    $PNQPOFOUT
13. *OTJEF
    1VTITQIFSF
    $MJFOU &OEQPJOU
    .BOBHFS -PBE
    #BMBODFS $JSDVJU
    #SFBLFS "VUIFOUJDBUJPO
    -BZFS N5-4
    
    0"VUI
     5SBOTQPSU
    -BZFS )JHIQFSGPSNBODF
    OPOCMPDLJOH
    DPNNVOJDBUJPO
    WJB
    
    
    
    
    
    
    
    
    
    
    
    
    
    
    
    
    
    
    
    
    
    
    
    /FUUZ .FTTBHF
    5SBOTGPSNBUJPO $POWFSUT
    GPS
    '$."1/4
    DPNQBUJCJMJUZ 6OJpFE
    *OUFSGBDF "VUPNBUJD
    3FUSZ 'FFECBDL *OUFSOBM
    $PNQPOFOUT 0O
    'BJMVSF
14. 6OJpFE
    *OUFSGBDF // Send a push notification to iOS val appleProfile = Profile.forApple( URI("https://api.push.apple.com"), certChain, privateKey, "bundleId") val client = PushClient.of(appleProfile) val push = Push.forApple( title = "Push title", body = "Push notification body") val req = PushRequest( provider = PushProvider.APPLE, deviceToken = "your_device_token_here", push = push) val result: PushResult = client.send(req) // Send a push notification to Android val firebaseProfile = Profile.forFirebase( URI("https://fcm.googleapis.com"), googleCredentials) val client = PushClient.of(firebaseProfile) val push = Push.forFirebase( title = "Push title", body = "Push notification body") val req = PushRequest( provider = PushProvider.FIREBASE, deviceToken = "your_device_token_here", push = push) val result: PushResult = client.send(req) 4JOHMF
    FOUSZ
    QPJOU
    GPS
    BMM
    QVTI
    QMBUGPSNT
15. • %JTUSJCVUFT
    USBGpD
    FWFOMZ
    BDSPTT
    BWBJMBCMF
    FOEQPJOUT • 4JNQMF
    GBJS
    BOE
    FGGFDUJWF
    VOEFS
    OPSNBM
    DPOEJUJPOT 3FUSZBXBSF
    -PBE
    #BMBODFS 3PVOE
    SPCJO
    BT
    UIF
    CBTF
    TUSBUFHZ &OTVSJOH
    )JHI
    %FMJWFSZ
    4VDDFTT
    5ISPVHI
    4NBSUFS
    5SBGGJD
    %JTUSJCVUJPO • 0O
    SFUSZ
    QSFWJPVTMZ
    BUUFNQUFE
    FOEQPJOUT
    BSF
    TLJQQFE • 1SFWFOUT
    SFQFBUFE
    GBJMVSFT
    DBVTFE
    CZ
    GBVMUZ
    OPEFT 3FUSZBXBSF
    MPHJD & & & & & & & & 33 & & & & & & & & 33 4FMFDU
    FOEQPJOU 4LJQ
    & TJODF
    JU
    XBT
    BMSFBEZ VTFE
    JO
    UIF
    QSFWJPVT
    BUUFNQU 4FMFDU
    FOEQPJOU
16. #FGPSF
    2VPUBBXBSFOFTT • #MJOE
    SFUSJFT
    DBO
    FYIBVTU
    RVPUB CMPDLJOH
    EFMJWFSZ
    PG
    PUIFS
    NFTTBHFT • 3FUSZ
    TUPSNT
    SFEVDF
    TZTUFN
    TUBCJMJUZ
    BOE
    PWFSBMM
    EFMJWFSZ
    FGpDJFODZ "GUFS
    2VPUBBXBSFOFTT • 2VPUBBXBSF
    MPHJD
    BWPJET
    SFUSZJOH
    FOEQPJOUT
    OFBS
    RVPUB
    MJNJUT • #BMBODFT
    IJHI
    TVDDFTT
    SBUF
    BOE
    RVPUB
    QSFTFSWBUJPO 2VPUBBXBSF
    "VUPNBUJD
    3FUSJFT 3FMJBCMF
    EFMJWFSZ
    XJUIPVU
    PWFSMPBEJOH "1/T 4FOE
    B
    QVTI
    OPUJpDBUJPO 'BJM 3FUSZ
    MPHJD 4VDDFTT 8BT TVDDFTTGVM )BT
    RVPUB SFNBJOJOH :FT 5PP
    NBOZ BUUFNQUT /P :FT :FT /P /P 3FTFOE )BOEMF B
    SFTQPOTF
17. • $JSDVJU
    #SFBLFS
    QFS
    &OEQPJOU • &BDI
    QVTI
    SFTVMU
    JT
    SFQPSUFE
    UP
    JUT
    DJSDVJU • 'BVMUZ
    &OEQPJOU • 0O
    GBJMVSF
    CSFBLFS
    PQFOT
    ˠ
    FOEQPJOU
    JT
    SFNPWFE • "VUP
    3FQMBDFNFOU • 4FMFDUT
    B
    GSFTI
    FOEQPJOU
    GSPN
    %/4SFGSFTIFE
    DBOEJEBUFT • -PBE
    #BMBODF
    4UBCJMJUZ • .BJOUBJOT
    FWFO
    USBGpD
    XJUI
    B
    DPOTJTUFOU
    QPPM 3FTJMJFOU
    &OEQPJOU
    .BOBHFNFOU
    XJUI
    $JSDVJU
    #SFBLFST $BOEJEBUF
    1PPM &OEQPJOU $JSDVJU
    #SFBLFS ⚡ *T
    UIF
    DJSDVJU PQFO &OEQPJOU
    .BOBHFS &OEQPJOU
    1PPM :FT %POF /P 
    3FNPWF
    UIF
    VOIFBMUIZ
    
    
    FOEQPJOU
    GSPN
    UIF
    QPPM 
    1SPNPUF
    B
    DBOEJEBUF
    
    
    
    UP
    BO
    FOEQPJOU 💥 $ & ⚡ 3FQPSU TVDDFTTGBJMVSF $MJFOU & ⚡ & ⚡ & ⚡ $ $ $ $
18. • 0ODBMM
    BMFSUT
    ESPQQFE
    
    ESBNBUJDBMMZ
    0WFS
    
    BMFSUTZFBS
    JO
    UIF
    MFHBDZ
    
    MPPTF
    UISFTIPMET
    0OMZ
    
    BMFSUT
    TP
    GBS
    JO
    
    XJUI
    TUSJDUFS
    DPOEJUJPOT • 'JOBM
    QVTI
    GBJMVSFT
    ESPQ
    UP
    OFBS
    [FSP 0OMZ
    B
    IBOEGVM
    PG
    FSSPST
    PCTFSWFE
    JO
    UIF
    MBTU
    
    EBZT 1VTITQIFSF
    $MJFOU
    JO
    "DUJPO 'SPN
    OPJTZ
    BMFSUT
    UP
    OFBS[FSP
    GBJMVSFT
19. *OJUJBM
    1VTITQIFSF
    *OGSBTUSVDUVSF 0OF
    SFWFSTF
    QSPYZ
    IBOEMFE
    BMM
    USBGpD -*/& .FTTBHJOH 4FSWFST "1/T )551 )551 )551 - )551 )551 1VTITQIFSF
    ,VCFSOFUFT
    $MVTUFS
20. • -
    -PBE
    #BMBODFS
    GBJMFE
    UP
    EJTUSJCVUF
    USBGpD
    FWFOMZ • -
    DPVMEO`U
    IBOEMF
    ,
    214 • 5IF
    -
    JUTFMG
    XBT
    B
    TJOHMF
    QPJOU
    PG
    GBJMVSF • $4-#
    XBT
    EJGpDVMU
    UP
    JNQMFNFOU
    JO
    ,VCFSOFUFT • *U
    EJE
    OPU
    TVQQPSU
    NVMUJSFHJPO
    BSDIJUFDUVSF 8IZ
    EJEO`U
    UIF
    *OJUJBM
    4ZTUFN
    4DBMF
21. $FOUSBM
    %PHNB
    $POUSPM
    1MBOF
    $%$1
    JT
    BO
    Y%4CBTFE
    TFSWJDF
    NFTI
    TPMVUJPO • &OBCMFT
    DMJFOUTJEF
    MPBE
    CBMBODJOH
    JO
    ,VCFSOFUFT • /P
    DFOUSBM
    MPBE
    CBMBODFS
    r
    BWPJET
    TJOHMF
    QPJOUT
    PG
    GBJMVSF • 3FBMUJNF
    FOEQPJOU
    EJTDPWFSZ
    BOE
    EZOBNJD
    VQEBUFT • 4VQQPSUT
    NVMUJEBUB
    DFOUFS
    BOE
    [POFBXBSF
    SPVUJOH $%$1
    0VS
    "OTXFS
    UP
    -FHBDZ
    3FMJBCJMJUZ
    *TTVFT
22. -*/& .FTTBHJOH 4FSWFST "1/T $FOUSBM
    %PHNB $POUSPM
    1MBOF 'FUDI
    FOEQPJOU
    JOGPSNBUJPO WJB
    Y%4
    QSPUPDPM %$ /PEF /PEF1PSU LVCFQSPYZ /PEF /PEF1PSU LVCFQSPYZ %$ /PEF /PEF1PSU LVCFQSPYZ /PEF /PEF1PSU LVCFQSPYZ &WFOMZ
    TFOE
    USBGpD
    UP
    ,VCFSOFUFT
    OPEFT
    VTJOH
    
    
    
    
    
    $4-#
23. "HFOEB • 0VS
    $IBMMFOHFT • )JHIMFWFM
    %FTJHO • *NQMFNFOUBUJPO
    %FUBJMT • 3FDBQ
    
    'VUVSF
    8PSL
    
24. • "EWBODFE
    MPBE
    CBMBODJOH
    BMHPSJUIN • ;POFBXBSF • "QQMJDBUJPOMFWFM
    IJOUT
    SFUSJFT
    DJSDVJU
    CSFBLFS
    TUBUVT
    RVPUB
    ʜ • 5ZQJDBM
    ,VCFSOFUFT
    TFUVQ
    /05
    SFBEZ
    GPS
    IJHI
    USBGpD • $MJFOUTJEF
    MPBE
    CBMBODJOH
    JT
    UIF
    XBZ • $POTJEFS
    HPJOH
    lTJEFDBSMFTTz 3FDBQ 3FMJBCMF
    QVTI
    OPUJpDBUJPO
    EFMJWFSZ
    JT
    $0.1-&9
    
25. • +BWBGSJFOEMZ
    "1* • J04
    -JWF
    "DUJWJUJFT
    TVQQPSU • 1MVHHBCMF
    DSFEFOUJBM
    TUPSBHF
    BDDFTT
    UPLFOT
    
    N5-4
    DFSUJGJDBUFT • 5SVMZ
    PQFO
    TPVSDF
    DPEF
    RVBMJUZ
    EPDT
    DPNNVOJUZ
    ʜ • 3FVTBCMF
    MPBE
    CBMBODFS
    JNQMFNFOUBUJPO • .BOBHFNFOU
    DPOTPMF 'VUVSF
    8PSL 8F
    DBO
    CVJME
    UIJT
    UPHFUIFS
    
26. HJUIVCDPNMJOFQVTITQIFSF HJUIVCDPNMJOFBSNFSJB