Connecting Thousands of Services at LY Corporation with Central Dogma Control Plane

$POOFDUJOH5IPVTBOETPG4FSWJDFTBU-: $PSQPSBUJPOXJUI$FOUSBM%PHNB$POUSPM1MBOF %FWFMPQFSF9QFSJFODF -*/&1MVT$PSQPSBUJPO .JOXPP4POH )BOOBN3IFF

"HFOEB • #BDLHSPVOE$POUFYU • 0VS$IBMMFOHFT • $FOUSBM%PHNB$POUSPM1MBOF • 4JEFDBSMFTT4FSWJDF.FTI •
3FDBQ'VUVSF8PSL

• .BOBHJOHBOEEJTUSJCVUJOHUSBGGJDQPMJDJFTGPSTFSWJDFUPTFSWJDF DPNNVOJDBUJPOUPUIFEBUBQMBOF • "EWBOUBHFT • $FOUSBMJ[FETFSWJDFDPOGJHVSBUJPONBOBHFNFOU • %ZOBNJDTFSWJDFSFDPOGJHVSBUJPO •
8FJNQMFNFOUFEUIJTPO$FOUSBM%PHNB $POUSPM1MBOF

8IBUJT$FOUSBM%PHNB $FOUSBM%PHNB 6* "QQMJDBUJPOT 7JFX$POGJHT 8BUDI$POGJHT 3FQPTJUPSZ4FSWJDFGPS5FYUVBM$POGJHVSBUJPO (JUCBTFE4UPSBHF .PEJGZ$POpHT

.JSSPSJOHGSPNBO&YUFSOBM(JU3FQPTJUPSZ .PEJGZ$POGJHT 7JFX$POpHT 8BUDI$POGJHT $IBOHF:PVS$POGJHVSBUJPO4BGFMZUISPVHI1VMM3FRVFTU (JU)VC $PVMECFBOZFYUFSOBMHJUQSPWJEFS 1SPQPTF $IBOHFT $FOUSBM%PHNB
6* "QQMJDBUJPOT (JUCBTFE4UPSBHF

3FDBQ'VUVSF8PSL

:PVS"QQ $POUSPM1MBOF • 4FSWJDFSFHJTUSZ • 1PMJDJFT • 5SBGpDDPOUSPM • $BOBSZSFMFBTFT
• 5SBGpDDPOUSPM • $MJFOUTJEFMPBECBMBODJOH $VTUPN.FTTBHF 4DIFNB 0VS -FHBDZ$POUSPM1MBOF%BZT 0UIFS"QQ 1SPKFDU.BOBHFNFOU 5PPM 1.$ $VTUPN $MJFOU :PVS1PE :PVS1PE :PVS1PE :PVS1PE :PVS1PE 1IZTJDBM BOE7JSUVBM .BDIJOFT

• 5JHIU$PVQMJOHXJUI1.$ LTTVQQPSU • /PEZOBNJDSFHJTUSBUJPO • -BDLPG*OUFSPQFSBCJMJUZ • 8IJDIDMJFOUTVQQPSUTUIFDVTUPNNFTTBHFTDIFNB •
-JNJUFE5SBGGJD$POUSPM • "SFHJTUSZ OPUBUSVFDPOUSPMQMBOF -JNJUBUJPOTPGUIF-FHBDZ4ZTUFN

• 4VQQPSUGPS%JWFSTF4PVSDFT r 1.$ 7.T 1.T BOELT • *OUFSPQFSBCJMJUZXJUI&OWPZ r
*OEVTUSZ4UBOEBSEY%41SPUPDPM • (JU0QT#BTFE8PSLqPX • )JHI3FMJBCJMJUZ • 3PCVTU"VUI 4ZTUFN 4IPVME8F.PWFUP"OPUIFS4PMVUJPO %FTJHO$POTJEFSBUJPOTGPSB/FX4ZTUFN

ʜ 8F DPVMEO`UGJOEPOFUIBU GJUTBMMPGUIFN

• 4VQQPSUGPS%JWFSTF4PVSDFT r 1.$ 7.T 1.T BOELT • *OUFSPQFSBCJMJUZXJUI&OWPZ r
*OEVTUSZ4UBOEBSEY%41SPUPDPM • (JU0QT#BTFE8PSLqPX .JSSPSJOH • )JHI3FMJBCJMJUZr .VMUJEBUBDFOUFSSFQMJDBUJPO • 3PCVTU"VUI 4ZTUFN 8BJU$FOUSBM%PHNB4VQQPSUT %FTJHO$POTJEFSBUJPOTGPSB/FX4ZTUFN

3FDBQ'VUVSF8PSL

:PVS"QQ $POUSPM1MBOF • 4FSWJDFSFHJTUSZ • 1PMJDJFT • "DDFTTDPOUSPM • -PBECBMBODJOH
• ;POFBXBSFSPVUJOH • "VUPNBUJDSFUSJFT • $JSDVJUCSFBLFST • ʜ Your Pod Your Pod Your Pod Your Pod Your Pod 1IZTJDBM BOE7JSUVBM .BDIJOFT • "DDFTTDPOUSPM • $PODVSSFODZDPOUSPM • 5SBGpDDPOUSPM • .FUSJDT • ʜ Y%41SPUPDPM 1SPKFDU.BOBHFNFOU 5PPM 1.$ 0UIFS "QQ 0VUCPVOE 4JEFDBS *OCPVOE 4JEFDBS 0UIFS "QQ • "DDFTTDPOUSPM • $MJFOUTJEFMPBECBMBODJOH • $JSDVJUCSFBLFS • "VUPNBUJDSFUSJFT • ʜ ,T&OEQPJOU1MVHJO %ZOBNJD3FHJTUSBUJPO

'SPN-FHBDZ4DIFNBUPY%43FTPVSDFT // Legacy schema { "zone": "DC-1", "groups": [ {
"name": "iPad" } ], "pmcProjectId": "foo", "pmcFarmId": "foo-BETA", "hostname": "foo-beta1-dev", "ip": "10.10.10.11", "port": 8080, "weight": 1000, "status": "NORMAL", "slow_start": 0.05 }, ... // xDS resources "locality": { "zone": "DC-1" }, "lbEndpoints": [ { "endpoint": { "address": { "socketAddress": { "address": "10.10.10.11", "portValue": 8080 } }, "hostname": "foo-beta1-dev" }, "metadata": { "filterMetadata": { "jp.co.lycorp.attributes": { "slowStart": 0.05 } } }, "loadBalancingWeight": 1000, "healthStatus": "HEALTHY" }, ... #VJMEZPVSPXOQMVHJOBOETFBNMFTTMZJOUFHSBUFXJUIY%4 JU`TFBTJFSUIBOZPVUIJOL

'SPNLT$POUSPM1MBOFUPY%43FTPVSDFT Control Plane Watch Changes // Registered via XdsKubernetesService //
on Central Dogma { "name": "groups/foo/k8s/endpointAggregators/foo/beta", "clusterName": "groups/foo/k8s/clusters/foo/beta", "localityLbEndpoints": [ { "watcher": { "serviceName": "foo-beta", "portName": "foo-port", "kubeconfig": { "controlPlaneUrl": "https://foo.com/v1/abcdef-ghijk", "credentialId": "foo-beta-cred" } }, "locality": { "zone": "dc-1" } }] }

"VUPNBUJD3FHJTUSBUJPOPO4UBSUVQ // Register via gRPC or HTTP API. rpc RegisterLocalityLbEndpoint(RegisterLocalityLbEndpointRequest)
returns (LocalityLbEndpoint) { option (google.api.http) = { patch: "/api/v1/xds/{endpoint_name=groups/*/endpoints/**}:registerLocalityLbEndpoint" body: "locality_lb_endpoint" }; }

• "TFSWJDFNFTIDPOUSPMQMBOF • *NQMFNFOUTUIFJOEVTUSZTUBOEBSEY%4QSPUPDPM • %FTJHOFEXJUI-:`TIFUFSPHFOFPVTJOGSBTUSVDUVSFJONJOE • "MMFYJTUJOH$FOUSBM%PHNBBEWBOUBHFT • (JU0QT#BTFE8PSLGMPX
)JHI3FMJBCJMJUZ 3PCVTU"VUIPSJ[BUJPO $FOUSBM%PHNB$POUSPM1MBOF

3FDBQ'VUVSF8PSL

3FDBQ"5ZQJDBM,T"QQMJDBUJPO "VUPNBUFEEFQMPZNFOU TDBMJOH BOENBOBHFNFOUPGDPOUBJOFSJ[FEBQQMJDBUJPOT

3FDBQ"EEJOH*TUJPJOUPUIF1JDUVSF *OGSBTUSVDUVSFMBZFSUIBUHJWFTBQQMJDBUJPOTDBQBCJMJUJFTMJLF [FSPUSVTUTFDVSJUZ PCTFSWBCJMJUZ BOE BEWBODFEUSBGGJDNBOBHFNFOU • 4FSWJDFSFHJTUSZ • "DDFTTDPOUSPM
• "EWBODFEMPBECBMBODJOH FH;POFBXBSFSPVUJOH • "VUPNBUJDSFUSJFT • $JSDVJUCSFBLFST •

3FDBQ8JUI$%$1 5IFGVODUJPOBMJUZPG*TUJPDBOCFSFQMBDFEXJUIUIF$FOUSBM%PHNB$POUSPM1MBOF

5FDIOJDBMMZUIFFYUFSOBMBQQMJDBUJPODBOBMTPTFOESFRVFTUTUISPVHIBOFOWPZTJEFDBS /PUNBOBHFECZLTNFBOT • *ODSFBTFEEFQMPZNFOUNBOBHFNFOUDPNQMFYJUZ • "OPUIFSQPJOUPGGBJMVSF 4JODFNJHSBUJOHFWFSZTFSWJDFUPLTBUPODFNBZOPUCFUSJWJBM $BMMJOHGSPN&YUFSOBM&OWJSPONFOUT

"MMPGUIFVTVBMHPPEJFT • ;FSPUSVTUTFDVSJUZ PCTFSWBCJMJUZ BOEBEWBODFEUSBGpDNBOBHFNFOU "EEFECPOVT • -FTTOFUXPSLIPQT MFTTMBUFODZ BOEMFTTQPJOUTPGGBJMVSFT
(PJOH4JEFDBSMFTT

"SNFSJB JTBNJDSPTFSWJDFGSBNFXPSLVTFEFYUFOTJWFMZXJUIJO-: (PJOH4JEFDBSMFTTXJUI"SNFSJB

4P 8IBUJT"SNFSJB "SNFSJB JTZPVSHPUPNJDSPTFSWJDFGSBNFXPSLGPSBOZ TJUVBUJPO:PVDBOCVJMEBOZUZQFPGNJDSPTFSWJDF MFWFSBHJOHZPVSGBWPSJUFUFDIOPMPHJFT JODMVEJOHH31$ 5ISJGU ,PUMJO
3FUSPpU 3FBDUJWF4USFBNT 4QSJOH#PPUBOE %SPQXJ[BSE *UJTPQFOTPVSDFECZUIFDSFBUPSPG/FUUZBOEIJT DPMMFBHVFTBU-:$PSQPSBUJPO // A simple HTTP client var client = WebClient.of("http://my-endpoint.com") .decorator(CircuitBreakerClient.newDecorator(..)) .decorator(RetryingClient.newDecorator(..)) .build(); // send a GET request var response = client.get("/").aggregate().join(); 8IBUJG"SNFSJB PGGFSFEBTQFDJBMDMJFOUXIJDI DPVMEBDUBTBEBUBQMBOF

"SNFSJB BTB%BUB1MBOF var bootstrapStr = """ dynamic_resources: lds_config: ... cluster_name:
controlplane_cluster static_resources: clusters: - name: controlplane_cluster load_assignment: ... socket_address: address: my-control-plane.com """; var xdsBootstrap = XdsBootstrap.of(fromYaml(bootstrap)); %JTDMBJNFS5IFEFTJHOBOEJNQMFNFOUBUJPOJTJOQSPHSFTTBOENBZCF TVCKFDUUPDIBOHF "UZQJDBMFOWPZCPPUTUSBQ DPOpHVSBUJPO 1BSTFUIFCPPUTUSBQJOUPBO9ET#PPUTUSBQ5IF"SNFSJB DMJFOU XJMMVTFUIJTCPPUTUSBQJOGPSNBUJPOUPBDUBTBEBUBQMBOF 8IJDIDPOUSPMQMBOFTIPVMEXFVTFUPGFUDISFTPVSDFT "SFTPVSDF JTBDPMMFDUJPOPGQPMJDJFTPGIPXUPSPVUFSFRVFTUT %FGJOFTIPXUPDPOOFDUUPB DPOUSPMQMBOF

"SNFSJB BTB%BUB1MBOF var xdsBootstrap = XdsBootstrap.of(bootstrap); var client = WebClient.of(ofListener("my-listener",
xdsBootstrap)); var res = client.get("/").aggregate().join(); Disclaimer: The design and implementation is in progress and may be subject to change How to connect to the control plane. Which resource should be used to route requests. A resource represents a collection of routing policies. The client automatically fetches the resource from the control plane and sends requests, effectively acting as a data plane. Create a client based on the resource name and the bootstrap configuration.

%ZOBNJD3FDPOGJHVSBUJPO var xdsBootstrap = XdsBootstrap.of(bootstrap); var client = WebClient.of(ofListener("my-listener", xdsBootstrap));
var res = client.get("/").aggregate().join(); // A simple HTTP client var client = WebClient.of("http://my-endpoint.com") .decorator(CircuitBreakerClient.newDecorator(..)) .decorator(RetryingClient.newDecorator(..)) .build(); // send a GET request var response = client.get("/").aggregate().join(); /PUJDFUIBU • /POFFEUPTQFDJGZUIFFOEQPJOU63- • /POFFEUPTQFDJGZ$JSDVJU#SFBLFS DPOGJHVSBUJPOT • /POFFEUPTQFDJGZ3FUSZJOH DPOGJHVSBUJPOT &WFSZUIJOHJT%ZOBNJD

*NBHJOFHPJOHDPNQMFUFMZ4JEFDBSMFTT We can still use envoy in k8s, but we
can also go completely sidecar-less. (if you would like!) Less hops means: • Less latency • Less points of failures • Less complexity • Easier debugging

)PX-*/&TFOET1VTI3FRVFTUT Tune into the Main Room C session for
details on how this was done. "Pushsphere: LINE's Secret Recipe for Reliable, Prompt and High-Volume Push Notifications"

3FDBQ'VUVSF8PSL

• $FOUSBM%PHNB$POUSPM1MBOF • "TFSWJDFNFTIDPOUSPMQMBOF EFTJHOFEGPSIFUFSPHFOFPVTJOGSBTUSVDUVSF • *OIFSJUTBMMUIFQSPWFOTUSFOHUITPG$FOUSBM%PHNB • (PJOH4JEFDBSMFTT •
$BMMLTTFSWJDFTGSPNFYUFSOBMFOWJSPONFOUT FBTJFSNJHSBUJPO • -FTTDPNQMFYJUZ MFTTIPQT MFTTGBJMVSF • 4VQQPSUFECZ"SNFSJB 3FDBQ

• $FOUSBM%PHNB$POUSPM1MBOF • 6TFS*OUFSGBDF • *NQSPWFE3FTPVSDF"VUIPSJ[BUJPO • 4JEFDBSMFTT%BUB1MBOF • #FUUFS*OUFSPQXJUIUIFY%41SPUPDPM
• %FCVH.FOV • $VTUPN'JMUFS3FHJTUSZ 'VUVSF8PSL 8FDBOCVJMEUIJTUPHFUIFS

HJUIVCDPNMJOFDFOUSBMEPHNB HJUIVCDPNMJOFBSNFSJB

Connecting Thousands of Services at LY Corporat...

Connecting Thousands of Services at LY Corporation with Central Dogma Control Plane

LINEヤフーTech (LY Corporation Tech) PRO

More Decks by LINEヤフーTech (LY Corporation Tech)

Featured

Transcript

$POOFDUJOH5IPVTBOETPG4FSWJDFTBU-: $PSQPSBUJPOXJUI$FOUSBM%PHNB$POUSPM1MBOF %FWFMPQFSF9QFSJFODF -*/&1MVT$PSQPSBUJPO .JOXPP4POH )BOOBN3IFF

"HFOEB • #BDLHSPVOE$POUFYU • 0VS$IBMMFOHFT • $FOUSBM%PHNB$POUSPM1MBOF • 4JEFDBSMFTT4FSWJDF.FTI •

• .BOBHJOHBOEEJTUSJCVUJOHUSBGGJDQPMJDJFTGPSTFSWJDFUPTFSWJDF DPNNVOJDBUJPOUPUIFEBUBQMBOF • "EWBOUBHFT • $FOUSBMJ[FETFSWJDFDPOGJHVSBUJPONBOBHFNFOU • %ZOBNJDTFSWJDFSFDPOGJHVSBUJPO •

8IBUJT$FOUSBM%PHNB $FOUSBM%PHNB 6* "QQMJDBUJPOT 7JFX$POGJHT 8BUDI$POGJHT 3FQPTJUPSZ4FSWJDFGPS5FYUVBM$POGJHVSBUJPO (JUCBTFE4UPSBHF .PEJGZ$POpHT

.JSSPSJOHGSPNBO&YUFSOBM(JU3FQPTJUPSZ .PEJGZ$POGJHT 7JFX$POpHT 8BUDI$POGJHT $IBOHF:PVS$POGJHVSBUJPO4BGFMZUISPVHI1VMM3FRVFTU (JU)VC $PVMECFBOZFYUFSOBMHJUQSPWJEFS 1SPQPTF $IBOHFT $FOUSBM%PHNB

"HFOEB • #BDLHSPVOE$POUFYU • 0VS$IBMMFOHFT • $FOUSBM%PHNB$POUSPM1MBOF • 4JEFDBSMFTT4FSWJDF.FTI •

:PVS"QQ $POUSPM1MBOF • 4FSWJDFSFHJTUSZ • 1PMJDJFT • 5SBGpDDPOUSPM • $BOBSZSFMFBTFT

• 5JHIU$PVQMJOHXJUI1.$ LTTVQQPSU • /PEZOBNJDSFHJTUSBUJPO • -BDLPG*OUFSPQFSBCJMJUZ • 8IJDIDMJFOUTVQQPSUTUIFDVTUPNNFTTBHFTDIFNB •

• 4VQQPSUGPS%JWFSTF4PVSDFT r 1.$ 7.T 1.T BOELT • *OUFSPQFSBCJMJUZXJUI&OWPZ r

ʜ 8F DPVMEO`UGJOEPOFUIBU GJUTBMMPGUIFN

• 4VQQPSUGPS%JWFSTF4PVSDFT r 1.$ 7.T 1.T BOELT • *OUFSPQFSBCJMJUZXJUI&OWPZ r

"HFOEB • #BDLHSPVOE$POUFYU • 0VS$IBMMFOHFT • $FOUSBM%PHNB$POUSPM1MBOF • 4JEFDBSMFTT4FSWJDF.FTI •

:PVS"QQ $POUSPM1MBOF • 4FSWJDFSFHJTUSZ • 1PMJDJFT • "DDFTTDPOUSPM • -PBECBMBODJOH

'SPN-FHBDZ4DIFNBUPY%43FTPVSDFT // Legacy schema { "zone": "DC-1", "groups": [ {

'SPNLT$POUSPM1MBOFUPY%43FTPVSDFT Control Plane Watch Changes // Registered via XdsKubernetesService //

"VUPNBUJD3FHJTUSBUJPOPO4UBSUVQ // Register via gRPC or HTTP API. rpc RegisterLocalityLbEndpoint(RegisterLocalityLbEndpointRequest)

• "TFSWJDFNFTIDPOUSPMQMBOF • *NQMFNFOUTUIFJOEVTUSZTUBOEBSEY%4QSPUPDPM • %FTJHOFEXJUI-:`TIFUFSPHFOFPVTJOGSBTUSVDUVSFJONJOE • "MMFYJTUJOH$FOUSBM%PHNBBEWBOUBHFT • (JU0QT#BTFE8PSLGMPX

"HFOEB • #BDLHSPVOE$POUFYU • 0VS$IBMMFOHFT • $FOUSBM%PHNB$POUSPM1MBOF • 4JEFDBSMFTT4FSWJDF.FTI •

3FDBQ"5ZQJDBM,T"QQMJDBUJPO "VUPNBUFEEFQMPZNFOU TDBMJOH BOENBOBHFNFOUPGDPOUBJOFSJ[FEBQQMJDBUJPOT

3FDBQ"EEJOHTUJPJOUPUIF1JDUVSF OGSBTUSVDUVSFMBZFSUIBUHJWFTBQQMJDBUJPOTDBQBCJMJUJFTMJLF [FSPUSVTUTFDVSJUZ PCTFSWBCJMJUZ BOE BEWBODFEUSBGGJDNBOBHFNFOU • 4FSWJDFSFHJTUSZ • "DDFTTDPOUSPM

3FDBQ8JUI$%$1 5IFGVODUJPOBMJUZPG*TUJPDBOCFSFQMBDFEXJUIUIF$FOUSBM%PHNB$POUSPM1MBOF

5FDIOJDBMMZUIFFYUFSOBMBQQMJDBUJPODBOBMTPTFOESFRVFTUTUISPVHIBOFOWPZTJEFDBS /PUNBOBHFECZLTNFBOT • *ODSFBTFEEFQMPZNFOUNBOBHFNFOUDPNQMFYJUZ • "OPUIFSQPJOUPGGBJMVSF 4JODFNJHSBUJOHFWFSZTFSWJDFUPLTBUPODFNBZOPUCFUSJWJBM $BMMJOHGSPN&YUFSOBM&OWJSPONFOUT

"MMPGUIFVTVBMHPPEJFT • ;FSPUSVTUTFDVSJUZ PCTFSWBCJMJUZ BOEBEWBODFEUSBGpDNBOBHFNFOU "EEFECPOVT • -FTTOFUXPSLIPQT MFTTMBUFODZ BOEMFTTQPJOUTPGGBJMVSFT

"SNFSJB JTBNJDSPTFSWJDFGSBNFXPSLVTFEFYUFOTJWFMZXJUIJO-: (PJOH4JEFDBSMFTTXJUI"SNFSJB

4P 8IBUJT"SNFSJB "SNFSJB JTZPVSHPUPNJDSPTFSWJDFGSBNFXPSLGPSBOZ TJUVBUJPO:PVDBOCVJMEBOZUZQFPGNJDSPTFSWJDF MFWFSBHJOHZPVSGBWPSJUFUFDIOPMPHJFT JODMVEJOHH31$ 5ISJGU ,PUMJO

"SNFSJB BTB%BUB1MBOF var bootstrapStr = """ dynamic_resources: lds_config: ... cluster_name:

"SNFSJB BTB%BUB1MBOF var xdsBootstrap = XdsBootstrap.of(bootstrap); var client = WebClient.of(ofListener("my-listener",

%ZOBNJD3FDPOGJHVSBUJPO var xdsBootstrap = XdsBootstrap.of(bootstrap); var client = WebClient.of(ofListener("my-listener", xdsBootstrap));

*NBHJOFHPJOHDPNQMFUFMZ4JEFDBSMFTT We can still use envoy in k8s, but we

)PX-*/&TFOET1VTI3FRVFTUT Tune into the Main Room C session for

"HFOEB • #BDLHSPVOE$POUFYU • 0VS$IBMMFOHFT • $FOUSBM%PHNB$POUSPM1MBOF • 4JEFDBSMFTT4FSWJDF.FTI •

• $FOUSBM%PHNB$POUSPM1MBOF • "TFSWJDFNFTIDPOUSPMQMBOF EFTJHOFEGPSIFUFSPHFOFPVTJOGSBTUSVDUVSF • *OIFSJUTBMMUIFQSPWFOTUSFOHUITPG$FOUSBM%PHNB • (PJOH4JEFDBSMFTT •

• $FOUSBM%PHNB$POUSPM1MBOF • 6TFSOUFSGBDF • NQSPWFE3FTPVSDF"VUIPSJ[BUJPO • 4JEFDBSMFTT%BUB1MBOF • #FUUFS*OUFSPQXJUIUIFY%41SPUPDPM

HJUIVCDPNMJOFDFOUSBMEPHNB HJUIVCDPNMJOFBSNFSJB