Realtime Data Platform: from Lakehouse to Streamhouse

3FBMUJNF%BUB1MBUGPSN GSPN-BLFIPVTFUP4USFBNIPVTF -:$PSQPSBUJPO 9JBP;IJZBO

🎯 0QFOSFTFBSDI PO3FBMUJNF%BUB1MBUGPSN &YQMPSFUIFMBUFTUUFDIOPMPHJFTBOEBSDIJUFDUVSFTMJLF4USFBNIPVTF 🔒 /PU SFQSFTFOUBUJWFPGJOIPVTFQSPKFDUT BU-:$PSQPSBUJPO 4DPQF PGUIF4FTTJPO
8IBUXFXJMMDPWFSUPEBZ

঻ ࢤ඙ 9JBP;IJZBO γϠ΢ ζ΢ΠΤϯ 🏠 தࠃॏܚ $IPOHRJOH $IJOB ","l%.BHJD$JUZz
🎓 ߳ߓதจେֶ 5IF$IJOFTF6OJWFSTJUZPG)POH,POH .BUI*OGPSNBUJPO&OHJOFFSJOH 👨💻 4PGUXBSF&OHJOFFS!-:$PSQPSBUJPO %BUB&OHJOFFSJOH%BUB1JQFMJOF&OHJOFFSJOH*OHFTUJPO1SPDFTTJOH ✨ *NQSPWJOHCJHEBUB TZTUFNTXJUI3VTU 0QFO%BUB%SJWFODPNNVOJUZ 4USFBNIPVTF "*OBUJWF 3VTU BOENPSF "CPVU.F !9JBP;IJZBO

💻 4PGUXBSFJTFBUJOHUIFXPSME &WFSZCVTJOFTTCFDBNFBTPGUXBSFDPNQBOZ 🤖 "*JTFBUJOHTPGUXBSF 🚀 3FBMUJNFJTFBUJOHFWFSZUIJOHFMTF -JWFEBTICPBSET 3FBMUJNFBMFSUT 0OUIFGMZQFSTPOBMJ[BUJPO
*OTUBOUGFFECBDLGSPN"*BHFOUT 🐢 .PTU%BUB1MBUGPSNTà /PUGVMMZTVQQPSUJOHSFBMUJNFTDFOBSJPT 🐇 3FBMUJNF%BUB1MBUGPSNà 4USFBNIPVTF 8IZ3FBMUJNF %BUB1MBUGPSN "MBSHFTIJGUIBQQFOJOHBSPVOEVT 🌏 ˠ💻 ˠ🤖 ˠ🚀

%BUB8BSFIPVTFBOE%BUB-BLF 5IFFBSMZEBZTPG%BUB1MBUGPSNT 4USVDUVSFE%BUB 4FNJTUSVDUVSFE%BUB 6OTUSVDUVSFE%BUB &5- &5- %BUB8BSFIPVTF #*3FQPSUT %BUB-BLF
)%'44 2VFSZ&OHJOF 4QBSL .- %BUB8BSFIPVTF %BUB-BLF %BUB5ZQF 4USVDUVSFE ✅ 4USVDUVSFE TFNJTUSVDUVSFE VOTUSVDUVSFE 4DBMBCJMJUZ %JGGJDVMUBOEFYQFOTJWF ✅ &BTZBUBMPXDPTU 5SBOTBDUJPO ✅ 4VQQPSUFE /PUTVQQPSUFE 2VFSZ1FSGPSNBODF ✅ )JHI -PX 🤔 $BOXFDPNCJOF UIFBEWBOUBHFT PGCPUITZTUFNTJOUPPOF

-BLFIPVTF %BUB-BLF %BUB8BSFIPVTF 4USVDUVSFE%BUB 4FNJTUSVDUVSFE%BUB 6OTUSVDUVSFE%BUB &5- -BLFIPVTF5BCMF'PSNBU *DFCFSH #*3FQPSUT
%BUB-BLF )%'44 2VFSZ&OHJOF 4QBSL .- -BLFIPVTF ,FZ1PJOU *OEFYEBUBGJMFTXJUITUBUTUPTQFFEVQRVFSJFT 5BCMF'PSNBUT "QBDIF*DFCFSH %FMUB-BLF "QBDIF)VEJ 1SPT ✅ 5SBOTBDUJPO IJHIRVFSZQFSGPSNBODF $POT 🚫 )JHIMBUFODZ NJOVUFT MPXUISPVHIQVUPGTUSFBNJOHXSJUF 🤔 )PXUPTVQQPSUMPXMBUFODZTUSFBNJOHQSPDFTTJOH

-BNCEB "SDIJUFDUVSF #BUDI1JQFMJOF 4USFBNJOH1JQFMJOF %BUB4PVSDFT #BUDI1JQFMJOF 4QBSL -BLFIPVTF4UPSBHF *DFCFSH 2VFSZ&OHJOF
%BUB6TFST 4USFBNJOH1JQFMJOF 'MJOL 4USFBNJOH4UPSBHF ,BGLB -BNCEB"SDIJUFDUVSF ,FZ1PJOU %VBMMBZFSEFTJHO CBUDI TUSFBNJOH 6TBHF .FSHFSFTVMUTPGCPUIMBZFSTBURVFSZTJEF 1SPT ✅ -PXMBUFODZTUSFBNJOHQSPDFTTJOH TVCTFDPOE $POT 🚫 $PNQMFYJUZ IJHIDPTU EFMBZJOIJTUPSJDBMEBUB NJOVUFT 🤔 *TJUGFBTJCMFUPVOJGZ UIFTFUXPEBUBQJQFMJOFT

,BQQB "SDIJUFDUVSF 6OJGZ#BUDIBOE4USFBNJOH1JQFMJOFT ˠ ,BGLB4PVSDFT 4USFBNJOH1JQFMJOF 'MJOL 4USFBNJOH4UPSBHF ,BGLB 2VFSZ&OHJOF
%BUB6TFST ,BQQB"SDIJUFDUVSF ,FZ1PJOU "MMEBUBBTTUSFBNT IJTUPSJDBMEBUBCZSFQMBZJOH 1SPT ✅ 4JNQMJDJUZ MPXMBUFODZ TVCTFDPOE $POT 🚫 &YQFOTJWFSFQMBZJOH JOFGGJDJFOURVFSJFT 🤔 $BOXFJNQSPWF,BQQB "SDIJUFDUVSFXJUI-BLFIPVTF 4UPSBHF

6OJGJFE/FBS3FBMUJNF1JQFMJOF ,BGLBˠ1BJNPO %BUB4PVSDFT 6OJGJFE1JQFMJOF 'MJOL 2VFSZ&OHJOF %BUB6TFST 🤔 )PXEPFT1BJNPO TVQQPSUCPUI4USFBNJOH
BOE-BLFIPVTF GFBUVSFT -BLFIPVTF4UPSBHF *DFCFSH NJOVUFT 4USFBNJOH4UPSBHF ,BGLB TVCTFDPOE 4USFBNJOHGJSTU-BLFIPVTF4UPSBHF 1BJNPO TVCNJOVUF

"-JUUMF.PSFPO1BJNPO 4USFBNJOHGJSTU-BLFIPVTF5BCMF'PSNBU 🤔 $BOXFGVSUIFSJNQSPWF6OJGJFE1JQFMJOFUPBDIJFWFTVCTFDPOEMBUFODZ IUUQTQBJNPOBQBDIFPSHEPDTNBTUFSDPODFQUTCBTJDDPODFQUT *OEFYEBUBGJMFTXJUITUBUT TJNJMBSUP*DFCFSH 8SJUFUPNVMUJQMFCVDLFUT JOQBSBMMFM 3FBEDIBOHFMPHPG1,UBCMFT
4USFBNJOHSFBE"QQFOEUBCMFT

6OJGJFE3FBMUJNF1JQFMJOF 1BJNPO 'MVTT %BUB4PVSDFT 6OJGJFE1JQFMJOF -BLFIPVTF4UPSBHF 1BJNPO TVCNJOVUF 2VFSZ&OHJOF %BUB6TFST
4USFBNJOH4UPSBHF 'MVTT TVCTFDPOE ,BGLB 'MVTT %BUB'PSNBU #JOBSZ SPXCBTFE ✅ "QBDIF"SSPX DPMVNOCBTFE 4DIFNB &YUFSOBMSFHJTUSZ ✅ *OUFSOBMTDIFNB 1SPKFDUJPO1VTIEPXO /PUTVQQPSUFE ✅ 4VQQPSUFE 2VFSZ*OUFSGBDF $POTVNFS3&45 ✅ "SSPX'MJHIU IJHIQFSGPSNBODF 📝 -FU`TTVNNBSJ[F XIBUXFIBWFEJTDVTTFETPGBS

4USFBNIPVTF 6OJGJFE3FBMUJNF1JQFMJOF %BUB8BSFIPVTF %BUB-BLF -BNCEB -BLFIPVTF ,BQQB 1BJNPO 'MVTT

✨ 8FIBWFTFFOMBUFTU4USFBNIPVTF "SDIJUFDUVSFXJUI1BJNPOBOE'MVTT 🤔 "SFUIFSFBOZPUIFSQPUFOUJBMJNQSPWFNFOUTBT3FBMUJNF%BUB1MBUGPSN )PXUPPSDIFTUSBUJPOPGEBUBQJQFMJOFT )PXUPJNQSPWFJU 1PUFOUJBM*NQSPWFNFOUT

%ZOBNJD %BUB'MPX 4UBUJD%BUB1JQFMJOFTˠ-JWFMZ$PNQPTBCMF%BUB'MPXT %BUB4PVSDFT *OHFTU 0%4 $MFBO %8% "HHSFHBUF %84
$VTUPNJ[F "%4 %BUB'MPX :".- 3FVTBCMF%BUB"DUJPOT 42- 'MPX"DUJPO3FHJTUSZ 'MPX.BUFSJBMJ[FS 'MPX5SJHHFS

"*OBUJWF $POUSPM1MBOF "VUPNBUFEˠ"VUPOPNPVT %BUB"DUJPO %BUB'MPX 'MPX"DUJPO3FHJTUSZ 'MPX5SJHHFS 'MPX1MBOOFS "DUJPO$PNQPTFS "HFOUJD"*
6TFS1SPNQU 0SDIFTUSBUPS 'MPX.BUFSJBMJ[FS

"*OBUJWF4USFBNIPVTF /FYUHFOFSBUJPO3FBMUJNF%BUB1MBUGPSN "*OBUJWF$POUSPM1MBOF %ZOBNJD%BUB'MPX 6OJGJFE3FBMUJNF1JQFMJOF

✨ 4USFBNIPVTF "SDIJUFDUVSFJTTUJMMRVJUFOFX 🧭 -FU`TTFFIPXJUDBOCSJOHSFBMWBMVF UPPVSCVTJOFTT 🚀 &YQMPSF CVJME BOETIBSF
4USFBNIPVTFJNQMFNFOUBUJPOTUPHFUIFS 5IF3PBE"IFBE

-BNCEB"SDIJUFDUVSF IUUQOBUIBONBS[DPNCMPHIPXUPCFBUUIFDBQUIFPSFNIUNM ,BQQB"SDIJUFDUVSF IUUQTXXXPSFJMMZDPNSBEBSRVFTUJPOJOHUIFMBNCEBBSDIJUFDUVSF -BLFIPVTF IUUQTXXXEBUBCSJDLTDPNCMPHXIBUJTBEBUBMBLFIPVTFIUNM
4USFBNIPVTF IUUQTXXXWFSWFSJDBDPNCMPHTUSFBNIPVTFVOWFJMFE "*/BUJWF IUUQTXXXTQMVOLDPNFO@VTCMPHMFBSOBJOBUJWFIUNM 3FMBUFE-JOLT

Realtime Data Platform: from Lakehouse to Strea...

Realtime Data Platform: from Lakehouse to Streamhouse

LINEヤフーTech (LY Corporation Tech) PRO

More Decks by LINEヤフーTech (LY Corporation Tech)

Other Decks in Technology

Featured

Transcript

3FBMUJNF%BUB1MBUGPSN GSPN-BLFIPVTFUP4USFBNIPVTF -:$PSQPSBUJPO 9JBP;IJZBO

🎯 0QFOSFTFBSDI PO3FBMUJNF%BUB1MBUGPSN &YQMPSFUIFMBUFTUUFDIOPMPHJFTBOEBSDIJUFDUVSFTMJLF4USFBNIPVTF 🔒 /PU SFQSFTFOUBUJWFPGJOIPVTFQSPKFDUT BU-:$PSQPSBUJPO 4DPQF PGUIF4FTTJPO

঻ ࢤ඙ 9JBP;IJZBO γϠ΢ ζ΢ΠΤϯ 🏠 தࠃॏܚ $IPOHRJOH $IJOB ","l%.BHJD$JUZz

💻 4PGUXBSFJTFBUJOHUIFXPSME &WFSZCVTJOFTTCFDBNFBTPGUXBSFDPNQBOZ 🤖 "*JTFBUJOHTPGUXBSF 🚀 3FBMUJNFJTFBUJOHFWFSZUIJOHFMTF -JWFEBTICPBSET 3FBMUJNFBMFSUT 0OUIFGMZQFSTPOBMJ[BUJPO

%BUB8BSFIPVTFBOE%BUB-BLF 5IFFBSMZEBZTPG%BUB1MBUGPSNT 4USVDUVSFE%BUB 4FNJTUSVDUVSFE%BUB 6OTUSVDUVSFE%BUB &5- &5- %BUB8BSFIPVTF #*3FQPSUT %BUB-BLF

-BLFIPVTF %BUB-BLF %BUB8BSFIPVTF 4USVDUVSFE%BUB 4FNJTUSVDUVSFE%BUB 6OTUSVDUVSFE%BUB &5- -BLFIPVTF5BCMF'PSNBU DFCFSH #3FQPSUT

-BNCEB "SDIJUFDUVSF #BUDI1JQFMJOF 4USFBNJOH1JQFMJOF %BUB4PVSDFT #BUDI1JQFMJOF 4QBSL -BLFIPVTF4UPSBHF *DFCFSH 2VFSZ&OHJOF

,BQQB "SDIJUFDUVSF 6OJGZ#BUDIBOE4USFBNJOH1JQFMJOFT ˠ ,BGLB4PVSDFT 4USFBNJOH1JQFMJOF 'MJOL 4USFBNJOH4UPSBHF ,BGLB 2VFSZ&OHJOF

6OJGJFE/FBS3FBMUJNF1JQFMJOF ,BGLBˠ1BJNPO %BUB4PVSDFT 6OJGJFE1JQFMJOF 'MJOL 2VFSZ&OHJOF %BUB6TFST 🤔 )PXEPFT1BJNPO TVQQPSUCPUI4USFBNJOH

"-JUUMF.PSFPO1BJNPO 4USFBNJOHGJSTU-BLFIPVTF5BCMF'PSNBU 🤔 $BOXFGVSUIFSJNQSPWF6OJGJFE1JQFMJOFUPBDIJFWFTVCTFDPOEMBUFODZ IUUQTQBJNPOBQBDIFPSHEPDTNBTUFSDPODFQUTCBTJDDPODFQUT OEFYEBUBGJMFTXJUITUBUT TJNJMBSUPDFCFSH 8SJUFUPNVMUJQMFCVDLFUT JOQBSBMMFM 3FBEDIBOHFMPHPG1,UBCMFT

6OJGJFE3FBMUJNF1JQFMJOF 1BJNPO 'MVTT %BUB4PVSDFT 6OJGJFE1JQFMJOF -BLFIPVTF4UPSBHF 1BJNPO TVCNJOVUF 2VFSZ&OHJOF %BUB6TFST

4USFBNIPVTF 6OJGJFE3FBMUJNF1JQFMJOF %BUB8BSFIPVTF %BUB-BLF -BNCEB -BLFIPVTF ,BQQB 1BJNPO 'MVTT

✨ 8FIBWFTFFOMBUFTU4USFBNIPVTF "SDIJUFDUVSFXJUI1BJNPOBOE'MVTT 🤔 "SFUIFSFBOZPUIFSQPUFOUJBMJNQSPWFNFOUTBT3FBMUJNF%BUB1MBUGPSN )PXUPPSDIFTUSBUJPOPGEBUBQJQFMJOFT )PXUPJNQSPWFJU 1PUFOUJBM*NQSPWFNFOUT

%ZOBNJD %BUB'MPX 4UBUJD%BUB1JQFMJOFTˠ-JWFMZ$PNQPTBCMF%BUB'MPXT %BUB4PVSDFT *OHFTU 0%4 $MFBO %8% "HHSFHBUF %84

"OBUJWF $POUSPM1MBOF "VUPNBUFEˠ"VUPOPNPVT %BUB"DUJPO %BUB'MPX 'MPX"DUJPO3FHJTUSZ 'MPX5SJHHFS 'MPX1MBOOFS "DUJPO$PNQPTFS "HFOUJD"

"OBUJWF4USFBNIPVTF /FYUHFOFSBUJPO3FBMUJNF%BUB1MBUGPSN "OBUJWF$POUSPM1MBOF %ZOBNJD%BUB'MPX 6OJGJFE3FBMUJNF1JQFMJOF

✨ 4USFBNIPVTF "SDIJUFDUVSFJTTUJMMRVJUFOFX 🧭 -FU`TTFFIPXJUDBOCSJOHSFBMWBMVF UPPVSCVTJOFTT 🚀 &YQMPSF CVJME BOETIBSF

-BNCEB"SDIJUFDUVSF IUUQOBUIBONBS[DPNCMPHIPXUPCFBUUIFDBQUIFPSFNIUNM ,BQQB"SDIJUFDUVSF IUUQTXXXPSFJMMZDPNSBEBSRVFTUJPOJOHUIFMBNCEBBSDIJUFDUVSF -BLFIPVTF IUUQTXXXEBUBCSJDLTDPNCMPHXIBUJTBEBUBMBLFIPVTFIUNM

'*/