Understanding_Thread_Tuning_for_Inference_Servers_of_Deep_Models.pdf

6OEFSTUBOEJOH5ISFBE5VOJOHGPS*OGFSFODF 4FSWFSTPG%FFQ.PEFMT %BUB(SPVQ ;IBO:JZBOH

• 8PSLJOHGPSJOUFSOBM"*QMBUGPSN • -PWFTQFSGPSNBODFFOHJOFFSJOH • )PTUJOH1ZUIPO.FFUVQ'VLVPLB "CPVU.F *%[IBOQPO

8IBUJTBO*OGFSFODF4FSWFS *OGFSFODF 4FSWFS DBU JNBHFKQH

• -BUFODZ5IFUJNFGSPNXIFOBSFRVFTUSFBDIFTUIFTFSWFS VOUJMUIFSFTQPOTFCFHJOTUPCFTFOU NFBTVSFECZ NJMMJTFDPOET • 5ISPVHIQVU)PXNBOZSFRVFTUTJTCFJOHQSPDFTTFEQFSVOJU UJNF NFBTVSFECZ3FRVFTUT1FS4FDPOE 314
-BUFODZBOE5ISPVHIQVU

8IZ1FSGPSNBODF -BUFODZ*NQBDUT69

8IZ1FSGPSNBODF 5ISPVHIQVU*NQBDUT$PTU

• %FGBVMUDPOGJHVSBUJPOTIBWFQJUGBMMT • 7FSZFBTZUPBQQMZ 8IZ5ISFBE5VOJOH

4JOHMFUISFBEFE4FSWFS .VMUJUISFBEFE4FSWFS $BTF4UVEJFT "HFOEB

1BSU*4JOHMFUISFBEFE4FSWFS

(PBMVOEFSTUBOEUIFSFMBUJPOTIJQPGMBUFODZ UISPVHIQVU BOE DPODVSSFODZ "HFOEB 5IFPSZPGTJOHMFUISFBEFEJOGFSFODFTFSWFS &YQFSJNFOUTXJUITJOHMFUISFBEFEJOGFSFODFTFSWFS 1BSU*4JOHMFUISFBEFE4FSWFS
"HFOEB

#BTJD.PEFM 4JOHMFUISFBEFE4FSWFS 4JOHMF$MJFOU &BDIUBTLUBLFTNT

#BTJD.PEFM 4JOHMFUISFBEFE4FSWFS 4JOHMF$MJFOU "TTVNQUJPOT • &BDIUBTLUBLFTNT • DMJFOU $POTFRVFODFT •
-BUFODZJTNT • 5ISPVHIQVUJTSQT

8JUI.PSF$MJFOUT 4JOHMFUISFBEFE4FSWFS $MJFOUT DPNQVUF NT RVFVF NT

8JUI.PSF$MJFOUT 4JOHMF5ISFBE4FSWFS $MJFOUT DPNQVUF NT RVFVF NT

• 5ISPVHIQVUSQT • -BUFODZDPODVSSFODZ UIFOVNCFSPGUIFDMJFOUT NT 5IF-BXPG0VS4ZTUFN

5IFGPMMPXJOHSFMBUJPOIPMETGPS"/:TZTUFN Latency = Concurrency Throughput -JUUMF`T-BX

&YQFSJNFOU*U

&YQFSJNFOU*U #FODINBSLTFUVQ 5SJUPO 1FSGPSNBODF "OBMZ[FS 5SJUPO*OGFSFODF4FSWFS 3FT/FU SBOEPNJOQVUTPGTIBQF
perf_analyzer -u ${TARGET_HOST} -m resnet50 --concurrency-range 1:8

&YQFSJNFOU*U QFSG@BOBMZ[FS PVUQVU DPODVSSFODZ

#FODINBSL3FTVMUT 5ISPVHIQVUBOE-BUFODZ 5ISPVHIQVUJTDPOTUBOU ✅ -BUFODZJTQSPQPSUJPOBMUP DPODVSSFODZ ✅

#FODINBSL3FTVMUT $PNQVUF5JNFBOE2VFVF5JNF $PNQVUFUJNFJTDPOTUBOU ✅ 2VFVFUJNFJTQSPQPSUJPOBMUP DPODVSSFODZ ✅

5IFPSZDPOGJSNFE🎉

• -BUFODZDPNQVUFUJNF RVFVFUJNF • -JUUMF`TMBX MBUFODZ UISPVHIQVU BOEDPODVSSFODZBSFSFMBUFE 4VNNBSZPG1BSU*

1BSU**.VMUJUISFBEFE4FSWFS

(PBM VOEFSTUBOEUIFMBUFODZUISPVHIQVUUSBEFPGG "HFOEB 5IFPSZPG.VMUJUISFBEFEJOGFSFODFTFSWFS &YQFSJNFOUTXJUI.VMUJUISFBEFEJOGFSFODFTFSWFS 1BSU**.VMUJUISFBEFE4FSWFS "HFOEB

• 8IBUJGUIFSFBSFDBTIJFST "2VFTUJPOPG4DBMJOH

)PSJ[POUBMWT7FSUJDBM4DBMJOH

)PSJ[POUBMWT7FSUJDBM4DBMJOH • &BDIUBTLUBLFTNT • 4FSWFDMJFOUTJOQBSBMMFM • &BDIUBTLUBLFTNT • 4FSWFDMJFOUTPOFCZPOF

8IFO/PU#VTZ )PSJ[POUBM4DBMJOH

8IFO/PU#VTZ 7FSUJDBM4DBMJOH

8IFO#VTZ )PSJ[POUBM4DBMJOH RVFVF NT DPNQVUF NT

8IFO#VTZ 7FSUJDBM4DBMJOH DPNQVUF NT RVFVF NT

• 7FSUJDBMTDBMJOHQSPWJEFTNJOJNVNMBUFODZ JGOPUCVTZ • )PSJ[POUBMTDBMJOHQSPWJEFTNBYJNVNUISPVHIQVU JGCVTZ -BUFODZ5ISPVHIQVU5SBEFPGG

&YQFSJNFOU*U

&YQFSJNFOU*U )PSJ[POUBM4DBMJOH 5SJUPO 1FSGPSNBODF "OBMZ[FS 5SJUPO*OGFSFODF4FSWFS • NPEFMJOTUBODFT • JOUSBPQQBSBMMFMJTN

&YQFSJNFOU*U 7FSUJDBM4DBMJOH 5SJUPO 1FSGPSNBODF "OBMZ[FS 5SJUPO*OGFSFODF4FSWFS • NPEFMJOTUBODFT • JOUSBPQQBSBMMFMJTN

# config.pbtxt platform: "onnxruntime_onnx" # Number of model instances (for
horizontal scaling) instance_group [{ count: 1 kind: KIND_CPU } ] # Intra-op parallelism (for vertical scaling) parameters { key: "intra_op_thread_count" value: { string_value: "1" } } &YQFSJNFOU*U $POGJHVSBUJPOTGPS5SJUPO*OGFSFODF4FSWFS

#FODINBSL3FTVMUT -BUFODZ5ISPVHIQVU5SBEFPGG

#FODINBSL3FTVMUT .JEEMFPGUIF3PBE

5IFPSZDPOGJSNFE🎉

-JNJUPG7FSUJDBM4DBMJOH

• 5IFSFBSFEJGGFSFOUXBZTPGTDBMJOHJOGFSFODFTFSWFSIPSJ[POUBM BOEWFSUJDBM • )PSJ[POUBMTDBMJOHQSPWJEFTNBYJNVNUISPVHIQVU • 7FSUJDBMTDBMJOHQSPWJEFTNJOJNVNMBUFODZ 4VNNBSZPG1BSU**

1BSU***$BTF4UVEJFT

"WPJE3BOEPN5VOJOH 5VOF .FBTVSF

8PSLXJUIB1SJODJQMFE"QQSPBDI )ZQPUIFTJ[F 5VOF .FBTVSF

• $BTF5ISPVHIQVUNBUUFST • $BTF8BUDIPVUGPSDPOUFYUTXJUDIFT • $BTF"WPJE$16UISPUUMJOHJODPOUBJOFSFOWJSPONFOU $BTF4UVEJFT

5ISPVHIQVU.BUUFST

• 0OFEBZ *TBXBUFBNEFQMPZTJOGFSFODFTFSWFSTXJUIJOUSBPQ QBSBMMFMJTN 5ISPVHIQVU.BUUFST

3FDBMM-JNJUPG7FSUJDBM4DBMJOH

5ISPVHIQVU.BUUFST )J DBOZPVUSZ 5'@/6.@*/53"@5)3&"%4 Ok, I will. 8PX UIFUISPVHIQVUJODSFBTFTCZ

• JODSFBTFPGUISPVHIQVUˠMFTTTFSWFST • 5IFTFTFSWFSTQSPDFTTLSFRVFTUTQFSTFDPOE 5ISPVHIQVU.BUUFST

8BUDIPVUGPS$POUFYU4XJUDIFT

8BUDIPVUGPS$POUFYU4XJUDIFT 0L *MMUBLFBMPPL6NN TZTUFN$16 VTBHFJT UIBUTBMPU 5IFMBUFODZJODSFBTFTXIFOUXPTFSWFST EFQMPZFEUPBTBNFOPEF 8PX UIFMBUFODZESPQT5IBOLT
)J DBOZPVEFDSFBTF 5'@/6.@*/53"@5)3&"%4

8BUDI0VUGPS$POUFYU4XJUDIFT NT NT

"WPJE$165ISPUUMJOH

"WPJE$165ISPUUMJOH -BUFODZJTNT 5IBU`TJODSFEJCMZTMPX )J PVSJOGFSFODFTFSWFSTBSFWFSZTMPX8F VTFHSBEJFOUCPPTUJOHNPEFM 8PX UIFMBUFODZJTNT OPX 0,
*GJHVSFEJUPVU5SZ 0.1@/6.@5)3&"%4

• 0VSJOGFSFODFTFSWFSQMBUGPSNSVOTPO,VCFSOFUFT • 5IFDPOUBJOFSSVOUJNFTUPQTUIFQSPDFTTFTXIJDIFYDFFETUIF $16MJNJU JGDPOGJHVSFE 8IBUJTB$165ISPUUMJOH

*OGFSFODFTFSWFSl:BZ*`NSVOOJOHPOBDPSFNBDIJOF*`MM TQJOVQUISFBETBOEEPUPOTPGJOGFSFODFTz ,VCFSOFUFTl/P ZPVBSFOPUBMMPXFEUPHFUUIBUNVDI$16 UJNFz 8IZ$165ISPUUMJOH

• %0/05SFMZPOEFGBVMUUISFBEDPOGJH • 4UBSUXJUIJOUSBPQQBSBMMFMJTN • 8BUDIPVUGPSTPNFLFZNFUSJDT 5BLFBXBZT

"HPPEVOEFSTUBOEJOHPGJOGFSFODFTFSWFSTDBOTPMWFSFBM QSPCMFNT *U`TGVO UPP $PODMVTJPO

Understanding_Thread_Tuning_for_Inference_Serve...

Understanding_Thread_Tuning_for_Inference_Servers_of_Deep_Models.pdf

More Decks by LINEヤフーTech (LY Corporation Tech)

Other Decks in Technology

Featured

Transcript