Slide 1

Slide 1 text

"MQIB(P"MQIB(P ;FSP ,FJUB8BUBOBCF

Slide 2

Slide 2 text

.PUJWBUJPO w 5SFFCBTFEEFDJTJPONBLJOHGSBNFXPSLJTDPNNPO BDSPTTSPCPUJDT "7 BOEFUDʜ w .POUF$BSMP5SFF4FBSDI .$54 JTPOFPGUIFNPTU TVDDFTTGVMNFUIPEBNPOHPUIFSUSFFTFBSDI BMHPSJUINT w 3FDFOU.$54CBTFEEFDFTJPONBLJOHGSBNFXPSL GPS"7 $BJ TJHOJpDBOUMZJOqVFODFECZ "MQIB(P

Slide 3

Slide 3 text

0WFSWJFXPGUIJT QSFTFOUBUJPO w *OUSPEVDUJPOUP(P w "MQIB(P w 4-1PMJDZ/FUXPSL w 3-1PMJDZ/FUXPSL w 7BMVF/FUXPSL w .$54 .POUF$BSMP5SFF4FBSDI w "MQIB(P;FSP w *NQSPWFNFOUTGSPN"MQIB(P

Slide 4

Slide 4 text

3VMFPG(P* 3FUSJFWFEGSPN8JLJQFEJB
 IUUQTFOXJLJQFEJBPSHXJLJ(P@ HBNF (PJTBOBEWFSTBSJBMHBNFXJUIUIFPCKFDUJWFPG TVSSPVOEJOHBMBSHFSUPUBMBSFBPGUIFCPBSEXJUIPOFT TUPOFTUIBOUIFPQQPOFOU"TUIFHBNFQSPHSFTTFT UIF QMBZFSTQPTJUJPOTUPOFTPOUIFCPBSEUPNBQPVUGPSNBUJPOT BOEQPUFOUJBMUFSSJUPSJFT$POUFTUTCFUXFFOPQQPTJOH GPSNBUJPOTBSFPGUFOFYUSFNFMZDPNQMFYBOENBZSFTVMUJOUIF FYQBOTJPO SFEVDUJPO PSXIPMFTBMFDBQUVSFBOEMPTTPG GPSNBUJPOTUPOFT 5IFGPVSMJCFSUJFT BEKBDFOUFNQUZQPJOUT PGBTJOHMFCMBDL TUPOF " BT8IJUFSFEVDFTUIPTFMJCFSUJFTCZPOF # $ BOE% 8IFO#MBDLIBTPOMZPOFMJCFSUZMFGU % UIBUTUPOFJTJOBUBSJz 8IJUFNBZDBQUVSFUIBUTUPOF SFNPWFGSPNCPBSE XJUIBQMBZ POJUTMBTUMJCFSUZ BU% "CBTJDQSJODJQMFPG(PJTUIBUBHSPVQPGTUPOFTNVTUIBWFBU MFBTUPOFMJCFSUZUPSFNBJOPOUIFCPBSE"MJCFSUZJTBOPQFO QPJOU JOUFSTFDUJPO CPSEFSJOHUIFHSPVQ"OFODMPTFEMJCFSUZ PSMJCFSUJFT JTDBMMFEBOlFZFz ؟ BOEBHSPVQPGTUPOFTXJUI UXPPSNPSFFZFTJTTBJEUPCFVODPOEJUJPOBMMZBMJWF4VDI HSPVQTDBOOPUCFDBQUVSFE FWFOJGTVSSPVOEFE

Slide 5

Slide 5 text

3VMFPG(P** 1PJOUTXIFSF #MBDLDBODBQUVSF8IJUF 1PJOUTXIFSF 8IJUFDBOOPUQMBDFTUPOF 'JHPG 0UTVLJ

Slide 6

Slide 6 text

3VMFPG(P*7
 7JDUPSZKVEHNFOU *GZPVXBOUUPLOPXNPSF KVTUBTL*WPPS&SJL 'JHPG 0UTVLJ 4DPSFPGTUPOFTPG FZFT ,PNJ#MBDL .PWFTpSTU UBLFTBIBOEJDBQ5ZQJDBMMZ QPJOUT #MBDLUFSSJUPSZ 8IJUF UFSSJUPSZ
 #MBDLXJOT

Slide 7

Slide 7 text

8IZ(PJTTPEJ⒏DVMU "QQSPY4J[FPGUIF4FBSDI4QBDF 0UIFMMP $IFTT 4IPHJ (P 5BCMFPG 0UVLJ 4J[FPGUIFTFBSDITQBDFJTFOPSNPVT

Slide 8

Slide 8 text

"MQIB(P

Slide 9

Slide 9 text

"CTUSBDU 5IFHBNFPG(PIBTMPOHCFFOWJFXFEBTUIFNPTUDIBMMFOHJOHPGDMBTTJD HBNFTGPSBSUJpDJBMJOUFMMJHFODFPXJOHUPJUTFOPSNPVTTFBSDITQBDFBOEUIF EJ⒏DVMUZPGFWBMVBUJOHCPBSEQPTJUJPOTBOENPWFT)FSFXFJOUSPEVDFBOFX BQQSPBDIUPDPNQVUFS(PUIBUVTFTWBMVFOFUXPSLTUPFWBMVBUFCPBSE QPTJUJPOTBOEQPMJDZOFUXPSLTUPTFMFDUNPWFT5IFTFEFFQOFVSBMOFUXPSLTBSF USBJOFECZBOPWFMDPNCJOBUJPOPGTVQFSWJTFEMFBSOJOHGSPNIVNBOFYQFSUHBNFT BOESFJOGPSDFNFOUMFBSOJOHGSPNHBNFTPGTFMGQMBZ8JUIPVUBOZMPPLBIFBE TFBSDI UIFOFVSBMOFUXPSLTQMBZ(PBUUIFMFWFMPGTUBUFPGUIFBSU.POUF $BSMPUSFFTFBSDIQSPHSBNTUIBUTJNVMBUFUIPVTBOETPGSBOEPNHBNFTPG TFMGQMBZ8FBMTPJOUSPEVDFBOFXTFBSDIBMHPSJUINUIBUDPNCJOFT.POUF $BSMPTJNVMBUJPOXJUIWBMVFBOEQPMJDZOFUXPSLT6TJOHUIJTTFBSDIBMHPSJUIN PVS QSPHSBN"MQIB(PBDIJFWFEBXJOOJOHSBUFBHBJOTUPUIFS(PQSPHSBNT BOEEFGFBUFEUIFIVNBO&VSPQFBO(PDIBNQJPOCZHBNFTUP5IJTJTUIF pSTUUJNFUIBUBDPNQVUFSQSPHSBNIBTEFGFBUFEBIVNBOQSPGFTTJPOBMQMBZFSJOUIF GVMMTJ[FEHBNFPG(P BGFBUQSFWJPVTMZUIPVHIUUPCFBUMFBTUBEFDBEFBXBZ

Slide 10

Slide 10 text

0WFSWJFXPG"MQIB(P 1PMJDZ /FUXPSL 7BMVF /FUXPSL 3PMMPVU 1PMJDZ 1SFEJDUJPOPG.PWF 1SFEJDUJPOPG.PWF 1SFEJDUJPOPG8JO3BUF 6TFEGPSQMBZPVU -PHJTUJD3FHSFTTJPO 'BTU 6TFEGPS/PEF4FMFDUJPO&YQBOTJPO $// 'BTU $// 3FDPSEPGTUSPOH QMBZFST 4FMGQMBZ 3- .$54

Slide 11

Slide 11 text

3PMMPVU1PMJDZ w -PHJTUJD3FHSFTTJPOXJUIXFMMLOPXOGFBUVSFT TFFUIFUBCMFCFMPX VTFEJO UIJTpFME w 5SBJOFEXJUINJMMJPOQPTJUJPOTGSPNUIF,(4(P4FSWFS IUUQT XXXHPLHTDPN w 5IJTNPEFMVTFEGPS3PMMPVU EFUBJMTXJMMCFFYQMBJOFEMBUFS *OUPUBM'FBUVSFT&YUFOEFE5BCMFPG 4JMWFS

Slide 12

Slide 12 text

-PHJTUJD3FHSFTTJPO x1 x2 x109747 Σ u = 109747 ∑ k=1 wk xk ˜ p = 1 1 + e−u w -PHJTUJD3FHSFTTJPOXJUIXFMMLOPXOGFBUVSFTVTFE JOUIJTpFME w 5SBJOFEXJUINJMMJPOQPTJUJPOTGSPNUIF,(4(P 4FSWFS IUUQTXXXHPLHTDPN

Slide 13

Slide 13 text

5SFF1PMJDZ w *UJTBMPHJTUJDSFHSFTTJPONPEFMXJUIBEEJUJPOBMGFBUVSFT w *NQSPWFEQFSGPSNBODFXJUIFYUSBDPNQVUBUJPOBMUJNF w 6TFEGPS&YQBOTJPOTUFQPG.POUF$BSMP5SFFTFBSDI w *OUPUBM'FBUVSFT&YUFOEFE5BCMFPG 4JMWFS

Slide 14

Slide 14 text

0WFSWJFXPG"MQIB(P 1PMJDZ /FUXPSL 7BMVF /FUXPSL 3PMMPVU 1PMJDZ 1SFEJDUJPOPG.PWF 1SFEJDUJPOPG.PWF 1SFEJDUJPOPG8JO3BUF 6TFEGPSQMBZPVU -PHJTUJD3FHSFTTJPO 'BTU 6TFEGPS/PEF4FMFDUJPO&YQBOTJPO $// 'BTU $// 3FDPSEPGTUSPOH QMBZFST 4FMGQMBZ 3- .$54

Slide 15

Slide 15 text

1PMJDZ/FUXPSL0WFSWJFX 'JHPG 4JMWFS w $POWPMVUJPOBM/FVSBM/FUXPSL w 5IFOFUXPSLpSTUJTUSBJOFE CZTVQFSWJTFEMFBSOJOH BMHPSJUINBOEMBUFSSFpOFECZ SFJOGPSDFNFOUMFBSOJOH w 5SBJOFEXJUI,(4EBUBTFU NJMMJPOQPTJUJPOTGSPN HBNFTQMBZFECZ ,(4UPEBO

Slide 16

Slide 16 text

4-QPMJDZOFUXPSL 0VUQVUJTQFSDFOUBHF'JH 0UTVLJ

Slide 17

Slide 17 text

4-1PMJDZ/FUXPSL w $POWPMVUJPOBM/FVSBM/FUXPSL w 5SBJOFEXJUI,(4EBUBTFUNJMMJPOQPTJUJPOTGSPNHBNFTQMBZFECZ,(4 UPEBO w $IBOOFMT 'FBUVSFT JTQSFQBSFE /FYUTMJEFFYQMBJOTEFUBJMT https://senseis.xmp.net/?Go 19 x 19 48 Channel 19 19 5 5 5 5 3 3 3 3 19 19 .... .... 3 3 3 3 19 19 19 19 0VUQVU1SPCPGUIFOFYUNPWF

Slide 18

Slide 18 text

*OQVUGFBUVSFT 4JMWFS /PUF.PTUPGUIFIBOENBEFGFBUVSFTIFSFBSFOPUOFX CVUDPNNPOMZVTFE JOUIJTpFME

Slide 19

Slide 19 text

3-1PMJDZ/FUXPSL w 5IFZGVSUIFSUSBJOFEUIFQPMJDZOFUXPSLCZQPMJDZHSBEJFOUSFJOGPSDFNFOU MFBSOJOH w 5SBJOJOHJTEPOFCZTFMGQMBZ w 5IFXJOSBUFPGUIF3-QPMJDZOFUXPSLPWFSUIFPSJHJOBM4-QPMJDZOFUXPSL XBT

Slide 20

Slide 20 text

0WFSWJFXPG"MQIB(P 1PMJDZ /FUXPSL 7BMVF /FUXPSL 3PMMPVU 1PMJDZ 1SFEJDUJPOPG.PWF 1SFEJDUJPOPG.PWF 1SFEJDUJPOPG8JO3BUF 6TFEGPSQMBZPVU -PHJTUJD3FHSFTTJPO 'BTU 6TFEGPS/PEF4FMFDUJPO&YQBOTJPO $// 'BTU $// 3FDPSEPGTUSPOH QMBZFST 4FMGQMBZ 3- .$54

Slide 21

Slide 21 text

7BMVF/FUXPSL w "MQIB(PVTFTUIF3-QPMJDZOFUXPSL UPHFOFSBUFUSBJOJOHEBUBGPSUIF 7BMVF/FUXPSL XIJDIQSFEJDUXJO SBUF w 5SBJOJOHEBUB 1PTJUJPO 8JO-PTF NJMMJPO w *UUPPLXFFLXJUI(16 w 5SBJOJOHBMTPUPPLXFFLXJUI (16 w 5IFOFUXPSLQSPWJEFT&WBMVBUJPO GVODUJPOGPS(P UIBUDPOTJEFSFEUPCF IBSEQSFWJPVTMZ 'JHPG 4JMWFS

Slide 22

Slide 22 text

3PMMPVU1PMJDZ 1PMJDZ/FUXPSL 7BMVF/FUXPSL .PEFM -PHJTUJD 3FHSFTTJPO $// -BZFST $// -BZFST 5JNFGPS FWBMVBUJPOPGB TUBUF ЖT NT NT 5JNFGPSQMBZPVU NPWFT NT T PGQMBZPVUT QFSTFD "CPVU "CPVU "DDVSBDZ

Slide 23

Slide 23 text

0WFSWJFXPG"MQIB(P 1PMJDZ /FUXPSL 7BMVF /FUXPSL 3PMMPVU 1PMJDZ 1SFEJDUJPOPG.PWF 1SFEJDUJPOPG.PWF 1SFEJDUJPOPG8JO3BUF 6TFEGPSQMBZPVU -PHJTUJD3FHSFTTJPO 'BTU 6TFEGPS/PEF4FMFDUJPO&YQBOTJPO $// 'BTU $// 3FDPSEPGTUSPOH QMBZFST 4FMGQMBZ 3- .$54

Slide 24

Slide 24 text

.$54&YBNQMF/JN w :PVDBOUBLFNPSFUIBOPOF TUPOFGSPNFUIFSMFGUPSSJHIU w :PVXJMMXJOXIFOZPVUBLF UIFMBTUTUPOF w 5IJTFYBNQMFGSPNIUUQ CMPHCSBJOQBEDPKQFOUSZ

Slide 25

Slide 25 text

(BNF5SFF (SFFO1MBZFSXIPNPWFTpSTUXJOT :FMMPX1MBZFSXIPNPWFTTFDPOEXJOT 3FUSJFWFEGSPNIUUQCMPHCSBJOQBEDPKQFOUSZ

Slide 26

Slide 26 text

.POUF$BSMP4JNVMBUJPO 3FUSJFWFEGSPNIUUQCMPHCSBJOQBEDPKQFOUSZ :PVDBOpOEPVU2WBMVFPGFBDITUBUFCZTJNVMBUJPO .$54JTBIFVSJTUJDUIBUFOBCMFVTF⒏DJFOUMZJOWFTUJHBUFQSPNJTJOHTUBUFT

Slide 27

Slide 27 text

.POUF$BSMP5SFF4FBSDI .POUF$BSMPUSFFTFBSDI .$54 JTBIFVSJTUJD TFBSDIBMHPSJUINGPSEFDJTJPOQSPDFTTFT 5IFGPDVTPG.POUF$BSMPUSFFTFBSDIJTPO UIFBOBMZTJTPGUIFNPTUQSPNJTJOHNPWFT FYQBOEJOHUIFTFBSDIUSFFCBTFEPOSBOEPN TBNQMJOHPGUIFTFBSDITQBDF5IFBQQMJDBUJPO PG.POUF$BSMPUSFFTFBSDIJOHBNFTJTCBTFE PONBOZQMBZPVUT*OFBDIQMBZPVU UIFHBNF JTQMBZFEPVUUPUIFWFSZFOECZTFMFDUJOH NPWFTBUSBOEPN5IFpOBMHBNFSFTVMUPG FBDIQMBZPVUJTUIFOVTFEUPXFJHIUUIF OPEFTJOUIFHBNFUSFFTPUIBUCFUUFSOPEFT BSFNPSFMJLFMZUPCFDIPTFOJOGVUVSF QMBZPVUT #SPXOF

Slide 28

Slide 28 text

.$54&YBNQMF N: 0, Q: 0 *OJUJBM4UBUF /PGWJTJUTUPUIFTUBUF 2&YQFDUFESFXBSE 4FMFDUJPO 4FMFDUOPEFUIBUNBYJNJ[FT Q(s, a) + Cp 2 log ns ns,a N: 1, Q: 0 N: 1, Q: 0 N: 0, Q: 0 N: 0, Q: 0 N: 0, Q: 0 'JSTUUFSN&TUJNBUFESFXBSE 4FDPOEUFSN#JBTUFSN
 *UCBMBODFT&YQMPSBUJPOWT&YQMPJUBUJPO "VFS 1 
 *OUIJTDBTF JU`TSBOEPN ᶃ ᶄ

Slide 29

Slide 29 text

N: 1, Q: 0 N: 1, Q: 0 N: 0, Q: 0 N: 0, Q: 0 N: 0, Q: 0 Win 3PMMPVU 3BOEPNMZQMBZHBNFBOE
 pOEPVUXJOMPTF N: 1, Q: 0 N: 1, Q: 1 N: 0, Q: 0 N: 0, Q: 0 N: 0, Q: 0 Win #BDLVQ 3FOFX2PGUIFTUBUF ᶅ ᶆ

Slide 30

Slide 30 text

N: 1, Q: 0 N: 1, Q: 1 N: 1, Q: 1 N: 1, Q: -1 N: 1, Q: 1 N: 5, Q: 0.25 N: 2, Q: 1 N: 1, Q: 1 N: 1, Q: -1 N: 1, Q: 1 N: 0, Q: 0 N: 0, Q: 0 N: 0, Q: 0 N: 5, Q: 0.25 N: 2, Q: 1 N: 1, Q: 1 N: 1, Q: -1 N: 1, Q: 1 &YQBOTJPO &YQBOEUSFFXIFOBOPEFJTWJTJUFE
 DFSUBJOQSFEFpOFEUJNFT
 JOUIJTDBTF ᶇ ᶈ ᶉ

Slide 31

Slide 31 text

w #JBTJTFWBMVBUFECZUIFPSJHJOBM#JBT0VUQVUPGUIF 4-QPMJDZOFUXPSL
 w &WBMVBUJPOPGXJOSBUFQMBZPVU0VUQVUPGUIF WBMVF/FUXPSL w .BTTJWFQBSBMMFMDPNQVUBUJPOVTJOHCPUI(16T BOE$16T Q(s, a) = (1 − λ) Wv (s, a) Nv (s, a) + λ Wr (s, a) Nr (s, a) u(s, a) = cpuct P(s, a) ∑ b Nr (s, b) 1 + Nr (s, a) .$54JO"MQIB(P 7BMVF/FUXPSL .$54 P(s, a)

Slide 32

Slide 32 text

1FSGPSNBODF 'JHVSFPG 4JMWFS

Slide 33

Slide 33 text

"MQIB(P;FSP

Slide 34

Slide 34 text

"CTUSBDU "MPOHTUBOEJOHHPBMPGBSUJpDJBMJOUFMMJHFODFJTBOBMHPSJUINUIBUMFBSOT UBCVMBSBTB TVQFSIVNBOQSPpDJFODZJODIBMMFOHJOHEPNBJOT3FDFOUMZ "MQIB(PCFDBNFUIFpSTUQSPHSBNUPEFGFBUBXPSMEDIBNQJPOJOUIFHBNFPG (P5IFUSFFTFBSDIJO"MQIB(PFWBMVBUFEQPTJUJPOTBOETFMFDUFENPWFTVTJOH EFFQOFVSBMOFUXPSLT5IFTFOFVSBMOFUXPSLTXFSFUSBJOFECZTVQFSWJTFE MFBSOJOHGSPNIVNBOFYQFSUNPWFT BOECZSFJOGPSDFNFOUMFBSOJOHGSPNTFMG QMBZ)FSFXFJOUSPEVDFBOBMHPSJUINCBTFETPMFMZPOSFJOGPSDFNFOU MFBSOJOH XJUIPVUIVNBOEBUB HVJEBODFPSEPNBJOLOPXMFEHFCFZPOE HBNFSVMFT"MQIB(PCFDPNFTJUTPXOUFBDIFSBOFVSBMOFUXPSLJTUSBJOFEUP QSFEJDU"MQIB(P`TPXONPWFTFMFDUJPOTBOEBMTPUIFXJOOFSPG"MQIB(P`T HBNFT5IJTOFVSBMOFUXPSLJNQSPWFTUIFTUSFOHUIPGUIFUSFFTFBSDI SFTVMUJOH JOIJHIFSRVBMJUZNPWFTFMFDUJPOBOETUSPOHFSTFMGQMBZJOUIFOFYUJUFSBUJPO 4UBSUJOHUBCVMBSBTB PVSOFXQSPHSBN"MQIB(P;FSPBDIJFWFETVQFSIVNBO QFSGPSNBODF XJOOJOHrBHBJOTUUIFQSFWJPVTMZQVCMJTIFE DIBNQJPO EFGFBUJOH"MQIB(P 5BCVMBSBTBJTB-BUJOQISBTFPGUFOUSBOTMBUFEBTDMFBOTMBUF

Slide 35

Slide 35 text

1PJOU%VBM/FUXPSL https://senseis.xmp.net/?Go 19 x 19 48 Channel 19 19 5 5 5 5 3 3 3 3 19 19 .... .... 3 3 3 3 19 19 19 19 Output1: Prediction of the next move 19 19 Output Layer Output2: Win Rate w -BZFST$POWPMVUJPOBM/FVSBM/FUXPSL w &BDIMBZFSYDPOWPMVUJPOMBZFS#BUDIOPSNBMJ[BUJPO3FMV w -BZFSdBSF3FT/FU w 5SBJOFECZTFMGQMBZ EFUBJMTBSFEFTDSJCFEMBUFS w $IBOOFMT 'FBUVSFT JTQSFQBSFE UIFOFYUTMJEFTIPXTEFUBJMT w -FBSOJOHNFUIPEPGUIJTOFUXPSLEJTDVTTFEMBUFS 'PSOPX MFU`TBTTVNFXFIBWFUSBJOFEJUOJDFMZ p v

Slide 36

Slide 36 text

"MQIB(P;FSPJTMFTTEFQFOET POIBOEDSBGUFEGFBUVSFT 'FBUVSFTPG"MQIB(P 4JMWFS 'FBUVSF PGQMBOFT 1PTJUJPOPGCMBDL TUPOFT 1PTJUJPOPGCMBDL TUPOFT 1PTJUJPOPGCMBDL TUPOFTL d TUFQT CFGPSF 1PTJUJPOPGXIJUF TUPOFTL d TUFQT CFGPSF 5VSO 'FBUVSFTPG"MQIB(P;FSP 4JMWFS

Slide 37

Slide 37 text

1PJOU*NQSPWFNFOUPG .$54 w .$54BMHPSJUINVTFTUIFGPMMPXJOHWBMVFGPSTUBUF TFMFDUJPO w /PQMBZPVU JUKVTUSFMJFTPOWBMVF Q(s, a) + u(s, a) Q(s, a) = W(s, a) N(s, a) u(s, a) = cpuct p(s, a) ∑ b N(s, b) 1 + N(s, a) 8JO3BUF #JBT 1SFEJDUJPO PGUIFNPWFB

Slide 38

Slide 38 text

.$544FMFDUJPO 4FMFDUUIFOPEFXIJDIIBTNBY Q(s, a) + u(s, a)

Slide 39

Slide 39 text

.$544FMFDUJPO &YQBOEUIFOPEF

Slide 40

Slide 40 text

.$544FMFDUJPO &WBMVBUFQBOEWVTJOHUIFEVBMOFUXPSL QXJMMCFVTFEGPSUIFDBMDVMBUJPOPG26 UIFXJOSBUFPOUIFTUBUFJTVQEBUFECZW p v = 70 %

Slide 41

Slide 41 text

.$54#BDLVQ 6QEBUFXJOSBUFPGFBDITUBUFBOEQSPQBHBUFVOUJMUIFSPPUOPEF p v = 70 %

Slide 42

Slide 42 text

1PJOU*NQSPWFNFOUT PO3- (p, v) = fθ (s)BOEl = (z − v)2 − πT log p + c∥θ∥2 w 5IFEVBMOFUXPSL QBSBNFUFS BDDVNVMBUFEBUB CZTFMGQMBZ TUFQSFQFBUFEUIPVTBOEUJNFT w #BTFEPOUIFSFTVMU VQEBUFUIFQBSBNFUFSPGUIF OFUXPSL TUFQ BOEHFUOFXQBSBNFUFS w -FUUXPOFUXPSLJOTUBOUJBUJPOTDPNQFUF VQEBUFUIF OFUXPSLQBSBNFUFSJGUIFOFXQBSBNFUFSTFUXJOT w 3FQFBUTUFQBOETUFQ θ′ θ

Slide 43

Slide 43 text

4UFQ%BUB"DDVNVMBUJPO w %PBTFMGQMBZ4UPSFUIFPVUDPNF[ w 4UPSFBMM T К [ UVQMFTJOUIFHBNF w 5IFQPMJDZКJTDBMDVMBUFEBT w 3FQFBUUIFBCPWFQSPDFTTFTUJNFT πa = N(s, a)1/γ ∑ b N(s, b)1/γ

Slide 44

Slide 44 text

4UFQ1BSBNFUFSVQEBUF w $BMDVMBUFUIFMPTTGVODUJPOVTJOH T К [ FWBMVBUFE JOUIFQSFWJPVTTUFQ (p, v) = fθ (s)BOEl = (z − v)2 − πT log p + c∥θ∥2 w 6QEBUFQBSBNFUFSVTJOHHSBEJFOUEFTDFOUNFUIPE θ′ ← θ − α ⋅ Δθ

Slide 45

Slide 45 text

&NQJSJDBMFWBMVBUJPOPG "MQIB(P;FSP 'JHPG 4JMWFS

Slide 46

Slide 46 text

1FSGPSNBODFPG"MQIB(P ;FSP 'JHPG 4JMWFS

Slide 47

Slide 47 text

https://research.fb.com/facebook-open-sources-elf-opengo/

Slide 48

Slide 48 text

3FGFSFODFT 4JMWFS % )VBOH " .BEEJTPO $+ (VF[ " 4JGSF - WBOEFO%SJFTTDIF ( ʜ)BTTBCJT % .BTUFSJOHUIF HBNFPG(PXJUIEFFQOFVSBMOFUXPSLTBOEUSFFTFBSDI/BUVSF rIUUQTEPJPSH OBUVSF
 "MQIB(P 0UTVLJ 5 .JZBLF 4BJLZPJHPFBJBSVGBHPLBJUBJTIJOTIP4IJOTPHBLVTIVNPOUFLBSVSPLJUBOTBLVLZPLB HBLVTIVLBSBNJUBTPOPTIJLVNJ4IPFJTIB 4JMWFS % 4DISJUUXJFTFS + 4JNPOZBO , "OUPOPHMPV * )VBOH " (VF[ " ʜ)BTTBCJT % .BTUFSJOHUIFHBNF PG(PXJUIPVUIVNBOLOPXMFEHF/BUVSF rIUUQTEPJPSHOBUVSF
 "MQIB(P;FSP #SPXOF $ 1PXMFZ & 8IJUFIPVTF % -VDBT 4 .FNCFS 4 $PXMJOH 1* ʜ$PMUPO 4 "4VSWFZPG.POUF $BSMP5SFF4FBSDI.FUIPET*&&&53"/4"$5*0/40/$0.165"5*0/"-*/5&--*(&/$&"/%"**/(".&4 IUUQTEPJPSH5$*"*( "VFS 1 'JOJUFUJNFBOBMZTJTPGUIFNVMUJBSNFECBOEJUQSPCMFNXJUILOPXOUSFOE*&&&$POHSFTTPO &WPMVUJPOBSZ$PNQVUBUJPO $&$ rIUUQTEPJPSH$&$ $BJ 1 -VP : 4BYFOB " )TV % -FF 84 -F54%SJWF%SJWJOHJOB$SPXECZ-FBSOJOHGSPN5SFF4FBSDI 3FUSJFWFEGSPNIUUQTBSYJWPSHQEGQEG