論文紹介: Exploiting Monolingual Data at Scale for Neural Machine Translation

&YQMPJUJOH.POPMJOHVBM %BUBBU4DBMFGPS/FVSBM .BDIJOF5SBOTMBUJPO -JKVO8V :JSFO8BOH :JOHDF9JB 5BP2JO +JBOIVBOH-BJ 5JF:BO-JV
&./-1 5.6,PNBDIJMBC.4BUPSV,BUTVNBUB

"CTUSBDU w )PXUPF⒎FDUJWFMZMFWFSBHFNPOPMJOHVBMEBUBJTBOJNQPSUBOU SFTFBSDIUPQJDGPS/.5  5IFSFBSFQMFOUZPGXPSLTTUVEZJOHUIJTQSPCMFN w #BDL5SBOTMBUJPO #5 JTUIFPOFPGUIFNPTUDJUFEBQQSPBDI  MFWFSBHFUIFUBSHFUTJEFNPOPMJOHVBMEBUB
w 0OUIFPUIFSIBOE UIFJOWFTUJHBUJPOPOTPVSDFTJEFNPOPMJOHVBMEBUB JTWFSZMJNJUFE  ;IBOHBOE;POH BOE6F⒏OHFUBM VTF  UIFTPVSDFTJEFEBUBUPNBLFUIFTZOUIFUJDUBSHFUEBUB w 5IJTQBQFSTUVEZIPXUPMFWFSBHFCPUITPVSDFTJEFBOEUBSHFUTJEF NPOPMJOHVBMEBUBUPCPPTUUIFBDDVSBDZPG/.5 !2

*NQSPWJOH/.5CZ .POPMJOHVBM%BUB w 5BSHFUTJEFNPOPMJOHVBMEBUB  GVTJPO/.5 -.NPEFM  #5NBLFUIFTZOUIFUJDTPVSDFTJEFEBUBGSPN  UBSHFUTJEFNPOPMJOHVBMEBUB w 4PVSDFTJEFNPOPMJOHVBMEBUB 
TFMGMFBSOJOHNBLFUIFTZOUIFUJDUBSHFUTJEFEBUB  GSPNTPVSDFTJEFNPOPMJOHVBMEBUB !3

TFMGMFBSOJOHNBLFUIFTZOUIFUJDUBSHFUTJEFEBUB  GSPNTPVSDFTJEFNPOPMJOHVBMEBUB !4 #5SFRVJSFTUSBJOJOHBOBEEJUJPOBMUBSHFUUPTPVSDF/.5 NPEFMHJWFOUIFCJMJOHVBMEBUBTFU  5IFUSBOTMBUJPOPVUQVUBOEUIFUBSHFUTJEFNPOPMJOHVBM EBUBUIFOQBJSFEBTTZOUIFUJDQBSBMMFMDPSQVTUPBVHNFOU UIFPSJHJOBMCJMJOHVBMEBUBTFU EJSFDUJPO&OHMJTIˠ(FSNBO TZOUIFUJDPSBOHF (FOVJOFCJUFYU  QBSBMMFMEBUB &O 5FYU %F 5FYU 5SBJO  %F&O #5 .PEFM  XJUICJUFYU .BLFTZOUIFUJDTSD VTJOH#5.PEFM  NPOPMJOHVBMEBUB %F  5FYU  NPOP &O 5FYU  TZOUIFUJD

TFMGMFBSOJOHNBLFUIFTZOUIFUJDUBSHFUTJEFEBUB  GSPNTPVSDFTJEFNPOPMJOHVBMEBUB !5 #5SFRVJSFTUSBJOJOHBOBEEJUJPOBMUBSHFUUPTPVSDF/.5 NPEFMHJWFOUIFCJMJOHVBMEBUBTFU  5IFUSBOTMBUJPOPVUQVUBOEUIFUBSHFUTJEFNPOPMJOHVBM EBUBUIFOQBJSFEBTTZOUIFUJDQBSBMMFMDPSQVTUPBVHNFOU UIFPSJHJOBMCJMJOHVBMEBUBTFU EJSFDUJPO&OHMJTIˠ(FSNBO TZOUIFUJDPSBOHF (FOVJOFCJUFYU  QBSBMMFMEBUB &O 5FYU %F 5FYU 5SBJO  %F&O #5 .PEFM  XJUICJUFYU .BLFTZOUIFUJDTSD VTJOH#5.PEFM  NPOPMJOHVBMEBUB %F  5FYU  NPOP &O 5FYU  TZOUIFUJD 3FMBUFE8PSL  &EVOPWFUBMpSTUMZQSPWJEFBO FYUFOTJWFBOBMZTJTPGUIFCBDL USBOTMBUJPOBUTDBMF

TFMGMFBSOJOHNBLFUIFTZOUIFUJDUBSHFUTJEFEBUB  GSPNTPVSDFTJEFNPOPMJOHVBMEBUB !6 4FMGMFBSOJOHBQQSPBDIHFOFSBUFTUIFTZOUIFUJDEBUBGPS UIFTPVSDFTJEFNPOPMJOHVBMEBUB XIJDIJTBTFNJ TVQFSWJTFENFUIPE EJSFDUJPO&OHMJTIˠ(FSNBO TZOUIFUJDPSBOHF (FOVJOFCJUFYU  QBSBMMFMEBUB &O 5FYU %F 5FYU 5SBJO  &O%F.PEFM  XJUICJUFYU .BLFTZOUIFUJDUHU VTJOHGPSXBSE.PEFM  NPOPMJOHVBMEBUB &O  5FYU  NPOP %F 5FYU  TZOUIFUJD

5SBJOJOH4USBUFHZ  /PUBUJPO w 9 :MBOHVBHF w UIFDPMMFDUJPOPGBMMTFOUFODFTGPSFBDIMBOHVBHF w UIFCJMJOHVBMUSBJOJOHQBJST  w
UIFDPMMFDUJPOPG NPOPMJOHVBMTFOUFODFT  w USBOTMBUJPONPEFM !7 X, Y B = {(xi , yi )}N i=1 xi ∈ X, yi ∈ Y xj ∈ X, yj ∈ Y Mx = {xj }Mx j=1 My = {yj }My j=1 f : X ↦ Y

5SBJOJOH4USBUFHZ 1SFQBSBUJPO w USBJOUXPUSBOTMBUJPONPEFMPOUIFCJMJOHVBMEBUB  w (FOFSBUFUIFGPMMPXJOHUXPTZOUIFUJDEBUBTFUT  GPSXBSEUSBOTMBUJPO  CBDLXBSEUSBOTMBUJPO w 5IJTQBQFSNBJOMZBEPQUCFBNTFBSDIUPHFOFSBUF
UIFTFOUFODFT !8 fb : X ↦ Y gb : Y ↦ X B Bs = {(x, fb (x))|x ∈ Mx } Bt = {(gb (y), y)|y ∈ My }

5SBJOJOH4USBUFHZ -BSHFTDBMFOPJTFEUSBJOJOH w 5IFZBEEOPJTFUPUIFTPVSDFTJEFEBUBPGCPUIBOEGPS USBJOJOHJOTUFBEPGEJSFDUMZVTJOHUIFNUPUSBJONPEFMT  ˠ#VJMEUXPGPMMPXJOHOPJTFEEBUBTFUT w OPJTFGVODUJPO  SBOEPNMZSFQMBDFBXPSEUPCFBTQFDJBMUPLFOXJUI QSPCBCJMJUZ 
SBOEPNMZESPQUIFXPSETXJUIQSPCBCJMJUZ  SBOEPNMZTIV⒐F TXBQ UIFXPSETXJUIDPOTUSBJOUUIBUUIF XPSETXJMMOPUCFTIV⒐FEGVSUIFSUIBOUISFFQPTJUJPOTEJTUBODF w 5IFZUIFOUSBJOBO/.5NPEFMGPS9UP:USBOTMBUJPOPO !9 Bn s = {(σ(x), y)|(x, y) ∈ Bs } Bn t = {(σ(x), y)|(x, y) ∈ Bt } σ B ∪ Bn s ∪ Bn t Bs Bt fn

5SBJOJOH4USBUFHZ $MFBOEBUBUVOJOH w 5IFZGVSUIFSpOFUVOFUIFOPJTFEUSBJOJOHNPEFMPOUIF DMFBOWFSTJPOPGUIFTZOUIFUJDEBUBXJUIPVUBEEJOHOPJTF NBOVBMMZ w 5SBJOBOPUIFSGPS9UP:USBOTMBUJPOBOEBOPUIFSGPS :UP9USBOTMBUJPO  ˠVTFUIFNUPCVJMEOFXTZOUIFUJDEBUB
  TVCTBNQMFTFOUFODFTUPGPSNBOE w 'JOFUVOFUIFUSBOTMBUJPONPEFMPOUIFOFXEBUB  JTJOJUJBMJ[FECZ   !10 min ∑ (x,y)∈B∪Bs s ∪Bs t − log P(y|x; f ) Bs Bt Bs s Bs t fb gb fn fn f

&YQFSJNFOUT TFUUJOH w UBTL8.5&O%F %F&O %F'S 'S%F w EBUB&O%F %F&O 
QBSBMMFMDPSQVT  8.5 .TFOUT &VSPQBSMW /FXT$PNNFOUBSZW $PNNPO$SBXM %PDVNFOUTQMJU3BQJEDPSQVT  8.51$ .TFOUT QSFQSPDFTTFE1SBDSBXM 8.5  NPOPMJOHVBMDPSQVT  &O %F/FXT$SBXMˠ.TFOUT  WBMJEBUJPOEBUB  OFXTUFTU  UFTUEBUB  OFXTUFTU w EBUB%F'S 'S%F  QBSBMMFMDPSQVT  8.5.8.51$.  NPOPMJOHVBMDPSQVT  /FXT$SBXM.TFOUT  WBMJEBUJPO UFTU8.5TFUUJOH w ,#1&USBOTGPSNFSCJHCBUDIUPLFOTQFS(161VQEBUFQBSBNFUFSFWFSZNJOJCBUDI !11 'PSOPJTFEUSBJOJOH UIFNPEFMTVTFEGPS USBOTMBUJOHNPOPMJOHVBMEBUBBSFUSBJOFEPO 8.5  'PSpOFUVOJOH UIFNPEFMTVTFEGPS USBOTMBUJOHNPOPMJOHVBMEBUBBSFUSBJOFEPO 8.51$

&YQFSJNFOUT 3FTVMU !12 5IJTQBQFSDPNQBSFTUIFJSTUSBUFHZUP UIFWBOJMMB#5 XIJDIDPOTJTUTPG. TZOUIFUJDEBUBBOE8.51$ 8.51$ #5

&YQFSJNFOUT 405" !13

"OBMZTJT 4PVSDFPS5BSHFU.POPEBUB w 5IFZDPNQBSFUISFFEJ⒎FSFOUXBZTUPVTFNPOPMJOHVBMEBUB  JODMVEJOHMFWFSBHJOHTPVSDFTJEFNPOPMJOHVBMEBUBPOMZ JF   UBSHFUTJEFNPOPMJOHVBMEBUBPOMZ JF
  UIFNPOPMJOHVBMEBUBGSPNCPUIUXPTJEFT JF w 5PUBMNPOPMJOHVBMTJ[F. w 8JUIPVUOPJTF !14 Bt Bs Bs + Bt

"OBMZTJT 4ZOUIFUJD%BUB(FOFSBUJPO w 5IFZDPOEVDUFYQFSJNFOUTPOEJ⒎FSFOUHFOFSBUFETZOUIFUJDEBUB UPWFSJGZXIFUIFS BEEJOHOPJTFJTFTTFOUJBM w 5IFZDPNQBSFUIFJSOPJTFEUSBJOJOHEBUBBOEXJUIBOPUIFSUXPCBTFMJOFT  BOEXJUIPVUBOZUSBOTGPSNBUJPO  BOEUIFSBOEPNMZTBNQMFETZOUIFUJDEBUB
  XIFSFFBDIUPLFOJOUIFUSBOTMBUJPOJTTBNQMFEGSPNUIFNVMUJOPNJBMEJTUSJCVUJPO EFUFSNJOFECZUIF/.5NPEFM 3BOEPNTBNQMF !15 Bn t Bn s Bs Bt Br t Bs 8JUIPVU$MFBO5VOJOH

"OBMZTJT 4DBMFPG.POPMJOHVBM%BUB 5IFZHJWFBDPNQBSJTPOPGEJ⒎FSFOUEBUBTDBMFTGPSFBDILJOEPGTZOUIFUJDEBUB !16 ˠ8.51$SFTVMU 5IFZWFSJGZUIBUTPVSDFTJEFNPOPMJOHVBMEBUBJTIFMQGVMBOEUIFCFTU XBZUPVTFJUJTUPDPNCJOFXJUIUBSHFUTJEFNPOPMJOHVBMEBUB 5IFZTIPXUIBUBEEJOHOPJTFUPTZOUIFUJDEBUBPVUQFSGPSNTUIBUXJUIPVU
OPJTF

"OBMZTJT 4ZOUIFUJD5VOJOH w *TJUIFMQGVMUPVTFUXPHSPVQTPGNPEFMTPGCVJMEJOHTZOUIFUJDEBUBGPSOPJTFE USBJOJOHBOEDMFBOUSBJOJOH   ˠ"UpOFUVOJOHTUFQ VTFBTVCTBNQMFPGTZOUIFUJDEBUBPGOPJTFEUSBJOJOHTUFQ w *TJUIFMQGVMUPVTFOPJTFEUSBJOJOHpSTUSFHBSEJOHGVUVSF#-&6TDPSFBDIJFWFECZ
UIFpOFUVOFTUFQ XJUIDMFBOUVOJOH   ˠ5SBJOJOHXJUIPVUOPJTFpSTUBOEUIFOpOFUVOJOHVTJOHDMFBOEBUB !17

$PODMVTJPO w 5IFZFYQMPJUUIFNPOPMJOHVBMEBUBBUTDBMFGPSUIFOFVSBM NBDIJOFUSBOTMBUJPO w 5IFZQSPQPTFBOF⒎FDUJWFUSBJOJOHTUSBUFHZUPCPPTUUIF/.5 QFSGPSNBODFCZMFWFSBHJOHCPUITPVSDFTJEFBOEUBSHFUTJEF NPOPMJOHVBMEBUB w 'PSGVUVSFXPSL
UIFZXPVMEMJLFUPWFSJGZPVSUSBJOJOHTUSBUFHZ PONPSFMBOHVBHFQBJSTBOEPUIFSTFRVFODFUPTFRVFODF UBTLT w 5IFZBSFJOUFSFTUFEJOTUVEZJOHUIFJSOPJTFEUSBJOJOHXJUIPUIFS EBUBBVHNFOUBUJPOBQQSPBDIFT !18

3FMBUFE8PSL w &EVOPWFUBM 6OEFSTUBOEJOHCBDLUSBOTMBUJPOBU TDBMF&./-1 w ;IBOHBOE;POH &YQMPJUJOH4PVSDFTJEF.POPMJOHVBM %BUBJO/FVSBM.BDIJOF5SBOTMBUJPO&./-1 w
4FOOSJDIFUBM *NQSPWJOH/FVSBM.BDIJOF5SBOTMBUJPO .PEFMTXJUI.POPMJOHVBM%BUB"$- w 6F⒏OHFUBM 5SBOTEVDUJWFMFBSOJOHGPSTUBUJTUJDBM NBDIJOFUSBOTMBUJPO"$- !19

論文紹介: Exploiting Monolingual Data at Scale for Neural Machine Translation

論文紹介: Exploiting Monolingual Data at Scale for Neural Machine Translation

Satoru Katsumata

More Decks by Satoru Katsumata

Other Decks in Research

Featured

Transcript

&YQMPJUJOH.POPMJOHVBM %BUBBU4DBMFGPS/FVSBM .BDIJOF5SBOTMBUJPO -JKVO8V :JSFO8BOH :JOHDF9JB 5BP2JO +JBOIVBOH-BJ 5JF:BO-JV

"CTUSBDU w )PXUPF⒎FDUJWFMZMFWFSBHFNPOPMJOHVBMEBUBJTBOJNQPSUBOU SFTFBSDIUPQJDGPS/.5  5IFSFBSFQMFOUZPGXPSLTTUVEZJOHUIJTQSPCMFN w #BDL5SBOTMBUJPO #5 JTUIFPOFPGUIFNPTUDJUFEBQQSPBDI  MFWFSBHFUIFUBSHFUTJEFNPOPMJOHVBMEBUB

*NQSPWJOH/.5CZ .POPMJOHVBM%BUB w 5BSHFUTJEFNPOPMJOHVBMEBUB  GVTJPO/.5 -.NPEFM  #5NBLFUIFTZOUIFUJDTPVSDFTJEFEBUBGSPN  UBSHFUTJEFNPOPMJOHVBMEBUB w 4PVSDFTJEFNPOPMJOHVBMEBUB

*NQSPWJOH/.5CZ .POPMJOHVBM%BUB w 5BSHFUTJEFNPOPMJOHVBMEBUB  GVTJPO/.5 -.NPEFM  #5NBLFUIFTZOUIFUJDTPVSDFTJEFEBUBGSPN  UBSHFUTJEFNPOPMJOHVBMEBUB w 4PVSDFTJEFNPOPMJOHVBMEBUB

*NQSPWJOH/.5CZ .POPMJOHVBM%BUB w 5BSHFUTJEFNPOPMJOHVBMEBUB  GVTJPO/.5 -.NPEFM  #5NBLFUIFTZOUIFUJDTPVSDFTJEFEBUBGSPN  UBSHFUTJEFNPOPMJOHVBMEBUB w 4PVSDFTJEFNPOPMJOHVBMEBUB

*NQSPWJOH/.5CZ .POPMJOHVBM%BUB w 5BSHFUTJEFNPOPMJOHVBMEBUB  GVTJPO/.5 -.NPEFM  #5NBLFUIFTZOUIFUJDTPVSDFTJEFEBUBGSPN  UBSHFUTJEFNPOPMJOHVBMEBUB w 4PVSDFTJEFNPOPMJOHVBMEBUB

5SBJOJOH4USBUFHZ  /PUBUJPO w 9 :MBOHVBHF w UIFDPMMFDUJPOPGBMMTFOUFODFTGPSFBDIMBOHVBHF w UIFCJMJOHVBMUSBJOJOHQBJST  w

5SBJOJOH4USBUFHZ 1SFQBSBUJPO w USBJOUXPUSBOTMBUJPONPEFMPOUIFCJMJOHVBMEBUB  w (FOFSBUFUIFGPMMPXJOHUXPTZOUIFUJDEBUBTFUT  GPSXBSEUSBOTMBUJPO  CBDLXBSEUSBOTMBUJPO w 5IJTQBQFSNBJOMZBEPQUCFBNTFBSDIUPHFOFSBUF

5SBJOJOH4USBUFHZ -BSHFTDBMFOPJTFEUSBJOJOH w 5IFZBEEOPJTFUPUIFTPVSDFTJEFEBUBPGCPUIBOEGPS USBJOJOHJOTUFBEPGEJSFDUMZVTJOHUIFNUPUSBJONPEFMT  ˠ#VJMEUXPGPMMPXJOHOPJTFEEBUBTFUT w OPJTFGVODUJPO  SBOEPNMZSFQMBDFBXPSEUPCFBTQFDJBMUPLFOXJUI QSPCBCJMJUZ

5SBJOJOH4USBUFHZ $MFBOEBUBUVOJOH w 5IFZGVSUIFSpOFUVOFUIFOPJTFEUSBJOJOHNPEFMPOUIF DMFBOWFSTJPOPGUIFTZOUIFUJDEBUBXJUIPVUBEEJOHOPJTF NBOVBMMZ w 5SBJOBOPUIFSGPS9UP:USBOTMBUJPOBOEBOPUIFSGPS :UP9USBOTMBUJPO  ˠVTFUIFNUPCVJMEOFXTZOUIFUJDEBUB

&YQFSJNFOUT TFUUJOH w UBTL8.5&O%F %F&O %F'S 'S%F w EBUB&O%F %F&O

&YQFSJNFOUT 3FTVMU !12 5IJTQBQFSDPNQBSFTUIFJSTUSBUFHZUP UIFWBOJMMB#5 XIJDIDPOTJTUTPG. TZOUIFUJDEBUBBOE8.51$ 8.51$ #5

&YQFSJNFOUT 405" !13

"OBMZTJT 4PVSDFPS5BSHFU.POPEBUB w 5IFZDPNQBSFUISFFEJ⒎FSFOUXBZTUPVTFNPOPMJOHVBMEBUB  JODMVEJOHMFWFSBHJOHTPVSDFTJEFNPOPMJOHVBMEBUBPOMZ JF   UBSHFUTJEFNPOPMJOHVBMEBUBPOMZ JF

"OBMZTJT 4ZOUIFUJD%BUB(FOFSBUJPO w 5IFZDPOEVDUFYQFSJNFOUTPOEJ⒎FSFOUHFOFSBUFETZOUIFUJDEBUB UPWFSJGZXIFUIFS BEEJOHOPJTFJTFTTFOUJBM w 5IFZDPNQBSFUIFJSOPJTFEUSBJOJOHEBUBBOEXJUIBOPUIFSUXPCBTFMJOFT  BOEXJUIPVUBOZUSBOTGPSNBUJPO  BOEUIFSBOEPNMZTBNQMFETZOUIFUJDEBUB

"OBMZTJT 4DBMFPG.POPMJOHVBM%BUB 5IFZHJWFBDPNQBSJTPOPGEJ⒎FSFOUEBUBTDBMFTGPSFBDILJOEPGTZOUIFUJDEBUB !16 ˠ8.51$SFTVMU 5IFZWFSJGZUIBUTPVSDFTJEFNPOPMJOHVBMEBUBJTIFMQGVMBOEUIFCFTU XBZUPVTFJUJTUPDPNCJOFXJUIUBSHFUTJEFNPOPMJOHVBMEBUB 5IFZTIPXUIBUBEEJOHOPJTFUPTZOUIFUJDEBUBPVUQFSGPSNTUIBUXJUIPVU

"OBMZTJT 4ZOUIFUJD5VOJOH w TJUIFMQGVMUPVTFUXPHSPVQTPGNPEFMTPGCVJMEJOHTZOUIFUJDEBUBGPSOPJTFE USBJOJOHBOEDMFBOUSBJOJOH   ˠ"UpOFUVOJOHTUFQ VTFBTVCTBNQMFPGTZOUIFUJDEBUBPGOPJTFEUSBJOJOHTUFQ w TJUIFMQGVMUPVTFOPJTFEUSBJOJOHpSTUSFHBSEJOHGVUVSF#-&6TDPSFBDIJFWFECZ

$PODMVTJPO w 5IFZFYQMPJUUIFNPOPMJOHVBMEBUBBUTDBMFGPSUIFOFVSBM NBDIJOFUSBOTMBUJPO w 5IFZQSPQPTFBOF⒎FDUJWFUSBJOJOHTUSBUFHZUPCPPTUUIF/.5 QFSGPSNBODFCZMFWFSBHJOHCPUITPVSDFTJEFBOEUBSHFUTJEF NPOPMJOHVBMEBUB w 'PSGVUVSFXPSL

3FMBUFE8PSL w &EVOPWFUBM 6OEFSTUBOEJOHCBDLUSBOTMBUJPOBU TDBMF&./-1 w ;IBOHBOE;POH &YQMPJUJOH4PVSDFTJEF.POPMJOHVBM %BUBJO/FVSBM.BDIJOF5SBOTMBUJPO&./-1 w