[論紹] DEEP CONTEXTUALIZED WORD REPRESENTATIONS

/""$-:0.*,"* UI+VMZ %&&1$0/5&956"-*;&% 803%3&13&4&/5"5*0/4 (3"%6"5&4$)00-PG*/'03."5*0/4$*&/$&BOE5&$)/0-0(: 04","6/*7 +6/:"5",":"."

1BQFS*/'0 .&1FUFST ./FVNBOO .*ZZFS .(BSEOFS $$MBSL ,-FFBOE-;FUUMFNPZFS l%FFQDPOUFYUVBMJ[FEXPSESFQSFTFOUBUJPOTz *O1SPDFFEJOHTPG/""$-)-5
#FTU1BQFS

$POUFOUT 0WFSWJFX &-.P&NCFEEJOHT GSPN-BOHVBHF.PEFMT &WBMVBUJPO "OBMZTJT
$PODMVTJPO

0WFSWJFX 1SPQPTFBOFXUZQFPGXPSESFQSFTFOUBUJPO&-.P • .PEFMTCPUIPG ØDPNQMFYDIBSBDUFSJTUJDTPGXPSEVTF FH TZOUBYBOETFNBOUJDT Ø)PXUIFTFVTFTWBSZBDSPTTMJOHVJTUJDDPOUFYUT JF UPNPEFMQPMZTFNZ
• #BTFEPOJOUFSOBMTUBUFTPGB#J-45.-BOHVBHFNPEFM • $BOCFFBTJMZUPBEEFEUPFYJTUJOHNPEFMT BOEJNQSPWFUIF405"BDSPTTNBOZUBTLT

$PODMVTJPO

&-.P&NCFEEJOHT GSPN-BOHVBHF.PEFMT 8PSESFQSFTFOUBUJPOGSPN#J-45.-BOHVBHF.PEFM 1SFUSBJOFE#J-45.-BOHVBHF.PEFM TNBMM DBU -45. -45. -45.
DVUF ɾɾɾ #J-45. #J-45. #J-45. -45. -45. -45. #J-45. #J-45. #J-45. &-.P ! "# "$ "%

#JEJSFDUJPOBMMBOHVBHFNPEFMT 5SBJOCJEJSFDUJPOBM-.CBTFEPO4UBDLFE#J-45. TNBMM DBU -45. -45. -45. DVUF ɾɾɾ
#J-45. #J-45. #J-45. -45. -45. -45. #J-45. #J-45. #J-45. DBU JT FPT !"#$%$&': ) *+, - log 1 2*|2,, … , 2*6,; 89, 8:;<=, 8> + log 1(2*A,, … , 2-; 89, 8:;<=, 8>) 89: Embedding matrix 8:;<=: parameters for LSTMs 8>: parameters for Softmax Layer

&-.P 8PSE3FQSFTFOUBUJPOPGUPLFO!" JTCFMPX !" -45. GPSXBSE -45. GPSXBSE #"
= %& '(, *&,+ '(, *&,+ '(|- = 1, … , 0 = *&,+ '(|- = 0, … , 0 %& '( XPSEFNCFEEJOHT *&,+ '(PVUQVUPG KUI MBZFS LUI DFMM CBDLXBSE -45. CBDLXBSE *&,+ '( *&,+ '( %&,+ '(

&-.P&NCFEEJOHT GSPN-BOHVBHF.PEFMT 0QUJNJ[FUIFXFJHIUPGFBDIIJEEFOMBZFSGPSFBDIUBTL -45. #J-45. -45. #J-45. &-.PL UBTL
! "# "$ "% &' ()*+' ,-.' = ( 0' ; 2,-.' = !,-.' 3 456 7 "4 ,-.'89,; <= >?@>9 TPGUNBYOPSNBMJ[FE XFJHIUT !,-.' XFJHIUGPSFOUJSF &-.P WFDUPS

"QQMZJOH&-.P UPTVQFSWJTFEUBTLT $PODBU &-.P BOEUBTL`TFNCFEEJOHIJEEFOMBZFSPVUQVU &-.PL UBTL 8PSE&NCFEEJOHL &-.PL
UBTL )JEEFO MBZFS PVUQVUL 3// $//

$PODMVTJPO

5SBJOJOHTFUUJOHT • /PGMBZFSTUBDLJOH MBZFST • /PGUSBJOJOHFQPDIT FQPDIT • -45. VOJUTJ[F
VOJUT • -45. EJNFOTJPO • $PSQVT #JMMJPO8PSE#FODINPSL • "QQMZJOHSFTJEVBMDPOOFDUJPOGPSGJSTUMBZFSUPTFDPOEMBZFS

&WBMVBUJPO3FTVMUT • "QQMZ&-.0UPBTJNQMFCBTFMJOFJOUBTLT 2" 35& 43- $PSFGFSFODF /&3 4FOUJNFOU
• "DIJFWFTDPSFTDPNQBSBCMFUPPSCFUUFSUIBO405"JOBMM

$PODMVTJPO

8IFSFUPJODMVEF&-.P • *O43- UBTLTQFDJGJDSFQSFTFOUBUJPOJTNPSFJNQPSUBOU UIVT&-.P JOUIFPVUQVUMBZFSJTMPXFGGFDU • 'PSNPEFMTXJUIBUUFOUJPOMBZFSBGUFS3//JUJTFGGFDUJWF UPBEE&-.P
UPUIFPVUQVUPG3//

8IBU*/'0TDBQUVSFECZ&-.P • $PNQBSFEXJUI(MPWF &-.P DBOXFMMDBQUVSFDPOUFYU JOGPSNBUJPO

4BNQMFFGGJDJFODZ • 6TJOH&-.P NBLFT MFBSOJOHNPSFFGGJDJFOU UIBOFYDJTUJOH NPEFM

7JTVBMJ[BUJPOPGMFBSOFEXFJHIUT • 7JTVBMJ[BUJPOPGXFJHIUJOH PG&-.P JOFBDIFOEUBTL • *OJOQVUMBZFS GJSTU#J-45. -BZFSJTNPTUFGGFDUJWF •
*OPVUQVUMBZFS VTJOH JOGPSNBUJPOPGBMMMBZFSTPO BWFSBHF

$PODMVTJPO

$PODMVTJPO 1SPQPTFEBOFXUZQFPGXPSESFQSFTFOUBUJPO&-.P • .PEFMTCPUIPG ØDPNQMFYDIBSBDUFSJTUJDTPGXPSEVTF FH TZOUBYBOETFNBOUJDT Ø)PXUIFTFVTFTWBSZBDSPTTMJOHVJTUJDDPOUFYUT JF UPNPEFMQPMZTFNZ
• #BTFEPOJOUFSOBMTUBUFTPGB#J-45.-BOHVBHFNPEFM • $BOCFFBTJMZUPBEEFEUPFYJTUJOHNPEFMT BOEJNQSPWFUIF405"BDSPTTNBOZUBTLT

[論紹] DEEP CONTEXTUALIZED WORD REPRESENTATIONS

[論紹] DEEP CONTEXTUALIZED WORD REPRESENTATIONS

onizuka laboratory

More Decks by onizuka laboratory

Other Decks in Research

Featured

Transcript

/""$-:0.," UI+VMZ %&&1$0/5&956"-;&% 803%3&13&4&/5"50/4 (3"%6"5&4$)00-PG/'03."50/4$&/$&BOE5&$)/0-0(: 04","6/7 +6/:"5",":"."

1BQFS/'0 .&1FUFST ./FVNBOO .ZZFS .(BSEOFS $$MBSL ,-FFBOE-;FUUMFNPZFS l%FFQDPOUFYUVBMJ[FEXPSESFQSFTFOUBUJPOTz *O1SPDFFEJOHTPG/""$-)-5

$POUFOUT 0WFSWJFX &-.P&NCFEEJOHT GSPN-BOHVBHF.PEFMT &WBMVBUJPO "OBMZTJT

0WFSWJFX 1SPQPTFBOFXUZQFPGXPSESFQSFTFOUBUJPO&-.P • .PEFMTCPUIPG ØDPNQMFYDIBSBDUFSJTUJDTPGXPSEVTF FH TZOUBYBOETFNBOUJDT Ø)PXUIFTFVTFTWBSZBDSPTTMJOHVJTUJDDPOUFYUT JF UPNPEFMQPMZTFNZ

$POUFOUT 0WFSWJFX &-.P&NCFEEJOHT GSPN-BOHVBHF.PEFMT &WBMVBUJPO "OBMZTJT

&-.P&NCFEEJOHT GSPN-BOHVBHF.PEFMT 8PSESFQSFTFOUBUJPOGSPN#J-45.-BOHVBHF.PEFM 1SFUSBJOFE#J-45.-BOHVBHF.PEFM TNBMM DBU -45. -45. -45.

#JEJSFDUJPOBMMBOHVBHFNPEFMT 5SBJOCJEJSFDUJPOBM-.CBTFEPO4UBDLFE#J-45. TNBMM DBU -45. -45. -45. DVUF ɾɾɾ

&-.P 8PSE3FQSFTFOUBUJPOPGUPLFO!" JTCFMPX !" -45. GPSXBSE -45. GPSXBSE #"

&-.P&NCFEEJOHT GSPN-BOHVBHF.PEFMT 0QUJNJ[FUIFXFJHIUPGFBDIIJEEFOMBZFSGPSFBDIUBTL -45. #J-45. -45. #J-45. &-.PL UBTL

"QQMZJOH&-.P UPTVQFSWJTFEUBTLT $PODBU &-.P BOEUBTL`TFNCFEEJOHIJEEFOMBZFSPVUQVU &-.PL UBTL 8PSE&NCFEEJOHL &-.PL

$POUFOUT 0WFSWJFX &-.P&NCFEEJOHT GSPN-BOHVBHF.PEFMT &WBMVBUJPO "OBMZTJT

5SBJOJOHTFUUJOHT • /PGMBZFSTUBDLJOH MBZFST • /PGUSBJOJOHFQPDIT FQPDIT • -45. VOJUTJ[F

&WBMVBUJPO3FTVMUT • "QQMZ&-.0UPBTJNQMFCBTFMJOFJOUBTLT 2" 35& 43- $PSFGFSFODF /&3 4FOUJNFOU

$POUFOUT 0WFSWJFX &-.P&NCFEEJOHT GSPN-BOHVBHF.PEFMT &WBMVBUJPO "OBMZTJT

8IFSFUPJODMVEF&-.P • *O43- UBTLTQFDJGJDSFQSFTFOUBUJPOJTNPSFJNQPSUBOU UIVT&-.P JOUIFPVUQVUMBZFSJTMPXFGGFDU • 'PSNPEFMTXJUIBUUFOUJPOMBZFSBGUFS3//JUJTFGGFDUJWF UPBEE&-.P

8IBU*/'0TDBQUVSFECZ&-.P • $PNQBSFEXJUI(MPWF &-.P DBOXFMMDBQUVSFDPOUFYU JOGPSNBUJPO

4BNQMFFGGJDJFODZ • 6TJOH&-.P NBLFT MFBSOJOHNPSFFGGJDJFOU UIBOFYDJTUJOH NPEFM

7JTVBMJ[BUJPOPGMFBSOFEXFJHIUT • 7JTVBMJ[BUJPOPGXFJHIUJOH PG&-.P JOFBDIFOEUBTL • *OJOQVUMBZFS GJSTU#J-45. -BZFSJTNPTUFGGFDUJWF •

$POUFOUT 0WFSWJFX &-.P&NCFEEJOHT GSPN-BOHVBHF.PEFMT &WBMVBUJPO "OBMZTJT

$PODMVTJPO 1SPQPTFEBOFXUZQFPGXPSESFQSFTFOUBUJPO&-.P • .PEFMTCPUIPG ØDPNQMFYDIBSBDUFSJTUJDTPGXPSEVTF FH TZOUBYBOETFNBOUJDT Ø)PXUIFTFVTFTWBSZBDSPTTMJOHVJTUJDDPOUFYUT JF UPNPEFMQPMZTFNZ