[論紹] DEEP CONTEXTUALIZED WORD REPRESENTATIONS

弊研究室で行なったNAACL読み会の発表資料です。

Transcript

1. /""$-
   :0.*,"*
   UI
   +VMZ
    %&&1
   $0/5&956"-*;&% 803%
   3&13&4&/5"5*0/4 (3"%6"5&
   4$)00-
   PG
   */'03."5*0/
   4$*&/$&
   BOE
   5&$)/0-0(:
   04","
   6/*7 +6/:"
   5",":"."
2. 1BQFS
   */'0 .
   &
   1FUFST
   .
   /FVNBOO
   .
   *ZZFS
   .
   (BSEOFS
   $
   $MBSL
   ,
   -FF
   BOE
   -
   ;FUUMFNPZFS l%FFQ
   DPOUFYUVBMJ[FE
   XPSE
   SFQSFTFOUBUJPOTz *O
   1SPDFFEJOHT
   PG
   /""$-)-5
   #FTU
   1BQFS
   
3. $POUFOUT  0WFSWJFX  &-.P
   &NCFEEJOHT GSPN
   -BOHVBHF
   .PEFMT  &WBMVBUJPO  "OBMZTJT  $PODMVTJPO
   
4. 0WFSWJFX 1SPQPTF
   B
   OFX
   UZQF
   PG
   XPSE
   SFQSFTFOUBUJPO
   &-.P • .PEFMT
   CPUI
   PG ØDPNQMFY
   DIBSBDUFSJTUJDT
   PG
   XPSE
   VTF
   FH
   TZOUBY
   BOE
   TFNBOUJDT Ø)PX
   UIFTF
   VTFT
   WBSZ
   BDSPTT
   MJOHVJTUJD
   DPOUFYUT
   JF
   UP
   NPEFM
   QPMZTFNZ • #BTFE
   PO
   JOUFSOBM
   TUBUFT
   PG
   B
   #J-45.
   -BOHVBHF
   NPEFM • $BO
   CF
   FBTJMZ
   UP
   BEEFE
   UP
   FYJTUJOH
   NPEFMT
   BOE
   JNQSPWF
   UIF
   405"
   BDSPTT
   NBOZ
   UBTLT
   
5. $POUFOUT  0WFSWJFX  &-.P
   &NCFEEJOHT GSPN
   -BOHVBHF
   .PEFMT  &WBMVBUJPO  "OBMZTJT  $PODMVTJPO
   
6. &-.P
   &NCFEEJOHT GSPN
   -BOHVBHF
   .PEFMT 8PSE
   SFQSFTFOUBUJPO
   GSPN
   #J-45.
   -BOHVBHF
   .PEFM
   1SFUSBJOFE
   #J-45.
   -BOHVBHF
   .PEFM TNBMM DBU -45. -45. -45. DVUF ɾɾɾ #J-45. #J-45. #J-45. -45. -45. -45. #J-45. #J-45. #J-45. &-.P ! "# "$ "%
7. #JEJSFDUJPOBM
   MBOHVBHF
   NPEFMT 5SBJO
   CJEJSFDUJPOBM
   -.
   CBTFE
   PO
   4UBDLFE
   #J-45.
   TNBMM DBU -45. -45. -45. DVUF ɾɾɾ #J-45. #J-45. #J-45. -45. -45. -45. #J-45. #J-45. #J-45. DBU JT FPT !"#$%$&': ) *+, - log 1 2*|2,, … , 2*6,; 89, 8:;<=, 8> + log 1(2*A,, … , 2-; 89, 8:;<=, 8>) 89: Embedding matrix 8:;<=: parameters for LSTMs 8>: parameters for Softmax Layer
8. &-.P 8PSE
   3FQSFTFOUBUJPO
   PG
   UPLFO
   !" JT
   CFMPX
   !" -45. GPSXBSE -45. GPSXBSE #" = %& '(, *&,+ '(, *&,+ '(|- = 1, … , 0 = *&,+ '(|- = 0, … , 0 %& '( XPSE
   FNCFEEJOHT *&,+ '(
   PVUQVU
   PG KUI MBZFS LUI DFMM CBDLXBSE -45. CBDLXBSE *&,+ '( *&,+ '( %&,+ '(
9. &-.P
   &NCFEEJOHT GSPN
   -BOHVBHF
   .PEFMT 0QUJNJ[F
   UIF
   XFJHIU
   PG
   FBDI
   IJEEFO
   MBZFS
   GPS
   FBDI
   UBTL
   -45. #J-45. -45. #J-45. &-.PL UBTL ! "# "$ "% &' ()*+' ,-.' = ( 0' ; 2,-.' = !,-.' 3 456 7 "4 ,-.'89,; <= >?@>9 
   TPGUNBYOPSNBMJ[FE
   XFJHIUT !,-.'  XFJHIU
   GPS
   FOUJSF
   &-.P WFDUPS
10. "QQMZJOH
    &-.P UP
    TVQFSWJTFE
    UBTLT $PODBU &-.P BOE
    UBTL`T
    FNCFEEJOH
    IJEEFO
    MBZFS
    PVUQVU
    &-.PL UBTL 8PSE
    &NCFEEJOHL &-.PL UBTL )JEEFO MBZFS PVUQVUL 3//
    $//
11. $POUFOUT  0WFSWJFX  &-.P
    &NCFEEJOHT GSPN
    -BOHVBHF
    .PEFMT  &WBMVBUJPO  "OBMZTJT  $PODMVTJPO 
    
12. 5SBJOJOH
    TFUUJOHT • /
    PG
    MBZFS
    TUBDLJOH
    
    
    MBZFST • /
    PG
    USBJOJOH
    FQPDIT
    
    
    FQPDIT • -45.
     VOJUTJ[F 
    
    VOJUT • -45.
     EJNFOTJPO 
     • $PSQVT 
    #JMMJPO
    8PSE
    #FODINPSL • "QQMZJOH
    SFTJEVBM
    DPOOFDUJPO
    GPS
    GJSTU
    MBZFS
    UP
    TFDPOE
    MBZFS
    
    
13. &WBMVBUJPO
    3FTVMUT • "QQMZ
    &-.0
    UP
    B
    TJNQMF
    CBTFMJOF
    JO
    
    UBTLT
    2"
    35&
    43-
    $PSFGFSFODF
    /&3
    4FOUJNFOU
    • "DIJFWF
    TDPSFT
    DPNQBSBCMF
    UP
    PS
    CFUUFS
    UIBO
    405"
    JO
    BMM
    
14. $POUFOUT  0WFSWJFX  &-.P
    &NCFEEJOHT GSPN
    -BOHVBHF
    .PEFMT  &WBMVBUJPO  "OBMZTJT  $PODMVTJPO
    
15. 8IFSF
    UP
    JODMVEF
    &-.P • *O
    43-
    UBTLTQFDJGJD
    SFQSFTFOUBUJPO
    JT
    NPSF
    JNQPSUBOU
    UIVT
    &-.P JO
    UIF
    PVUQVU
    MBZFS
    JT
    MPX
    FGGFDU • 'PS
    NPEFMT
    XJUI
    BUUFOUJPO
    MBZFS
    BGUFS
    3//
    JU
    JT
    FGGFDUJWF
    UP
    BEE
    &-.P UP
    UIF
    PVUQVU
    PG
    3//
    
16. 8IBU
    */'0T
    DBQUVSFE
    CZ
    &-.P • $PNQBSFE
    XJUI
    (MPWF
    &-.P DBO
    XFMM
    DBQUVSF
    DPOUFYU
    JOGPSNBUJPO
    
17. 4BNQMF
    FGGJDJFODZ • 6TJOH
    &-.P NBLFT
    MFBSOJOH
    NPSF
    FGGJDJFOU
    UIBO
    FYDJTUJOH NPEFM
    
18. 7JTVBMJ[BUJPO
    PG
    MFBSOFE
    XFJHIUT • 7JTVBMJ[BUJPO
    PG
    XFJHIUJOH
    PG
    &-.P JO
    FBDI
    FOE
    UBTL • *O
    JOQVU
    MBZFS
    GJSTU
    #J-45. -BZFS
    JT
    NPTU
    FGGFDUJWF • *O
    PVUQVU
    MBZFS
    VTJOH
    JOGPSNBUJPO
    PG
    BMM
    MBZFST
    PO
    BWFSBHF
    
19. $POUFOUT  0WFSWJFX  &-.P
    &NCFEEJOHT GSPN
    -BOHVBHF
    .PEFMT  &WBMVBUJPO  "OBMZTJT  $PODMVTJPO
    
20. $PODMVTJPO 1SPQPTFE
    B
    OFX
    UZQF
    PG
    XPSE
    SFQSFTFOUBUJPO
    &-.P • .PEFMT
    CPUI
    PG ØDPNQMFY
    DIBSBDUFSJTUJDT
    PG
    XPSE
    VTF
    FH
    TZOUBY
    BOE
    TFNBOUJDT Ø)PX
    UIFTF
    VTFT
    WBSZ
    BDSPTT
    MJOHVJTUJD
    DPOUFYUT
    JF
    UP
    NPEFM
    QPMZTFNZ • #BTFE
    PO
    JOUFSOBM
    TUBUFT
    PG
    B
    #J-45.
    -BOHVBHF
    NPEFM • $BO
    CF
    FBTJMZ
    UP
    BEEFE
    UP
    FYJTUJOH
    NPEFMT
    BOE
    JNQSPWF
    UIF
    405"
    BDSPTT
    NBOZ
    UBTLT