Save 37% off PRO during our Black Friday Sale! »

Human+AI Crowd Task Assignment Considering Result Quality Requirements (HCOMP2021)

Human+AI Crowd Task Assignment Considering Result Quality Requirements (HCOMP2021)

Masaki Kobayashi, Kei Wakabayashi, and Atsuyuki Morishima, "Human+AI Crowd Task Assignment Considering Result Quality Requirements", Proc. of the 9th AAAI Conference on Human Computation and Crowdsourcing (HCOMP2021)., Online conference, November 2021, pp. 97-107.

Aebe46af935b615cf6e742878cff52d9?s=128

Masaki KOBAYASHI

November 19, 2021
Tweet

Transcript

  1. )VNBO "*$SPXE5BTL"TTJHONFOU $POTJEFSJOH3FTVMU2VBMJUZ3FRVJSFNFOUT .BTBLJ,PCBZBTIJ ,FJ8BLBCBZBTIJ BOE"UTVZVLJ.PSJTIJNB JST CREST CyborgCrowd Immediate,

    United, Optimized
  2. 1PJOUTPG5IJT5BML  2VBMJUZDPOUSPMPGUBTLSFTVMUTJOUIF)VNBO "*$SPXEXJMM CFBOJNQPSUBOUJTTVFJOUIFOFBSGVUVSF  &YJTUJOHUFDIOJRVFTBSFTVCPQUJNBMJOUFSNTPGFGGJDJFODZBOE TDBMBCJMJUZ  8FQSPQPTFUBTLBTTJHONFOUBMHPSJUINT

    XIJDIUBLFBEWBOUBHF PGUIFTXFFUTQPUTPGFBDICMBDLCPY"*XPSLFS  0VSNFUIPEDBOSFEVDFIVNBOXPSLFSBTTJHONFOUTXIJMF TBUJTGZJOHUIFRVBMJUZSFRVJSFNFOUJOUIFQSFTFODFPGNBOZCMBDL CPY"*XPSLFST 
  3. 4PNF"QQMJDBUJPOT/FFEUP1SPDFTT#JH%BUB "DDVSBUFMZCVU*NNFEJBUFMZ FH /BUVSBMEJTBTUFSSFTQPOTF $SPXETPVSDJOHJTFGGFDUJWFGPSTVDIBQQT  "FSJBM1IPUP -BCFMJOH -BCFMFE"FSJBM1IPUP (PWFSONFOU

    3FRVFTUFS /FFEFWJEFODF JNNFEJBUFMZ /FYU"DUJPO
  4. 0VUTPVSDJOH"*QSPHSBN%FWFMPQNFOUJT $PNNPO1SBDUJDF  4NBSUFS"*.PEFM %BUBTFU $SPXETPVSDFE $SPXETPVSDJOH $SPXE8PSLFST .BDIJOF-FBSOJOH.PEFMT FH

    ,BHHMF "*$SPXE
  5. $PNCJOJOH)VNBOTBOE"*JTOPU"MXBZT&BTZ 8PSLGMPXUPMBCFMJNBHFTVTJOHDSPXETPVSDJOHBOE"*XJUIRVBMJUZ BDDVSBDZ   #FHJO $SPXETPVSDFGPS GFXJNBHFTUP NBLFBEBUBTFU 4UBSUB

    DPNQFUJUJPOGPS "*EFWFMPQNFOU *TUIF"* BDDFQUBCMF &OE :FT "TTJHOUIF"* GPSSFNBJOJOH JNBHFT 6TFIVNBOTGPS "--JNBHFT /P 3FEFTJHO UIFXPSLGMPX *NQSPWFUIFEBUBTFU • "DUJWFMFBSOJOH • &OTFNCMFMFBSOJOH • "VUP.- • .-XJUISFKFDUPQUJPOT %FDJTJPONBLJOH $PNCJOJOHUIFNXFMMUPBDIJFWFUIFSFRVJSFERVBMJUZJTIBSE OPUTDBMBCMF )PXDBOXFBDIJFWF UIFSFRVJSFERVBMJUZ
  6. )VNBO "*$SPXE 8FQSPQPTFUPEFBMXJUI"*NPEFMTBTCMBDLCPY"*XPSLFST  0VS"QQSPBDI )PXDBOXFDPOUSPMUIFUBTLSFTVMURVBMJUZ &WBMVBUFUIFRVBMJUZ PGXPSLFST 5BTLBTTJHONFOU #MBDL#PY

    "*8PSLFS EFWFMPQ 2VBMJUZSFRVJSFNFOU 5BTLT 3FRVFTUFS
  7. 0VS$POUSJCVUJPOT  8FEFGJOFBOPWFM)VNBO "*$SPXE5BTL"TTJHONFOU 1SPCMFN )"$5"1 UIBUBTTJHOTBGJYFETFUPGUBTLTUPIVNBO BOE"*XPSLFST  8FQSPQPTFUXPBMHPSJUINT

    UIBUQSPWJEFFGGJDJFOUTPMVUJPOTUP UIF)"$5"1XJUITPNFUIFPSFUJDBMHVBSBOUFFT  8FDPOEVDUFEBOFYUFOTJWFTFUPGFYQFSJNFOUTVTJOHPQFO CFODINBSLBOESFBMXPSMEEBUBTFUT 0VSDPEFBOEEBUBBSFBWBJMBCMFBUIUUQTHJUIVCDPNDSPXEV)"$5"1'SBNFXPSL 
  8. 1PJOUTPG5IJT5BML  2VBMJUZDPOUSPMPGUBTLSFTVMUTJOUIF)VNBO "*$SPXEXJMMCF BOJNQPSUBOUJTTVFJOUIFOFBSGVUVSF  &YJTUJOHUFDIOJRVFTBSFTVCPQUJNBMJOUFSNTPGFGGJDJFODZ BOETDBMBCJMJUZ  8FQSPQPTFUBTLBTTJHONFOUBMHPSJUINT

    XIJDIUBLFBEWBOUBHF PGUIFTXFFUTQPUTPGFBDICMBDLCPY"*XPSLFS  0VSNFUIPEDBOSFEVDFIVNBOXPSLFSBTTJHONFOUTXIJMF TBUJTGZJOHUIFRVBMJUZSFRVJSFNFOUJOUIFQSFTFODFPGNBOZCMBDL CPY"*XPSLFST 
  9. )VNBO "*$SPXE5BTL"TTJHONFOU1SPCMFN *OQVU 5BTLT "*8PSLFSTBOEUIF2VBMJUZSFRVJSFNFOU NJOJNVNBDDVSBDZ 0VUQVU "TFUPGUBTLBTTJHONFOUT UBTL XPSLFS

    QBJST $POTUSBJOUT  4BUJTGZUIFRVBMJUZSFRVJSFNFOU   .BYJNJ[FBTTJHONFOUUP"*XPSLFST  0VS"QQSPBDI )"$5"1 " # " % " # $ # AI Workers Human Workers Tasks while keeping the accuracy requirement Task Cluster: subset of tasks Task (Label)
  10. 4PMWFUIF)"$5"1 XJUI "DUJWF-FBSOJOH<>CBTFE"TTJHONFOU • "TTJHOUBTLTUPIVNBOTVOUJMUIF"*XPSLFSBDIJFWFTUIFUISFTIPME BMMPSOPUIJOH  • .-NPEFMDBORVFSZTPNFVOMBCFMFEEBUBQPJOUTUPJNQSPWFUIFJSQFSGPSNBODF 

    <>:BO:BO 3PNFS3PTBMFT (MFOO'VOH BOE+FOOJGFS(%Z"DUJWFMFBSOJOHGSPNDSPXET*$.-` DPNQMFUFEUBTLT IVNBOMBCFMT "MMPS/PUIJOH 0OMZ)VNBOT #FHJO "MMUBTLT XFSF MBCFMFE &OE :FT /P "TTJHODIPTF UBTLTUPIVNBOT 5SBJO"*XPSLFST CZIVNBOMBCFMT "*8PSLFS DIPPTF/ VOMBCFMFEUBTLT "TTJHOUBTLTUP UIF"*XPSLFST &WBMVBUF"*8PSLFS BDDVSBDZ "DDFQU 3FKFDU
  11. 4IPSUDPNJOHTPG&YJTUJOH.FUIPET  5IFNFUIPEBTTJHOTUBTLTUP"*TCZUIFlBMMPSOPUIJOHz TUSBUFHZ DPOTFRVFOUMZ XFDBOOPUFYQMPJUBOZPGUIFCFOFGJUTPGUIF"*CFGPSF UIF"*JTBDDFQUFE  5IFNFUIPEBTTVNFTBTJOHMF.-NPEFMDPOTFRVFOUMZ XFDBOOPU

    MFWFSBHFNBOZPUIFS"*T JOUIFDSPXE  5PHVBSBOUFFUIFRVBMJUZPGUBTLSFTVMUT JUOFFETUPDPNCJOFPUIFS NFUIPET FH TUBUJTUJDBMUFTUJOHXJUISBOEPNTBNQMFT  8FQSPQPTFUIFPSFUJDBMMZHVBSBOUFFEBMHPSJUINTCBTFEPOUIF EJWJEFBOEDPORVFSTUSBUFHZUPVUJMJ[FEJWFSTF"*XPSLFST 
  12. 1PJOUTPG5IJT5BML  2VBMJUZDPOUSPMPGUBTLSFTVMUTJOUIF)VNBO "*$SPXEXJMMCF BOJNQPSUBOUJTTVFJOUIFOFBSGVUVSF  &YJTUJOHUFDIOJRVFTBSFTVCPQUJNBMJOUFSNTPGFGGJDJFODZBOE TDBMBCJMJUZ  8FQSPQPTFUBTLBTTJHONFOUBMHPSJUINT

    XIJDIUBLF BEWBOUBHFPGUIFTXFFUTQPUTPGFBDICMBDLCPY"*XPSLFS  0VSNFUIPEDBOSFEVDFIVNBOXPSLFSBTTJHONFOUTXIJMF TBUJTGZJOHUIFRVBMJUZSFRVJSFNFOUJOUIFQSFTFODFPGNBOZCMBDL CPY"*XPSLFST 
  13. "*8PSLFSXJTF&WBMVBUJPO CBTFMJOF  5BTL$MVTUFSXJTF &WBMVBUJPO QSPQPTFE &WBMVBUJPOUBSHFU 5BTL "OPUIFS UBTLDMVTUFS

    5BTLDMVTUFS 2VBMJUZ 3FRVJSFNFOU "DDVSBDZ 3FKFDU "DDVSBDZ 3FKFDU "DDVSBDZ "DDFQU 5BTLDMVTUFSXJTFDBOBTTJHONPSFUBTLTUPNVMUJQMF"*T 0VS*EFB
  14. $MVTUFSXJTF5FTUCBTFE"TTJHONFOU $5" $5"FWBMVBUFTUBTLDMVTUFSTGSPN"*XPSLFSTVTJOHIVNBOXPSLFSMBCFMT  #FHJO "TTJHO VOMBCFMFEUBTLT UPIVNBOT 5SBJO"*XPSLFST CZIVNBOMBCFMT

    "MMUBTLT XFSF MBCFMFE "TTJHOUBTLTUP UIF"*XPSLFST &OE :FT /P 'PSFBDI UBTLDMVTUFS 4UBUJTUJDBMUFTU "DDFQU 3FKFDU DPNQMFUFEUBTLT IVNBOMBCFMT %JWJEFBOE$PORVFS "MMPS/PUIJOH 0OMZ)VNBOT 1SPQPTFE.FUIPE
  15. $MVTUFSXJTF5FTUCBTFE "TTJHONFOU $5" 5IFRVBMJUZPGUIFUBTLSFTVMUTGSPN"* XPSLFST TBUJTGJFTUIFRVBMJUZSFRVJSFNFOU  (MPCBM5FTUCBTFE "TTJHONFOU (5"

    5IFRVBMJUZPGUIFPWFSBMMUBTLSFTVMUT TBUJTGJFTUIFRVBMJUZSFRVJSFNFOU (5"BTTJHONPSFUBTLTUP"*XPSLFST &WBMVBUJPOUBSHFU 5BTL 5BTLDMVTUFS "DDFQUFE UBTLDMVTUFS T "TTJHOFEUP IVNBOXPSLFST 1SPQPTFE.FUIPE 1SPQPTFE.FUIPE .PSFPQUJNBM
  16. 1PJOUTPG5IJT5BML  2VBMJUZDPOUSPMPGUBTLSFTVMUTJOUIF)VNBO "*$SPXEXJMMCF BOJNQPSUBOUJTTVFJOUIFOFBSGVUVSF  &YJTUJOHUFDIOJRVFTBSFTVCPQUJNBMJOUFSNTPGFGGJDJFODZBOE TDBMBCJMJUZ  8FQSPQPTFUBTLBTTJHONFOUBMHPSJUINT

    XIJDIUBLFBEWBOUBHF PGUIFTXFFUTQPUTPGFBDICMBDLCPY"*XPSLFS  0VSNFUIPEDBOSFEVDFIVNBOXPSLFSBTTJHONFOUTXIJMF TBUJTGZJOHUIFRVBMJUZSFRVJSFNFOUJOUIFQSFTFODFPGNBOZ CMBDLCPY"*XPSLFST 
  17. &YQFSJNFOUVTJOH0QFO#FODINBSL ,./*45  5BTLT DMBTTDMBTTJGJDBUJPO GPSLJNBHFT ,./*45 )VNBO8PSLFST "OTXFSDPSSFDUMBCFM "*8PSLFST

    EJGGFSFOU .-NPEFMTGSPN TDJLJUMFBSO 5IFZXPSLJOQBSBMMFM 2VBMJUZ3FRVJSFNFOU     5BTL"TTJHONFOU"MHPSJUINT  8PSLFSXJTF3BOEPN4BNQMJOH5FTU 85"  "DUJWF-FBSOJOHCBTFE "-"   $5"BOE(5" 1FSGPSNBODF.FUSJDT UBTLBTTJHONFOUTUP"*XPSLFST      KXPDQODEHOV      FRPSOHWHGWDVNV :7$      KXPDQODEHOV      $/$      KXPDQODEHOV      &7$      KXPDQODEHOV      *7$ T  T  T  T  .-1$MBTTJGJFS &YUSB5SFF$MBTTJGJFS -PHJTUJD3FHSFTTJPO ,NFBOT %FDJTJPO5SFF$MBTTJGJFS 47$  ,/FJHICPST$MBTTJGJFS (BVTTJBO1SPDFTT$MBTTJGJFS  .VMUJOPNJBM/# "EB#PPTU$MBTTJGJFS 1BTTJWF"HHSFTTJWF$MBTTJGJFS 3JEHF$MBTTJGJFS 3JEHF$MBTTJGJFS$7 $PNQMFNFOU/# /FBSFTU$FOUSPJE
  18. 3FTVMUPG0QFO#FODINBSL ,./*45       KXPDQODEHOV 

        FRPSOHWHGWDVNV :7$      KXPDQODEHOV      $/$      KXPDQODEHOV      &7$      KXPDQODEHOV      *7$ T  T  T  T  • 'PSFBDIR BMMNFUIPETXPSLFEUPNFFUUIFRVBMJUZSFRVJSFNFOUBUMFBTU • $5"BOE(5"BTTJHOFEUBTLTUP"*XPSLFSTCZEJWJEFBOEDPORVFSTUSBUFHZ XIJMFTBUJTGZJOHUIFSFRVJSFNFOU • (5"PVUQFSGPSNFEPUIFSTBUUIFOVNCFSPGBTTJHOFEUBTLTUP"*XPSLFST %JWJEFBOE$PORVFS "MMPS/PUIJOH #FTU
  19. &YQFSJNFOUVTJOH3FBMXPSME%BUBTFUT  5BTLT %JTBTUFSEBNBHFBTTFTTNFOU GPSGMPPEFEBSFB UBTLT )VNBO8PSLFST "NB[PO.FDIBOJDBM5VSL "*8PSLFST "*8PSLFST

    5IFPOFJTEFWFMPQFECZBOPOZNPVT 5BTL"TTJHONFOU"MHPSJUINT  "DUJWF-FBSOJOHCBTFE "-"   (5" 1FSGPSNBODF.FUSJDT -BCFMFESFTVMUT
  20.  ˙ OPOGMPPEFE˙ GMPPEFE˙ DPWFSFECZDMPVET (5"GPVOEBHPPEUBTL DMVTUFS XIJDIJTGPSOPO GMPPEFEQBSUTBOE JNNFEJBUFMZBTTJHOFEUIF

    UBTLT %BSLDPMPSNFBOTUIFMBCFMGSPNIVNBOXPSLFST -JHIUDPMPSNFBOTUIFMBCFMGSPN"*XPSLFST )VNBOXPSLFSTXFSF BTTJHOFEUBTLTUIBUGPDVTFE PONPSFDPNQMFYBSFBT
  21. 4VNNBSZ • 8FJOUSPEVDFEBOPWFM)VNBO "*DSPXEUBTLBTTJHONFOUQSPCMFN )"$5"1 BOEQSPQPTFEUXPBMHPSJUINT $5" (5"  •

    0VSSFTVMUTTVHHFTUUIFQPTTJCJMJUZPGFGGJDJFOUUBTLQSPDFTTJOHCZ BQQSPQSJBUFMZTIBSJOHUBTLTCFUXFFO"*XPSLFSTBOEIVNBO XPSLFST  'VUVSF8PSL *NQSPWFBMHPSJUINTCZ  QBSBMMFMUBTLDMVTUFSTFWBMVBUJPO   QSVOJOHUBTLDMVTUFSTUIBUBSFVOMJLFMZUPCFBDDFQUFE