How should we manage experiments?

7f7abff13e7c5ad15bfa3a823313ef9d?s=47 chie8842
April 12, 2018

How should we manage experiments?

7f7abff13e7c5ad15bfa3a823313ef9d?s=128

chie8842

April 12, 2018
Tweet

Transcript

  1. )PXTIPVMEXFNBOBHFFYQFSJNFOUT  .-LJUDIFO $IJF)BZBTIJEB

  2. $IJF)BZBTIJEB !DIJF  •  .BDIJOF-FBSOJOHBOE*OGSBTUSVDUVSF&OHJOFFSBU $PPLQBE*OD •  $POUSJCVUPSPG"QBDIF4QBSL )BEPPQ BOETPPO

    •  (SBEVBUFE0TBLB6OJWBOE/"*45 –  &OHBHFEJO7PJDFDPOWFSTJPOSFTFBSDI *DPOUSJCVUFEBSUJDMFTBCPVUTDJFOUJGJD DPNQVUJOHXJUI1ZUIPO *UbTPOTBMF"QSJM
  3. 5IJOHT*XBOUUPEP •  *OBEEJUJPOUPXPSLPONBDIJOFMFBSOJOH *`EMJLFUP JNQSPWFUIFNBDIJOFMFBSOJOHFYQFSJNFOU EFQMPZNFOU  BOENPOJUPSJOHQSPDFTTGSPNUIFWJFXQPJOUPGTPGUXBSF FOHJOFFSJOH

  4. 1BTUXPSLFBTZ@OPUJGJFS •  4JNQMFEFDPSBUPSGVODUJPOUPOPUJGZPOTMBDL NBJM PS .BDEFTLUPQXIFOZPVSTDSJQUGJOJTIFE –  IUUQTHJUIVCDPNDIJFFBTZ@OPUJGJFS But Hyperdash

    is very convenient
  5. 5IFMJGFDZDMFPGUIFNBDIJOFMFBSOJOHFYQFSJNFOU •  8IFOXFXPSLPONBDIJOFMFBSOJOH XFMPPQGPMMPXJOH TUFQT –  %FDJEFFWBMVBUJPOJOEFY –  &YFDVUFFYQFSJNFOUT • 

    'FBUVSF&OHJOFFSJOH •  5SBJONPEFM •  &WBMVBUFNPEFM •  $IBOHFDPOEJUJPOT –  4VNNBSJ[FBOEWJTVBMJ[FUIFSFTVMUPGFYQFSJFODFTGPSSFQPSUTPS QBQFST Loop many times
  6. "WBSJFUZPGQBUUFSOT •  $SFBUFGFBUVSFEBUBTFUGSPNIVHFEBUB –  )PXNVDIEBUBEPXFVTF  –  8IBULJOEPGEBUBEPXFVTF  BDUJPOMPH

    VTFSBOEJUFNDPOUFYUT JNBHFT FUD  –  )PXUPJNQMFNFOUGFBUVSFFOHJOFFSJOH /PSNBMJ[BUJPO .JTTJOH WBMVFQSPDFTTJOH FUD  •  5SZBWBSJFUZPGBMHPSJTNT •  1BSBNFUFS5VOJOH –  &HCBUDITJ[F MFBSOJOHSBUF FUD
  7. %JGGJDVMUZJONBOBHJOHFYQFSJNFOUT •  8FIBWFUPNBOBHFTPNBOZJOGPSNBUJPO –  %P[FOTPGFYQFSJNFOUT –  .BOZQBSBNFUFST –  TPVSDFDPEFWFSTJPO – 

    &OWJSPONFOUBM*OGPSNBUJPO •  3FTPVSDFTJ[F •  6TF(16  •  'SBNFXPSLWFSTJPOT •  &YQFSJNFOUQJQFMJOFPGUFODPOUBJOTTFWFSBMOVNCFSPG TUFQT
  8. *NQPSUBOUUIJOHTGPSFYQFSJNFOUNBOBHFNFOU •  4BWFBMMOFDFTTBSZJOGPSNBUJPOFGGJDJFOUMZ •  #FBCMFUPTFBSDIBOEBDDFTTQBTUJOGPSNBUJPOFBTJMZ •  #FBCMFUPTVNNBSJ[FTFWFSBMFYQFSJNFOUTFBTJMZ 

  9. &YJTUJOHXPSLT •  4BDSFE IUUQTHJUIVCDPN*%4*"TBDSFE  •  '(-BC IUUQTHJUIVCDPN,BJYIJO'(-BC  • 

    5IFZBSFUPPMTUPNBOBHFFYQFSJNFOUMPHT •  5IFZVTF.POHP%#GPSEBUBTUPDLFSBOEQSPWJEFTUIF SJDI6* •  *U`TWFSZDPOWFOJFOUGPSTDBMBCMFFYQFSJNFOUT –  4IBSFFYQFSJNFOUTXJUI.VMUJQMFFOWJSPONFOUT NVMUJQMF SFTFBSDIFS FUD •  #VUUIFZBSFPWFSTQFDGPSNFʜ –  /POFDFTTBSZPGSJDIQMBUGPSN –  *EPO`UXBOUUPNBJOUBJOB.POHP%# –  *EPO`UMJLFUPDIBOHFNZTDSJQUTGPSMPHNBOBHFNFOU
  10. &YJTUJOHXPSLT •  $PNFUNM IUUQTXXXDPNFUNM  –  $MPVE4FSWJDFGPSMPHUSBDLJOH –  4IBSFMPHT NFNPSBOEVNGVODUJPO

    FBTZUPVTF –  -PPLTHPPE #VUʜ –  8FDBO`UVQMPBEBEEJUJPOBMGJMFTTVDIBTUSBJOFENPEFMTPSEBUB –  /PDPMMBCPSBUJPOXJUIHJU –  8FIBWFUPUBLFDBSFOPUUPVQMPBEDPOGJEFOUJBMJOGPSNBUJPO
  11. FYQTUPDL •  *DSFBUFEBUPPMXIJDITPMWFTUIFDPNQMFYJUZPGUIF FYQFSJNFOUNBOBHFNFOU –  IUUQTHJUIVCDPNDIJFFYQTUPDL

  12. $PODFQUPGFYQTUPDL •  *UIBTWFSZTJNQMFBSDIJUFDUVSF –  /PEFQFOEFODZ –  /PNBJOUFOBODF •  8FEPO`UOFFEUPDIBOHFPVSTDSJQUT – 

    0OMZBEEJOHUXPPSUISFFDPNNBOET •  *UIBTHPPEMPH"DDFTTJCJMJUZXJUI7JTVBMJ[BUJPO –  8FDBODPNQBSFQBTUFYQFSJNFOUT –  8FDBOBDDFTTFBDIMPHGSPNBCSPXTFS
  13. 6TBHFPOGVODUJPOBMJ[FETDSJQUT *GZPVBMSFBEZIBWFGVODUJPOBMJ[FEUBSHFUQSPDFTT ZPVDBO VTFUIFGPMMPXJOHEFDPSBUPS from expstock import expstock a=1 b=2

    e = expstock.ExpStock(exp_name=‘test_experiment’) @expstock.expstock(e) def run(a, b): return a + b e.append_param(a=a, b=b) run(a, b)
  14. 6TBHFXJUI+VQZUFS/PUFCPPL 8JUIJOUFSBDUJWFTDSJQUTFHFYQFSJNFOUTPO+VQZUFS /PUFCPPL ZPVDBOVTFUIFGPMMPXJOHDPNNBOE from expstock import expstock a=1 b=2

    e = expstock.ExpStock(exp_name=‘test_experiment’) e.append_param(a=a, b=b) e.pre_stock() a + b e.post_stock()
  15. %JSFDUPSZ4USVDUVSF -PHTBSFTUPDLFEBUFBDITVCEJSFDUPSZ experiments ᵓᴷᴷ <yyyymmdd_hhmmss>_<experiment_name> ᴹ ᵓᴷᴷ exec_time.txt ᴹ ᵓᴷᴷ

    git_diff.txt ᴹ ᵓᴷᴷ git_head.txt ᴹ ᵓᴷᴷ machine_info.txt ᴹ ᵓᴷᴷ memo.txt ᴹ ᵓᴷᴷ params.txt ᴹ ᵓᴷᴷ report.txt ᴹ ᵓᴷᴷ result.txt ᴹ ᵓᴷᴷ stderr.txt ᴹ ᵋᴷᴷ stdout.txt
  16. 4BWFBEEJUJPOBMGJMFT •  :PVDBOTBWFBEEJUJPOBMGJMFTBUTBNFEJSFDUPSZ # save keras model from keras.models import

    load_model model_file = os.path.join(e.log_dirname, ‘model.h5’) model.save(model_file) # save matplotlib firure plt_png = os.path.join(e.log_dirname, ‘plt.png’) plt.savefig(plt_png)
  17. FYQTUPDLTFSWFS •  &YQTUPDLIBTBTJNQMF6*JNQMFNFOUBUJPOXJUICPUUMF BOE42-JUF

  18. 'VUVSFXPSL •  *ODSFBTFMPHNFUSJDT –  .BDIJOF3FTPVSDFVUJMJ[BUJPO –  'SBNFXPSLWFSTJPO •  'VODUJPOUPFEJUNFUSJDTPO6* • 

    FUD