Pro Yearly is on sale from $80 to $50! »

Async, Persistent, Fast, and Sable "Enought" Queue/Worker Using Go and PostgreSQL

36e72b299b441378e41b6c445296b959?s=47 Akira Chiku
November 05, 2017

Async, Persistent, Fast, and Sable "Enought" Queue/Worker Using Go and PostgreSQL

36e72b299b441378e41b6c445296b959?s=128

Akira Chiku

November 05, 2017
Tweet

Transcript

  1. 1 "TZOD 1FSTJTUFOU 'BTU  BOE4UBCMF&OPVHI˒ 2VFVF8PSLFS 6TJOH(PBOE1PTUHSF42- (P$PO +1/PW

     "LJSB$IJLV
  2. NF 2 /BNF"LJSB$IJLV 5XJUUFS!@BDIJLV (JU)VC!BDIJLV 'JSF+VHHMFS (PPHMF4FBSDI"LJSB$IJLV'JSF

  3. BHFOEB 3 Ø CBDLHSPVOE QSPEVDUTJ[FUFBN Ø XPSLFSTFSWFSQBDLBHFTUSVDUVSF Ø XIZ1PTUHSF42-GPSRVFVFXPSLFSTZTUFN Ø

    RVFHPMJCSBSZJOUFSOBM Ø BQQMJDBUJPODPEF Ø PVUTJEFPGUSBOTBDUJPO FYUFSOBM"1*T Ø QSPEVDUJPOQFSGPSNBODFTUBUT
  4. 4 CBDLHSPVOE QSPEVDUTJ[FUFBN

  5. CBDLHSPVOE QSPEVDU 5 Ø QSPEVDU Ø "QQCBTFEJOTUBOU BOESFMPBEBCMFWJSUVBMQMBTUJD 7*4"QSFQBJEDBSEGPSFWFSZPOF Ø

    J04 "OESPJE 8FC Ø IUUQTWBOEMFKQ
  6. CBDLHSPVOE TJ[F 6 Ø TJ[F BTPG/PW  Ø PGFOEQPJOUT JOUFSOBM"1*

     Ø PGFYUFSOBMTFSWJDFT Ø OPUJODMVEJOH4FOUSZ/FX3FMJD4UBUVT1BHFJP Ø PGUBCMFT 1PTUHSF42-  Ø PGMJOFT (PDPEFXJUIPVUWFOEPS Ø "MM  Ø OPUFTUT  Ø OPHFOFSBUFEDPEF OPUFTUT 
  7. CBDLHSPVOE UFBN 7 NFNCFS EFTJHO GSPOUFOE CBDLFOE *OGSB "84 LPOPD

    NPRBEB ZPTVLF ZPTIJEB BDIJLV "TPG/PW  IJSPBLJT
  8. 8 XPSLFSTFSWFS QBDLBHFTUSVDUVSF

  9. QSPKFDUTUSVDUVSF 9 WBOEMFTFSWFS WBOEMFXPSLFS WBOEMFDMJ WBOEMF TFSWJDF NPEFM TFSWJDF TFSWJDF

    &YUFSOBM "1* DMJFOU
  10. XPSLFSTFSWFSQBDLBHFTUSVDUVSF 10  WBOEMFBQJ  NPEFM JOUFSBDUJPOXJUISECNT EBUBNPEFMJOUIFTFSWJDF OPUKVTUSECNT UBCMFSFQSFTFOUBUJPOT

     TFSWJDF EBUBNPEFMTJOUFSBDUXJUIFBDIPUIFS BMTP FYUFSOBM"1*TBSFDBMMFEJOTFSWJDFT  JBQJ SFRVFTUSFTQPOTFTUSVDUT SFRVFTUWBMJEBUJPOGVODUJPOT  IBOEMFST KPCT TFSWFS DPOG FUD  FYTSWDMJFOU  FYTSWDMJFOU  DNE WBOEMFTFSWFS WBOEMFXPSLFS WBOEMFDMJ FYTSWNPDLTFSWFS FYTSWNPDLTFSWFS  ANPEFMAQLH GPS3%#.4 SFMBUFEJOUFSBDUJPOT ATFSWJDFAQLH GPSNPEFMT BOEFYU"1*JOUFSBDUJPOT AWBOEMFAQLH GPSIUUQ IBOEMFSTIUUQ TFSWFSXPSLFSTBTZODKPCT 6TFADNEAUPDSFBUF EJGGFSFOUUZQFTPGCJOBSJFT
  11. 11 1PTUHSF42- GPSRVFVFXPSLFS

  12. 1PTUHSF42-GPSRVFVFXPSLFS 12 Ø CBTFMJOFJT DPOTJTUFODZPWFSQFSGPSNBODF Ø XFBSFOPUCVJMEJOH35# OPS)'5TZTUFN Ø USBOTBDUJPOJTBXFTPNF

    EJTDVTTFEMBUFS Ø XFEJEOUXBOUUPBEEBOPUIFSNPWJOHQBSU Ø 3BCJUU.2;FSP.2"84424˘ Ø POMZPOFGVMMUJNFCBDLFOEEFW BDIJLV BUUIBUNPNFOU Ø XFBSFVTJOH1PTUHSF42-BTPVSQSJNBSZEBUBTUPSF BOE(P BTNBJOMBOHVBHF Ø RVFVFXPSLFSMJCSBSZDBMMFERVFHP XIJDIXJMMCFEJTDVTTFE MBUFS JTXSJUUFOJO(P 42- 1PTUHSF42-
  13. 13 RVFHPMJCSBSZJOUFSOBM

  14. 2VFBOERVFHP 14 Ø IUUQTHJUIVCDPNDIBOLTRVF Ø 2VF LFɪ PSLBZ Ø ˑ2VFJTBIJHIQFSGPSNBODFBMUFSOBUJWFUP%FMBZFE+PC

    PS 2VFVF$MBTTJD UIBUJNQSPWFTUIFSFMJBCJMJUZPGZPVS BQQMJDBUJPOCZQSPUFDUJOHZPVSKPCTXJUIUIFTBNF "$*% HVBSBOUFFT BTUIFSFTUPGZPVSEBUB˒ Ø IUUQTHJUIVCDPNCHFOUSZRVFHP Ø ˑRVFHPJTBGVMMZJOUFSPQFSBCMF(PMBOH QPSUPG $ISJT )BOLT 3VCZ2VFRVFVJOHMJCSBSZ GPS1PTUHSF42-2VFVTFT 1PTUHSF42-TBEWJTPSZMPDLTGPSTQFFEBOESFMJBCJMJUZ˒
  15. RVFHPPWFSWJFX 15 "1* 4FSWFS 8PSLFS 4FSWFS RVF@KPCT   

    "1*TFSWFSNPEJGJFTDVSSFOU EBUBXIFOBVTFSSFRVFTUT • .PEJGZSFRVFTUFEEBUB • &ORVFVFBKPCGPSGVSUIFSXPSL • %PBCPWFUXPTUFQTJOPOF USBOTBDUJPO  "XPSLFSEFRVFVFTBKPC  BOEQSPDFTTFTJU • -PDLBKPCUPQSFWFOUJUGSPN CFJOHFYFDVUFENVMUJQMFUJNFTCZ PUIFSXPSLFST • 8PSLPOUIFUBTL • $PNNJUJGTVDDFFEFE SPMMCBDL NPEJGJDBUJPOTJGGBJMFE 8PSLFS 1PPM 8PSLFS 1PPM
  16. RVFHPFORVFVF 16 Ø *UJTKVTUJOTFSUJOHBSPXUPBUBCMFOBNFE RVF@KPCT Ø 2VFVF Ø RVF@KPCTUBCMFIBTARVFVFADPMVNO Ø

    EJGGFSFOUXPSLFSQPPMTGPSEJGGFSFOURVFVFT Ø 1SJPSJUZ Ø TNBMMJOU SFQSFTFOUTQSJPSJUZXJUIJOUIF TBNFRVFVF Ø "SHT Ø +40/UZQFDPMVNO Ø 5IJTBMMPXT"1*TFSWFSUPFORVFVFBKPCJOB USBOTBDUJPOBMPOHXJUIPUIFSEBUBCBTF DIBOHFT Ø "KPCDBOˏUCFTFFOGSPNXPSLFSTCFGPSF UIFGJOBMDPNNJU "1* 4FSWFS RVF@KPCT
  17. RVFHPEFRVFVF 17 Ø &BDIXPSLFSTJOBXPSLFSQPPMBSFRVFSZJOHUP GJOEBKPCUPXPSLPO Ø 'JSTUJO'JSTUPVU MPPLJOHBUSVO@BU DPMVNO Ø

    -PDLBKPCUPQSFWFOUGSPNPUIFSXPSLFST HSBCJOH UIFTBNFKPC Ø "EWJTPSZMPDLXJUISFDVSTJWF$5&JTVTFEUP MPDLBKPCXJUIPVUCMPDLJOHPUIFSXPSLFST Ø 8PSLPOUIFKPC Ø *GJUTVDDFFEFE EFMFUFBKPCGSPNRVF@KPCT UBCMF Ø *GJUGBJMFE JODSFNFOUFSSPS@DPVOU VQEBUF SVO@BU GPSUIFOFYUSFUSZ BOESFDPSEFSSPS NFTTBHFJORVF@KPCTUBCMF 8PSLFS 4FSWFS 8PSLFS 1PPM 8PSLFS 1PPM RVF@KPCT
  18. RVFHPBEWJTPSZMPDL  18 Ø 2VFVFTZTUFNOFFETTPNFTPSUPGMPDLJOHNFDIBOJTNUPSVO XPSLFSTDPODVSSFOUMZ Ø "EWJTPSZ-PDL Ø 4JODF1PTUHSF42-

    SFMFBTFEJO Ø *UˏTBQSFUUZNBUVBSFGVODUJPO Ø ˑ1PTUHSF42- QSPWJEFTBNFBOTGPSDSFBUJOHMPDLTUIBUIBWF BQQMJDBUJPOEFGJOFENFBOJOHT5IFTFBSFDBMMFE BEWJTPSZ MPDLT CFDBVTFUIFTZTUFNEPFTOPUFOGPSDFUIFJSVTFˋ JUJT VQUPUIFBQQMJDBUJPOUPVTFUIFNDPSSFDUMZ˒
  19. RVFHPBEWJTPSZMPDL  19 Ø *UEPFTOˏUXSJUFUPUIFEJTL Ø -PDLTBSFTUPSFEJOBTIBSFENFNPSZQPPM Ø #FUUFSQFSGPSNBODFUIBONBOBHJOHKPCTUBUVTUBCMF Ø

    #FODINBSL Ø IUUQKPIUPQHCMPHTQPUKQRVFVFTRVFVFT UIFZBMMGBMMEPXOIUNM Ø *UEPFTOˏUCMPDLPUIFSRVFSJFT Ø 6OMJLFSPXMPDLJOHˑTFMFDU˘GPSVQEBUF˒ Ø 4FTTJPO"4&-&$5QH@BEWJTPSZ@MPDL  SFUVSOUSVF Ø 4FTTJPO#4&-&$5QH@BEWJTPSZ@MPDL  SFUVSOGBMTF Ø "QQMJDBUJPOOFFETUPIBOEMFUIJT#PPMFBOWBMVF
  20. RVFHPXPSLFSXPSLFSQPPM  20 // WorkerPool is a pool of Workers,

    // each working jobs from the queue Queue // at the specified Interval using the WorkMap. type WorkerPool struct { WorkMap WorkMap Interval time.Duration Queue string c *Client workers []*Worker mu sync.Mutex done bool }
  21. RVFHPXPSLFSXPSLFSQPPM  21 // WorkFunc is a function that performs

    a Job. // If an error is returned, the job // is reenqueued with exponential backoff. type WorkFunc func(j *Job) error // WorkMap is a map of Job names to WorkFuncs // that are used to perform Jobs of a // given type. type WorkMap map[string]WorkFunc
  22. RVFHPXPSLFSXPSLFSQPPM  22 // Start starts all of the Workers

    in the WorkerPool. func (w *WorkerPool) Start() { w.mu.Lock() defer w.mu.Unlock() for i := range w.workers { w.workers[i] = NewWorker(w.c, w.WorkMap) w.workers[i].Interval = w.Interval w.workers[i].Queue = w.Queue go w.workers[i].Work() } }
  23. RVFHPXPSLFSXPSLFSQPPM  23 // Worker is a single worker that

    pulls jobs off the specified Queue. If no Job // is found, the Worker will sleep for Interval seconds. type Worker struct { // Interval is the amount of time that this Worker should sleep before trying // to find another Job. Interval time.Duration // Queue is the name of the queue to pull Jobs off of. The default value, "", // is usable and is the default for both que-go and the ruby que library. Queue string c *Client m WorkMap mu sync.Mutex done bool ch chan struct{} }
  24. RVFHPXPSLFSXPSLFSQPPM  24 // Work pulls jobs off the Worker's

    Queue at its Interval. This function only // returns after Shutdown() is called, so it should be run in its own goroutine. func (w *Worker) Work() { for { select { case <-w.ch: log.Println("worker done") return case <-time.After(w.Interval): for { if didWork := w.WorkOne(); !didWork { break // didn't do any work, go back to sleep } } } } }
  25. RVFHPESBXCBDLT  25 Ø 4FWFSFQFSGPSNBODFEFHSBEBUJPOVOEFSNBTTJWFFORVFVF BOE MPOHSVOOJOHKPCT Ø IUUQTCSBOEVSPSHQPTUHSFTRVFVFT Ø

    4UBUFEQMBJOMZ PVSSPPUQSPCMFNJTUIBUUIFKPCUBCMFT JOEFYIBTCFDPNFMFTTVTFGVMUPUIFQPJOUXIFSFVTJOHJU JTOUNVDIGBTUFSUIBOBGVMMTFRVFOUJBMTDBO Ø IUUQTHJUIVCDPNCSBOEVSRVFEFHSBEBUJPOUFTU Ø MPOHSVOOJOHKPC TMFFQTFDBCPVUNJO Ø BCPVUKPCFORVFVFE QFSTFD Ø BGUFSNJO  KPCTJOUIFRVFUBCMF Ø TJHOJGJDBOUMPDLJOHQFSGPSNBODFEFHSBEBUJPO
  26. RVFHPESBXCBDLT  26 Ø BEWJTPSZMPDLJTEBUBCBTFXJEF BOEUIFLFZIBTUPCF JOUJOU Ø SFMBUJWFMZDPNQMFY8*5)3&$634*7&RVFSZUPMPDLJE Ø

    IUUQTCSBOEVSPSHQPTUHSFTRVFVFTMPDLJOHBMHPSJUINT Ø TLJQMPDLFENJHIUQFSGPSNCFUUFSXJUITJNQMFSJNQMFNFOUBUJPO Ø GSPN1PTUHSF42- Ø IUUQTCMPHOERVBESBOUDPNXIBUJTTFMFDUTLJQMPDLFE GPSJOQPTUHSFTRM Ø 5IJTNJHIUDPNQMJDBUFFSSPSIBOEMJOH BOEFYQ CBDLPGG SFUSZ BCJU CVUXPSUIBUSZ Ø 5IPTFBCPWFBSFXIZ*TUBUFEˑFOPVHI˒JOUIFUJUMFPGUIJTTMJEF
  27. 27 BQQMJDBUJPODPEF

  28. BQQMJDBUJPODPEF 28 Ø GVOD RVF+PC FSSPS Ø 5IJTJTUIFTJHOBUVSFXFIBWFUPXSJUF Ø 'BJSMZTJNQMF

    Ø RVF+PC DPOUBJOTEBUBCBTFQPPM Ø 5IJTJTVTFEBMMEBUBCBTFPQFSBUJPOTJOKPCQSPDFTT Ø RVF+PC BMTPIBT"SHT XIJDIDPSSFTQPOETUPBSHT DPMVNOPG RVF@KPCTUBCMF Ø +40/UZQFDPMVNO Ø 6ONBSTIBMM "SHT <>CZUFJOUPQSFEFGJOFETUSVDU Ø *UJTWFSZVTFGVMUPIBWFKPCHMPCBMWBMVFT FH"1*DMJFOU  BQQMJDBUJPODPOGJH JOBTUSVDU Ø -FUUIBUTUSVDU IBWFKPCNFUIPETTBUJTGJFTˑGVOD RVF+PC  FSSPS˒TJHOBUVSF
  29. BQQMJDBUJPODPEF  29 // BaseApp base app global values type

    BaseApp struct { Config *Config EstcClient *estc.Client } // JobApp job global values type JobApp struct { BaseApp Logger *log.Logger } IUUQTHJUIVCDPNBDIJLVHPUPEPJUCMPCNBTUFSKPCHP
  30. BQQMJDBUJPODPEF  30 IUUQTHJUIVCDPNBDIJLVHPUPEPJUCMPCNBTUFSKPCHP // UpdateUserInfo update user info job

    func (app *JobApp) UpdateUserInfo(j *qg.Job) error { var args UpdateUserInfoArgs if err := json.Unmarshal(j.Args, &args); err != nil { return errors.Wrap(err, "failed to unmarshal UpdateTodoETCArgs") } dbReq := &model.TodoUser{ UUID: args.UserID, Username: args.Username, Email: args.Email, Status: args.Status, } if err := service.UpdateUser(j.Tx(), dbReq); err != nil { return errors.Wrap(err, "service.UpdateUser failed") } ctx := app.Context() req := &estc.UpdateUserRequest{ ID: args.UserID, UserName: args.Username, Email: args.Email, } u, err := app.EstcClient.UpdateUser(ctx, req) if err != nil { return errors.Wrap(err, "EstcClient.UpdateUser failed") } app.Logger.Printf("userID=%s,email=%s", u.ID, u.Email) return nil }
  31. BQQMJDBUJPODPEF  31 IUUQTHJUIVCDPNBDIJLVHPUPEPJUCMPCNBTUFSXPSLFSHP jobs := JobApp{ BaseApp: BaseApp{ Config:

    appCfg, }, } wm := qg.WorkMap{ "updateUserJob": jobs.UpdateUserInfo, } wPoolLow := qg.NewWorkerPool(qc, wm, appCfg.NumWorkers) wPoolLow.Queue = QueNameLowPriority wPoolHigh := qg.NewWorkerPool(qc, wm, appCfg.NumWorkers) wPoolHigh.Queue = QueNameHighPriority
  32. BQQMJDBUJPODPEFUFTU  32 IUUQTHJUIVCDPNBDIJLVHPUPEPJUCMPCNBTUFSKPC@UFTUHP func TestUpdateUserInfo(t *testing.T) { app, tx,

    _, cleanup := testSetupJobApp(t) defer cleanup() u := model.TestCreateUserData(t, tx, &model.TodoUser{}) args := UpdateUserInfoArgs{ UserID: u.UUID, Email: fmt.Sprintf("%s@example.com", u.UUID), Status: "inactive", Username: "updated-username", } jsonArgs, err := json.Marshal(args) if err != nil { t.Fatal(err) } ts := httptest.NewServer(estc.TestNewMux(estc.DefaultHandlerMap)) defer ts.Close() app.Config.EstcConfig.BaseEndpoint = ts.URL j := qg.TestInjectJobTx(&qg.Job{Args: jsonArgs}, tx) if err := app.UpdateUserInfo(j); err != nil { t.Fatal(err) } }
  33. 33 PVUTJEFPGUSBOTBDUJPO FYUFSOBM"1*TFSWJDFT

  34. 34 "1* 4FSWFS 8PSLFS 4FSWFS RVF@KPCT 8PSLFS 1PPM 8PSLFS 1PPM

    PVUTJEFPGUSBOTBDUJPO &YUFSOBM "1* PVUTJEFPG USBOTBDUJPO
  35. 35 &YUFSOBM"1*TFSWJDFT

  36. 36 'BJM

  37. CFQSFQBSFEGPS"1*GBJMVSFT 37 Ø ,FFQJUˑKPCUSBOTBDUJPO˒ Ø :PVEPOˏUIBWFUPQBZFYUSBNFOUBMDPTUGPSEFTJHOJOHUIF CFTUSFUSZQSPDFTT Ø *GZPVVTFNVMUJQMFUSBOTBDUJPOTJOPOFKPC ZPVIBWFUP

    UIJOLUISPVHIIPXZPVDBOHVBSEUIFEBUBJOUIFGJSTUUY  XIFOUIFTFDPOEUY GBJMT Ø 8IFOJUJTSFBMMZOFFEFEUPIBWFNVMUJQMFUSBOTBDUJPOT UIFO DPOTJEFSUPEJWJEFBKPCJOUPUXPKPCT Ø 5IFGJSTUKPCFORVFVFUIFTFDPOEKPC Ø 8IFOFYUFSOBM"1*BDDFTTJTSFRVJSFE LFFQBTLJOH˒XIBUJGUIJT GBJMT DPVMEXFQSPUFDUPVSEBUBJOUFHSJUZ ˒
  38. +PCDBUFHPSJFT 38 Ø $6% ˘$SFBUF6QEBUF%FMFUF Ø 3 ˘3FBE 3%#.4 &YUFSOBM"1*

    OB 3 $6% OB 3 $6% Ø $BUFHPSZ Ø 3%#.4$36% BOESFBE FYUFSOBM"1* Ø $BUFHPSZ Ø 3%#.4$36% BOE$6% FYUFSOBM"1*
  39. $BUFHPSZ 39 3%#.4 &YUFSOBM"1* OB 3 $6% OB 3 $6%

    Ø $BUFHPSZKPCDBOCFDPNQMFUFEXJUIJO3%#.4 Ø +PCJTQSPUFDUFECZUSBOTBDUJPO Ø 4VDDFTTPSGBJMVSFJTBUPNJD Ø *GKPCGBJMTCZ3%#.4PQFSBUJPOPSSFBEFYUFSOBM"1*T KVTU SPMMCBDLBMMUIFEBUBDIBOHFT BOESFUSZ
  40. $BUFHPSZ 40 3%#.4 &YUFSOBM"1* OB 3 $6% OB 3 $6%

    Ø $BMMFYUFSOBM"1*TUPDIBOHFUIFJSEBUB BOEBMTPNPEJGZEBUBJO 3%#.4 Ø %BUBDIBOHFTJO3%#.4DBOCFSPMMFCBDLFE JGUIFQSPDFTT GBJMFE CVUEBUBNPEJGJDBUJPOTUISPVHIFYUFSOBM"1*TDBOˏU CFDBODFMFE Ø *UNJHIUCFQPTTJCMFUPSFTUPSFFYUFSOBMEBUBCBDLUPQSFWJPVT TUBUFJOFYDFQUJPOIBOEMJOH CVUJUXJMMNBLFTJNQMFUIJOHT DPNQMJDBUFE Ø 5SZOPUUPDBMMUIFTBNF"1*NVMUJQMFUJNFTFWFOXIFOJUIBTUP SFUSZ Ø 5SZUPDBMMFYUFSOBM"1*BUUIFMBTUQBSUPGUIFQSPDFTT Ø 5SZOPUUPVTFNVMUJQMFFYUFSOBM"1*TJOKPC Ø USBOTBDUJPO KPC BUNPTUFYUFSOBM$6%"1*
  41. IPXNVDIJTˑFOPVHI˒ 41 Ø "TZPVDBOTFF UIFSFJTOPQFSGFDUCFBVUJGVMTJNQMFTPMVUJPOGPS FYUFSOBM"1*TFSWJDFFSSPSIBOEMJOH LFFQJOHBMMUIFEBUB JOUFHSJUZBOEQFSGPSNBODF˘ Ø *O+BQBOFTF

    XFDBMMUIJTTJUVBUJPOˑװ׏גְֻ׃ַזְ˒
  42. 42 װ׏גְֹתׅ