Share
Embed
Async, Persistent, Fast, and Stable "Enought" Queue/Worker Using Go and PostgreSQL
Akira Chiku
November 05, 2017
Transcript
42 slides
-
BHFOEB 3 Ø CBDLHSPVOE QSPEVDUTJ[FUFBN Ø XPSLFSTFSWFSQBDLBHFTUSVDUVSF Ø XIZ1PTUHSF42-GPSRVFVFXPSLFSTZTUFN Ø
RVFHPMJCSBSZJOUFSOBM Ø BQQMJDBUJPODPEF Ø PVUTJEFPGUSBOTBDUJPO FYUFSOBM"1*T Ø QSPEVDUJPOQFSGPSNBODFTUBUT -
CBDLHSPVOE QSPEVDU 5 Ø QSPEVDU Ø "QQCBTFEJOTUBOU BOESFMPBEBCMFWJSUVBMQMBTUJD 7*4"QSFQBJEDBSEGPSFWFSZPOF Ø
J04 "OESPJE 8FC Ø IUUQTWBOEMFKQ -
CBDLHSPVOE TJ[F 6 Ø TJ[F BTPG/PW Ø PGFOEQPJOUT JOUFSOBM"1*
Ø PGFYUFSOBMTFSWJDFT Ø OPUJODMVEJOH4FOUSZ/FX3FMJD4UBUVT1BHFJP Ø PGUBCMFT 1PTUHSF42- Ø PGMJOFT (PDPEFXJUIPVUWFOEPS Ø "MM Ø OPUFTUT Ø OPHFOFSBUFEDPEF OPUFTUT -
CBDLHSPVOE UFBN 7 NFNCFS EFTJHO GSPOUFOE CBDLFOE *OGSB "84 LPOPD
NPRBEB ZPTVLF ZPTIJEB BDIJLV "TPG/PW IJSPBLJT -
XPSLFSTFSWFSQBDLBHFTUSVDUVSF 10 WBOEMFBQJ NPEFM JOUFSBDUJPOXJUISECNT EBUBNPEFMJOUIFTFSWJDF OPUKVTUSECNT UBCMFSFQSFTFOUBUJPOT
TFSWJDF EBUBNPEFMTJOUFSBDUXJUIFBDIPUIFS BMTP FYUFSOBM"1*TBSFDBMMFEJOTFSWJDFT JBQJ SFRVFTUSFTQPOTFTUSVDUT SFRVFTUWBMJEBUJPOGVODUJPOT IBOEMFST KPCT TFSWFS DPOG FUD FYTSWDMJFOU FYTSWDMJFOU DNE WBOEMFTFSWFS WBOEMFXPSLFS WBOEMFDMJ FYTSWNPDLTFSWFS FYTSWNPDLTFSWFS ANPEFMAQLH GPS3%#.4 SFMBUFEJOUFSBDUJPOT ATFSWJDFAQLH GPSNPEFMT BOEFYU"1*JOUFSBDUJPOT AWBOEMFAQLH GPSIUUQ IBOEMFSTIUUQ TFSWFSXPSLFSTBTZODKPCT 6TFADNEAUPDSFBUF EJGGFSFOUUZQFTPGCJOBSJFT -
1PTUHSF42-GPSRVFVFXPSLFS 12 Ø CBTFMJOFJT DPOTJTUFODZPWFSQFSGPSNBODF Ø XFBSFOPUCVJMEJOH35# OPS)'5TZTUFN Ø USBOTBDUJPOJTBXFTPNF
EJTDVTTFEMBUFS Ø XFEJEOUXBOUUPBEEBOPUIFSNPWJOHQBSU Ø 3BCJUU.2;FSP.2"84424˘ Ø POMZPOFGVMMUJNFCBDLFOEEFW BDIJLV BUUIBUNPNFOU Ø XFBSFVTJOH1PTUHSF42-BTPVSQSJNBSZEBUBTUPSF BOE(P BTNBJOMBOHVBHF Ø RVFVFXPSLFSMJCSBSZDBMMFERVFHP XIJDIXJMMCFEJTDVTTFE MBUFS JTXSJUUFOJO(P 42- 1PTUHSF42- -
2VFBOERVFHP 14 Ø IUUQTHJUIVCDPNDIBOLTRVF Ø 2VF LFɪ PSLBZ Ø ˑ2VFJTBIJHIQFSGPSNBODFBMUFSOBUJWFUP%FMBZFE+PC
PS 2VFVF$MBTTJD UIBUJNQSPWFTUIFSFMJBCJMJUZPGZPVS BQQMJDBUJPOCZQSPUFDUJOHZPVSKPCTXJUIUIFTBNF "$*% HVBSBOUFFT BTUIFSFTUPGZPVSEBUB˒ Ø IUUQTHJUIVCDPNCHFOUSZRVFHP Ø ˑRVFHPJTBGVMMZJOUFSPQFSBCMF(PMBOH QPSUPG $ISJT )BOLT 3VCZ2VFRVFVJOHMJCSBSZ GPS1PTUHSF42-2VFVTFT 1PTUHSF42-TBEWJTPSZMPDLTGPSTQFFEBOESFMJBCJMJUZ˒ -
RVFHPPWFSWJFX 15 "1* 4FSWFS 8PSLFS 4FSWFS RVF@KPCT
"1*TFSWFSNPEJGJFTDVSSFOU EBUBXIFOBVTFSSFRVFTUT • .PEJGZSFRVFTUFEEBUB • &ORVFVFBKPCGPSGVSUIFSXPSL • %PBCPWFUXPTUFQTJOPOF USBOTBDUJPO "XPSLFSEFRVFVFTBKPC BOEQSPDFTTFTJU • -PDLBKPCUPQSFWFOUJUGSPN CFJOHFYFDVUFENVMUJQMFUJNFTCZ PUIFSXPSLFST • 8PSLPOUIFUBTL • $PNNJUJGTVDDFFEFE SPMMCBDL NPEJGJDBUJPOTJGGBJMFE 8PSLFS 1PPM 8PSLFS 1PPM -
RVFHPFORVFVF 16 Ø *UJTKVTUJOTFSUJOHBSPXUPBUBCMFOBNFE RVF@KPCT Ø 2VFVF Ø RVF@KPCTUBCMFIBTARVFVFADPMVNO Ø
EJGGFSFOUXPSLFSQPPMTGPSEJGGFSFOURVFVFT Ø 1SJPSJUZ Ø TNBMMJOU SFQSFTFOUTQSJPSJUZXJUIJOUIF TBNFRVFVF Ø "SHT Ø +40/UZQFDPMVNO Ø 5IJTBMMPXT"1*TFSWFSUPFORVFVFBKPCJOB USBOTBDUJPOBMPOHXJUIPUIFSEBUBCBTF DIBOHFT Ø "KPCDBOˏUCFTFFOGSPNXPSLFSTCFGPSF UIFGJOBMDPNNJU "1* 4FSWFS RVF@KPCT -
RVFHPEFRVFVF 17 Ø &BDIXPSLFSTJOBXPSLFSQPPMBSFRVFSZJOHUP GJOEBKPCUPXPSLPO Ø 'JSTUJO'JSTUPVU MPPLJOHBUSVO@BU DPMVNO Ø
-PDLBKPCUPQSFWFOUGSPNPUIFSXPSLFST HSBCJOH UIFTBNFKPC Ø "EWJTPSZMPDLXJUISFDVSTJWF$5&JTVTFEUP MPDLBKPCXJUIPVUCMPDLJOHPUIFSXPSLFST Ø 8PSLPOUIFKPC Ø *GJUTVDDFFEFE EFMFUFBKPCGSPNRVF@KPCT UBCMF Ø *GJUGBJMFE JODSFNFOUFSSPS@DPVOU VQEBUF SVO@BU GPSUIFOFYUSFUSZ BOESFDPSEFSSPS NFTTBHFJORVF@KPCTUBCMF 8PSLFS 4FSWFS 8PSLFS 1PPM 8PSLFS 1PPM RVF@KPCT -
RVFHPBEWJTPSZMPDL 18 Ø 2VFVFTZTUFNOFFETTPNFTPSUPGMPDLJOHNFDIBOJTNUPSVO XPSLFSTDPODVSSFOUMZ Ø "EWJTPSZ-PDL Ø 4JODF1PTUHSF42-
SFMFBTFEJO Ø *UˏTBQSFUUZNBUVBSFGVODUJPO Ø ˑ1PTUHSF42- QSPWJEFTBNFBOTGPSDSFBUJOHMPDLTUIBUIBWF BQQMJDBUJPOEFGJOFENFBOJOHT5IFTFBSFDBMMFE BEWJTPSZ MPDLT CFDBVTFUIFTZTUFNEPFTOPUFOGPSDFUIFJSVTFˋ JUJT VQUPUIFBQQMJDBUJPOUPVTFUIFNDPSSFDUMZ˒ -
RVFHPBEWJTPSZMPDL 19 Ø *UEPFTOˏUXSJUFUPUIFEJTL Ø -PDLTBSFTUPSFEJOBTIBSFENFNPSZQPPM Ø #FUUFSQFSGPSNBODFUIBONBOBHJOHKPCTUBUVTUBCMF Ø
#FODINBSL Ø IUUQKPIUPQHCMPHTQPUKQRVFVFTRVFVFT UIFZBMMGBMMEPXOIUNM Ø *UEPFTOˏUCMPDLPUIFSRVFSJFT Ø 6OMJLFSPXMPDLJOHˑTFMFDU˘GPSVQEBUF˒ Ø 4FTTJPO"4&-&$5QH@BEWJTPSZ@MPDL SFUVSOUSVF Ø 4FTTJPO#4&-&$5QH@BEWJTPSZ@MPDL SFUVSOGBMTF Ø "QQMJDBUJPOOFFETUPIBOEMFUIJT#PPMFBOWBMVF -
RVFHPXPSLFSXPSLFSQPPM 20 // WorkerPool is a pool of Workers,
// each working jobs from the queue Queue // at the specified Interval using the WorkMap. type WorkerPool struct { WorkMap WorkMap Interval time.Duration Queue string c *Client workers []*Worker mu sync.Mutex done bool } -
RVFHPXPSLFSXPSLFSQPPM 21 // WorkFunc is a function that performs
a Job. // If an error is returned, the job // is reenqueued with exponential backoff. type WorkFunc func(j *Job) error // WorkMap is a map of Job names to WorkFuncs // that are used to perform Jobs of a // given type. type WorkMap map[string]WorkFunc -
RVFHPXPSLFSXPSLFSQPPM 22 // Start starts all of the Workers
in the WorkerPool. func (w *WorkerPool) Start() { w.mu.Lock() defer w.mu.Unlock() for i := range w.workers { w.workers[i] = NewWorker(w.c, w.WorkMap) w.workers[i].Interval = w.Interval w.workers[i].Queue = w.Queue go w.workers[i].Work() } } -
RVFHPXPSLFSXPSLFSQPPM 23 // Worker is a single worker that
pulls jobs off the specified Queue. If no Job // is found, the Worker will sleep for Interval seconds. type Worker struct { // Interval is the amount of time that this Worker should sleep before trying // to find another Job. Interval time.Duration // Queue is the name of the queue to pull Jobs off of. The default value, "", // is usable and is the default for both que-go and the ruby que library. Queue string c *Client m WorkMap mu sync.Mutex done bool ch chan struct{} } -
RVFHPXPSLFSXPSLFSQPPM 24 // Work pulls jobs off the Worker's
Queue at its Interval. This function only // returns after Shutdown() is called, so it should be run in its own goroutine. func (w *Worker) Work() { for { select { case <-w.ch: log.Println("worker done") return case <-time.After(w.Interval): for { if didWork := w.WorkOne(); !didWork { break // didn't do any work, go back to sleep } } } } } -
RVFHPESBXCBDLT 25 Ø 4FWFSFQFSGPSNBODFEFHSBEBUJPOVOEFSNBTTJWFFORVFVF BOE MPOHSVOOJOHKPCT Ø IUUQTCSBOEVSPSHQPTUHSFTRVFVFT Ø
4UBUFEQMBJOMZ PVSSPPUQSPCMFNJTUIBUUIFKPCUBCMFT JOEFYIBTCFDPNFMFTTVTFGVMUPUIFQPJOUXIFSFVTJOHJU JTOUNVDIGBTUFSUIBOBGVMMTFRVFOUJBMTDBO Ø IUUQTHJUIVCDPNCSBOEVSRVFEFHSBEBUJPOUFTU Ø MPOHSVOOJOHKPC TMFFQTFDBCPVUNJO Ø BCPVUKPCFORVFVFE QFSTFD Ø BGUFSNJO KPCTJOUIFRVFUBCMF Ø TJHOJGJDBOUMPDLJOHQFSGPSNBODFEFHSBEBUJPO -
RVFHPESBXCBDLT 26 Ø BEWJTPSZMPDLJTEBUBCBTFXJEF BOEUIFLFZIBTUPCF JOUJOU Ø SFMBUJWFMZDPNQMFY8*5)3&$634*7&RVFSZUPMPDLJE Ø
IUUQTCSBOEVSPSHQPTUHSFTRVFVFTMPDLJOHBMHPSJUINT Ø TLJQMPDLFENJHIUQFSGPSNCFUUFSXJUITJNQMFSJNQMFNFOUBUJPO Ø GSPN1PTUHSF42- Ø IUUQTCMPHOERVBESBOUDPNXIBUJTTFMFDUTLJQMPDLFE GPSJOQPTUHSFTRM Ø 5IJTNJHIUDPNQMJDBUFFSSPSIBOEMJOH BOEFYQ CBDLPGG SFUSZ BCJU CVUXPSUIBUSZ Ø 5IPTFBCPWFBSFXIZ*TUBUFEˑFOPVHI˒JOUIFUJUMFPGUIJTTMJEF -
BQQMJDBUJPODPEF 28 Ø GVOD RVF+PC FSSPS Ø 5IJTJTUIFTJHOBUVSFXFIBWFUPXSJUF Ø 'BJSMZTJNQMF
Ø RVF+PC DPOUBJOTEBUBCBTFQPPM Ø 5IJTJTVTFEBMMEBUBCBTFPQFSBUJPOTJOKPCQSPDFTT Ø RVF+PC BMTPIBT"SHT XIJDIDPSSFTQPOETUPBSHT DPMVNOPG RVF@KPCTUBCMF Ø +40/UZQFDPMVNO Ø 6ONBSTIBMM "SHT <>CZUFJOUPQSFEFGJOFETUSVDU Ø *UJTWFSZVTFGVMUPIBWFKPCHMPCBMWBMVFT FH"1*DMJFOU BQQMJDBUJPODPOGJH JOBTUSVDU Ø -FUUIBUTUSVDU IBWFKPCNFUIPETTBUJTGJFTˑGVOD RVF+PC FSSPS˒TJHOBUVSF -
BQQMJDBUJPODPEF 29 // BaseApp base app global values type
BaseApp struct { Config *Config EstcClient *estc.Client } // JobApp job global values type JobApp struct { BaseApp Logger *log.Logger } IUUQTHJUIVCDPNBDIJLVHPUPEPJUCMPCNBTUFSKPCHP -
BQQMJDBUJPODPEF 30 IUUQTHJUIVCDPNBDIJLVHPUPEPJUCMPCNBTUFSKPCHP // UpdateUserInfo update user info job
func (app *JobApp) UpdateUserInfo(j *qg.Job) error { var args UpdateUserInfoArgs if err := json.Unmarshal(j.Args, &args); err != nil { return errors.Wrap(err, "failed to unmarshal UpdateTodoETCArgs") } dbReq := &model.TodoUser{ UUID: args.UserID, Username: args.Username, Email: args.Email, Status: args.Status, } if err := service.UpdateUser(j.Tx(), dbReq); err != nil { return errors.Wrap(err, "service.UpdateUser failed") } ctx := app.Context() req := &estc.UpdateUserRequest{ ID: args.UserID, UserName: args.Username, Email: args.Email, } u, err := app.EstcClient.UpdateUser(ctx, req) if err != nil { return errors.Wrap(err, "EstcClient.UpdateUser failed") } app.Logger.Printf("userID=%s,email=%s", u.ID, u.Email) return nil } -
BQQMJDBUJPODPEF 31 IUUQTHJUIVCDPNBDIJLVHPUPEPJUCMPCNBTUFSXPSLFSHP jobs := JobApp{ BaseApp: BaseApp{ Config:
appCfg, }, } wm := qg.WorkMap{ "updateUserJob": jobs.UpdateUserInfo, } wPoolLow := qg.NewWorkerPool(qc, wm, appCfg.NumWorkers) wPoolLow.Queue = QueNameLowPriority wPoolHigh := qg.NewWorkerPool(qc, wm, appCfg.NumWorkers) wPoolHigh.Queue = QueNameHighPriority -
BQQMJDBUJPODPEFUFTU 32 IUUQTHJUIVCDPNBDIJLVHPUPEPJUCMPCNBTUFSKPC@UFTUHP func TestUpdateUserInfo(t *testing.T) { app, tx,
_, cleanup := testSetupJobApp(t) defer cleanup() u := model.TestCreateUserData(t, tx, &model.TodoUser{}) args := UpdateUserInfoArgs{ UserID: u.UUID, Email: fmt.Sprintf("%[email protected]", u.UUID), Status: "inactive", Username: "updated-username", } jsonArgs, err := json.Marshal(args) if err != nil { t.Fatal(err) } ts := httptest.NewServer(estc.TestNewMux(estc.DefaultHandlerMap)) defer ts.Close() app.Config.EstcConfig.BaseEndpoint = ts.URL j := qg.TestInjectJobTx(&qg.Job{Args: jsonArgs}, tx) if err := app.UpdateUserInfo(j); err != nil { t.Fatal(err) } } -
34 "1* 4FSWFS 8PSLFS 4FSWFS RVF@KPCT 8PSLFS 1PPM 8PSLFS 1PPM
PVUTJEFPGUSBOTBDUJPO &YUFSOBM "1* PVUTJEFPG USBOTBDUJPO -
CFQSFQBSFEGPS"1*GBJMVSFT 37 Ø ,FFQJUˑKPCUSBOTBDUJPO˒ Ø :PVEPOˏUIBWFUPQBZFYUSBNFOUBMDPTUGPSEFTJHOJOHUIF CFTUSFUSZQSPDFTT Ø *GZPVVTFNVMUJQMFUSBOTBDUJPOTJOPOFKPC ZPVIBWFUP
UIJOLUISPVHIIPXZPVDBOHVBSEUIFEBUBJOUIFGJSTUUY XIFOUIFTFDPOEUY GBJMT Ø 8IFOJUJTSFBMMZOFFEFEUPIBWFNVMUJQMFUSBOTBDUJPOT UIFO DPOTJEFSUPEJWJEFBKPCJOUPUXPKPCT Ø 5IFGJSTUKPCFORVFVFUIFTFDPOEKPC Ø 8IFOFYUFSOBM"1*BDDFTTJTSFRVJSFE LFFQBTLJOH˒XIBUJGUIJT GBJMT DPVMEXFQSPUFDUPVSEBUBJOUFHSJUZ ˒ -
+PCDBUFHPSJFT 38 Ø $6% ˘$SFBUF6QEBUF%FMFUF Ø 3 ˘3FBE 3%#.4 &YUFSOBM"1*
OB 3 $6% OB 3 $6% Ø $BUFHPSZ Ø 3%#.4$36% BOESFBE FYUFSOBM"1* Ø $BUFHPSZ Ø 3%#.4$36% BOE$6% FYUFSOBM"1* -
$BUFHPSZ 39 3%#.4 &YUFSOBM"1* OB 3 $6% OB 3 $6%
Ø $BUFHPSZKPCDBOCFDPNQMFUFEXJUIJO3%#.4 Ø +PCJTQSPUFDUFECZUSBOTBDUJPO Ø 4VDDFTTPSGBJMVSFJTBUPNJD Ø *GKPCGBJMTCZ3%#.4PQFSBUJPOPSSFBEFYUFSOBM"1*T KVTU SPMMCBDLBMMUIFEBUBDIBOHFT BOESFUSZ -
$BUFHPSZ 40 3%#.4 &YUFSOBM"1* OB 3 $6% OB 3 $6%
Ø $BMMFYUFSOBM"1*TUPDIBOHFUIFJSEBUB BOEBMTPNPEJGZEBUBJO 3%#.4 Ø %BUBDIBOHFTJO3%#.4DBOCFSPMMFCBDLFE JGUIFQSPDFTT GBJMFE CVUEBUBNPEJGJDBUJPOTUISPVHIFYUFSOBM"1*TDBOˏU CFDBODFMFE Ø *UNJHIUCFQPTTJCMFUPSFTUPSFFYUFSOBMEBUBCBDLUPQSFWJPVT TUBUFJOFYDFQUJPOIBOEMJOH CVUJUXJMMNBLFTJNQMFUIJOHT DPNQMJDBUFE Ø 5SZOPUUPDBMMUIFTBNF"1*NVMUJQMFUJNFTFWFOXIFOJUIBTUP SFUSZ Ø 5SZUPDBMMFYUFSOBM"1*BUUIFMBTUQBSUPGUIFQSPDFTT Ø 5SZOPUUPVTFNVMUJQMFFYUFSOBM"1*TJOKPC Ø USBOTBDUJPO KPC BUNPTUFYUFSOBM$6%"1*