Subee Pub/Sub Worker Framework Implementation Go Conference Spring - @hlts2

$ whoami • Hiroto Funakoshi • @hlts2 • ۦ͚ग़͠ͷ৽ଔΤϯδχΞ • αʔϏεͷΞʔΩςΫνϟ΍ج൫։ൃʹڵຯ͕͋Δ

$ whoami • Hiroto Funakoshi • @hlts2 • ۦ͚ग़͠ͷ৽ଔΤϯδχΞ • αʔϏεͷΞʔΩςΫνϟ΍ج൫։ൃʹڵຯ͕͋Δ WantedlyΠϯλʔϯͰ։ൃͨ͠OSSʹ͍ͭͯൃද͠·͢

໨࣍ • Pub/Subͱ͸ʁ • Subeeͷ঺հ • Subeeͷར༻ํ๏

Pub/Sub?

Pub/Sub • ඇಉظϝοηʔδϯάύϥμΠϜͷҰछ • ϝοηʔδૹ৴ऀ͕ड৴ऀΛ૝ఆͤͣʹ
 ϝοηʔδΛૹΕΔΑ͏ϓϩάϥϜ͞Εͨ΋ͷ
 
 
 
 
 
 Go Publisher
 ʢProducerʣ Ruby Python Go Go Subscriber
 ʢConsumerʣ Pub/Sub
 ʢBrokerʣ

ͳͥඞཁʁ

Ϣʔεέʔε https://cloud.google.com/pubsub/docs/overview?hl=ja#scenarios

Ϣʔεέʔε https://cloud.google.com/pubsub/docs/overview?hl=ja#scenarios

Ϣʔεέʔε https://cloud.google.com/pubsub/docs/overview?hl=ja#scenarios

Ϣʔεέʔε https://cloud.google.com/pubsub/docs/overview?hl=ja#scenarios QueueʢඇಉظAPIʣ

Ϣʔεέʔε https://cloud.google.com/pubsub/docs/overview?hl=ja#scenarios Πϕϯτ఻ൖ

ϢʔεέʔεʢඇಉظAPIʣ API Server Push Server

ϢʔεέʔεʢඇಉظAPIʣ API Server Push Server HTTP or gRPC Push ґཔ

ϢʔεέʔεʢඇಉظAPIʣ API Server Push Server Push ґཔ × 100ສ HTTP or gRPC

ϢʔεέʔεʢඇಉظAPIʣ API Server Push Server Push ґཔ × 100ສ HTTP or gRPC Push Server͕൵໐Λ͋͛Δ

ϢʔεέʔεʢඇಉظAPIʣ API Server Push Server Pub/Sub

ϢʔεέʔεʢඇಉظAPIʣ API Server Push Server Pub/Sub Push ґཔ × 100ສ

ϢʔεέʔεʢඇಉظAPIʣ API Server Push Server Pub/Sub ࣗ෼ͷϖʔεͰॲཧ Push ґཔ × 100ສ

ϢʔεέʔεʢඇಉظAPIʣ API Server Push Server Pub/Sub ࣗ෼ͷϖʔεͰॲཧ Push ґཔ × 100ສ Push ServerͷෛՙΛ཈͑Δ͜ͱ͕ग़དྷΔ

ϢʔεέʔεʢΠϕϯτ఻ൖʣ User Service A Service B Service C Service

ϢʔεέʔεʢΠϕϯτ఻ൖʣ User Service A Service B Service C Service ProfileΛߋ৽ HTTP or gRPC

ϢʔεέʔεʢΠϕϯτ఻ൖʣ User Service A Service B Service C Service Profile͕ߋ৽ HTTP or gRPC ґଘ͢ΔαʔϏεΛ֮͑Δͷ͕େม

ϢʔεέʔεʢΠϕϯτ఻ൖʣ User Service A Service Pub/Sub B Service C Service

ϢʔεέʔεʢΠϕϯτ఻ൖʣ User Service A Service Pub/Sub B Service C Service ProfileΛߋ৽

ϢʔεέʔεʢΠϕϯτ఻ൖʣ User Service A Service Pub/Sub B Service C Service ProfileΛߋ৽

ϢʔεέʔεʢΠϕϯτ఻ൖʣ User Service A Service Pub/Sub B Service C Service ProfileΛߋ৽ υϝΠϯڥք͕Ͱ͖Δ

ϢʔεέʔεʢΠϕϯτ఻ൖʣ User Service A Service Pub/Sub B Service C Service ProfileΛߋ৽ υϝΠϯڥք͕Ͱ͖Δ ґଘ͢ΔαʔϏεΛ஌Βͳͯ͘΋͍͍

࣮૷ྫʢCloud Pub/Subʣ ctx := context.Background() err := subscription.Receive(ctx, func(ctx context.Context, msg *pubsub.Message) { var post Post if err := json.Unmarshal(msg.Data, &post); err != nil { log.Printf("could not decode message data: %#v", msg) msg.Ack() return } if err := update(post); err != nil { log.Printf("[ID %d] could not update: %v", post.Id, err) msg.Nack() return } msg.Ack() })

࣮૷ྫʢCloud Pub/Subʣ ctx := context.Background() err := subscription.Receive(ctx, func(ctx context.Context, msg *pubsub.Message) { var post Post if err := json.Unmarshal(msg.Data, &post); err != nil { log.Printf("could not decode message data: %#v", msg) msg.Ack() return } if err := update(post); err != nil { log.Printf("[ID %d] could not update: %v", post.Id, err) msg.Nack() return } msg.Ack() })

࣮૷ྫʢCloud Pub/Subʣ ctx := context.Background() err := subscription.Receive(ctx, func(ctx context.Context, msg *pubsub.Message) { var post Post if err := json.Unmarshal(msg.Data, &post); err != nil { log.Printf("could not decode message data: %#v", msg) msg.Ack() return } if err := update(post); err != nil { log.Printf("[ID %d] could not update: %v", post.Id, err) msg.Nack() return } msg.Ack() }) • ϝοηʔδΛਖ਼͘͠ॲཧग़དྷͨࣄΛPub/Subʹ఻͑Δ ‣ Pub/Sub͸ϝοηʔδΛ࡟আ͢Δ
 ɹɹɹɹϝοηʔδ͕࠶ૹ͞Εͳ͍

࣮૷ྫʢCloud Pub/Subʣ ctx := context.Background() err := subscription.Receive(ctx, func(ctx context.Context, msg *pubsub.Message) { var post Post if err := json.Unmarshal(msg.Data, &post); err != nil { log.Printf("could not decode message data: %#v", msg) msg.Ack() return } if err := update(post); err != nil { log.Printf("[ID %d] could not update: %v", post.Id, err) msg.Nack() return } msg.Ack() }) • ϝοηʔδΛਖ਼͘͠ॲཧग़དྷͳ͔ͬͨࣄΛPub/Subʹ఻͑Δ ‣ Pub/Sub͸ϝοηʔδΛ࡟আ͠ͳ͍
 ɹɹɹɹɹɹɹϝοηʔδ͕࠶ૹ͞ΕΔ

Pub/Subಋೖͷ՝୊఺ • Worker࣮૷͕ඞཁ ‣ ୯Ұɾෳ਺ϝοηʔδΛॲཧ ‣ Consume࿙Εͳ͘Graceful Shutdown ‣ ϝτϦΫεͱΤϥʔϨϙʔςΟϯά • ຊ࣭Ͱ͸ͳ͍ίʔυʢج൫ͷ࣮૷ʣ͕૿͑Δ • ϚΠΫϩαʔϏεΛΨϯΨϯ࡞Δ૊৫Ͱ͸ 
 Workerͷίʔυ͕ڞ௨Խग़དྷ͍ͯͳ͍ͱࣅͨΑ͏ͳίʔυ͕ࢄཚ

Pub/Subಋೖͷ՝୊఺ • Worker࣮૷͕ඞཁ ‣ ୯Ұɾෳ਺ϝοηʔδΛॲཧ ‣ Consume࿙Εͳ͘Graceful Shutdown ‣ ϝτϦΫεͱΤϥʔϨϙʔςΟϯά • ຊ࣭Ͱ͸ͳ͍ίʔυʢج൫ͷ࣮૷ʣ͕૿͑Δ • ϚΠΫϩαʔϏεΛΨϯΨϯ࡞Δ૊৫Ͱ͸ 
 Workerͷίʔυ͕ڞ௨Խग़དྷ͍ͯͳ͍ͱࣅͨΑ͏ͳίʔυ͕ࢄཚ Կ౓΋࣮૷͢Δͷʁ

ctx := context.Background() err := subscription.Receive(ctx, func(ctx context.Context, msg *pubsub.Message) { var post Post if err := json.Unmarshal(msg.Data, &post); err != nil { msg.Ack() return } if err := update(post); err != nil { msg.Nack() return } msg.Ack() })

ctx := context.Background() err := subscription.Receive(ctx, func(ctx context.Context, msg *pubsub.Message) { var post Post if err := json.Unmarshal(msg.Data, &post); err != nil { msg.Ack() return } if err := update(post); err != nil { msg.Nack() return } msg.Ack() }) LoggerΛ͍Ε͍ͨʂʂ

ctx := context.Background() err := subscription.Receive(ctx, func(ctx context.Context, msg *pubsub.Message) { start := time.Now() log.Println("Receive message") log.Printf("Start consume process") defer log.Printf("End consume process. time: %v", time.Since(start)) var post Post if err := json.Unmarshal(msg.Data, &post); err != nil { log.Printf("could not decode message data: %#v", msg) msg.Ack() return } if err := update(post); err != nil { log.Printf("[ID %d] could not update: %v", post.Id, err) msg.Nack() return } msg.Ack() })

ctx := context.Background() err := subscription.Receive(ctx, func(ctx context.Context, msg *pubsub.Message) { start := time.Now() log.Println("Receive message") log.Printf("Start consume process") defer log.Printf("End consume process. time: %v", time.Since(start)) var post Post if err := json.Unmarshal(msg.Data, &post); err != nil { log.Printf("could not decode message data: %#v", msg) msg.Ack() return } if err := update(post); err != nil { log.Printf("[ID %d] could not update: %v", post.Id, err) msg.Nack() return } msg.Ack() })

ctx := context.Background() err := subscription.Receive(ctx, func(ctx context.Context, msg *pubsub.Message) { start := time.Now() log.Println("Receive message") log.Printf("Start consume process") defer log.Printf("End consume process. time: %v", time.Since(start)) var post Post if err := json.Unmarshal(msg.Data, &post); err != nil { log.Printf("could not decode message data: %#v", msg) msg.Ack() return } if err := update(post); err != nil { log.Printf("[ID %d] could not update: %v", post.Id, err) msg.Nack() return } msg.Ack() }) Panic RecoverೖΕ͍ͨʂʂ

ctx := context.Background() err := subscription.Receive(ctx, func(ctx context.Context, msg *pubsub.Message) { start := time.Now() log.Println("Receive message") log.Printf("Start consume process") defer func() { log.Printf("End consume process. time: %v", time.Since(start)) if p := recover(); p != nil { log.Printf("Revoery from panic: %v", p) } }() var post Post if err := json.Unmarshal(msg.Data, &post); err != nil { log.Printf("could not decode message data: %#v", msg) msg.Ack() return } if err := update(post); err != nil { log.Printf("[ID %d] could not update: %v", post.Id, err) msg.Nack() return } msg.Ack() })

ctx := context.Background() err := subscription.Receive(ctx, func(ctx context.Context, msg *pubsub.Message) { start := time.Now() log.Println("Receive message") log.Printf("Start consume process") defer func() { log.Printf("End consume process. time: %v", time.Since(start)) if p := recover(); p != nil { log.Printf("Revoery from panic: %v", p) } }() var post Post if err := json.Unmarshal(msg.Data, &post); err != nil { log.Printf("could not decode message data: %#v", msg) msg.Ack() return } if err := update(post); err != nil { log.Printf("[ID %d] could not update: %v", post.Id, err) msg.Nack() return } msg.Ack() })

ctx := context.Background() err := subscription.Receive(ctx, func(ctx context.Context, msg *pubsub.Message) { start := time.Now() log.Println("Receive message") log.Printf("Start consume process") defer func() { log.Printf("End consume process. time: %v", time.Since(start)) if p := recover(); p != nil { log.Printf("Revoery from panic: %v", p) } }() var post Post if err := json.Unmarshal(msg.Data, &post); err != nil { log.Printf("could not decode message data: %#v", msg) msg.Ack() return } if err := update(post); err != nil { log.Printf("[ID %d] could not update: %v", post.Id, err) msg.Nack() return } msg.Ack() }) Error ReportingೖΕ͍ͨʂʂ

raven.SetDSN("https://:@sentry.io/") ctx := context.Background() err := subscription.Receive(ctx, func(ctx context.Context, msg *pubsub.Message) { start := time.Now() log.Println("Receive message") log.Printf("Start consume process") defer func() { log.Printf("End consume process. time: %v", time.Since(start)) if p := recover(); p != nil { log.Printf("Revoery from panic: %v", p) } }() var post Post if err := json.Unmarshal(msg.Data, &post); err != nil { raven.CaptureError(err, nil) log.Printf("could not decode message data: %#v", msg) msg.Ack() return } if err := update(post); err != nil { raven.CaptureError(err, nil) log.Printf("[ID %d] could not update: %v", post.Id, err) msg.Nack() return } msg.Ack() })

raven.SetDSN("https://:@sentry.io/") ctx := context.Background() err := subscription.Receive(ctx, func(ctx context.Context, msg *pubsub.Message) { start := time.Now() log.Println("Receive message") log.Printf("Start consume process") defer func() { log.Printf("End consume process. time: %v", time.Since(start)) if p := recover(); p != nil { log.Printf("Revoery from panic: %v", p) } }() var post Post if err := json.Unmarshal(msg.Data, &post); err != nil { raven.CaptureError(err, nil) log.Printf("could not decode message data: %#v", msg) msg.Ack() return } if err := update(post); err != nil { raven.CaptureError(err, nil) log.Printf("[ID %d] could not update: %v", post.Id, err) msg.Nack() return } msg.Ack() })

raven.SetDSN("https://:@sentry.io/") ctx := context.Background() err := subscription.Receive(ctx, func(ctx context.Context, msg *pubsub.Message) { start := time.Now() log.Println("Receive message") log.Printf("Start consume process") defer func() { log.Printf("End consume process. time: %v", time.Since(start)) if p := recover(); p != nil { log.Printf("Revoery from panic: %v", p) } }() var post Post if err := json.Unmarshal(msg.Data, &post); err != nil { raven.CaptureError(err, nil) log.Printf("could not decode message data: %#v", msg) msg.Ack() return } if err := update(post); err != nil { raven.CaptureError(err, nil) log.Printf("[ID %d] could not update: %v", post.Id, err) msg.Nack() return } msg.Ack() }) StatsΛऔΓ͍ͨʂʂ

raven.SetDSN("https://:@sentry.io/") ctx := context.Background() err := subscription.Receive(ctx, func(ctx context.Context, msg *pubsub.Message) { start := time.Now() log.Println("Receive message") log.Printf("Start consume process") statsHandler.HandleProcess(ctx, &Receive{ BeginTime: start, }) defer func() { log.Printf("End consume process. time: %v", time.Since(start)) if p := recover(); p != nil { raven.CaptureError(err, nil) log.Printf("Revoery from panic: %v", p) } }() consumeBeginTime := time.Now() var err error ctx = statsHandler.TagProcess(ctx, &ConsumeBeginTag{}) defer func() { statsHandler.HandleProcess(ctx, &ConsumeEnd{ BeginTime: consumeBeginTime, EndTime: time.Now(), Error: err, }) statsHandler.HandleProcess(ctx, &End{ MsgCount: 1, BeginTime: start, EndTime: time.Now(), }) } var post Post if err = json.Unmarshal(msg.Data, &post); err != nil { raven.CaptureError(err, nil) log.Printf("could not decode message data: %#v", msg) msg.Ack() return } if err = update(post); err != nil { raven.CaptureError(err, nil) log.Printf("[ID %d] could not update: %v", post.Id, err) msg.Nack() return } msg.Ack() })

raven.SetDSN("https://:@sentry.io/") ctx := context.Background() err := subscription.Receive(ctx, func(ctx context.Context, msg *pubsub.Message) { start := time.Now() log.Println("Receive message") log.Printf("Start consume process") statsHandler.HandleProcess(ctx, &Receive{ BeginTime: start, }) defer func() { log.Printf("End consume process. time: %v", time.Since(start)) if p := recover(); p != nil { raven.CaptureError(err, nil) log.Printf("Revoery from panic: %v", p) } }() consumeBeginTime := time.Now() var err error ctx = statsHandler.TagProcess(ctx, &ConsumeBeginTag{}) defer func() { statsHandler.HandleProcess(ctx, &ConsumeEnd{ BeginTime: consumeBeginTime, EndTime: time.Now(), Error: err, }) statsHandler.HandleProcess(ctx, &End{ MsgCount: 1, BeginTime: start, EndTime: time.Now(), }) } var post Post if err = json.Unmarshal(msg.Data, &post); err != nil { raven.CaptureError(err, nil) log.Printf("could not decode message data: %#v", msg) msg.Ack() return } if err = update(post); err != nil { raven.CaptureError(err, nil) log.Printf("[ID %d] could not update: %v", post.Id, err) msg.Nack() return } msg.Ack() })

raven.SetDSN("https://:@sentry.io/") ctx := context.Background() err := subscription.Receive(ctx, func(ctx context.Context, msg *pubsub.Message) { start := time.Now() log.Println("Receive message") log.Printf("Start consume process") statsHandler.HandleProcess(ctx, &Receive{ BeginTime: start, }) defer func() { log.Printf("End consume process. time: %v", time.Since(start)) if p := recover(); p != nil { raven.CaptureError(err, nil) log.Printf("Revoery from panic: %v", p) } }() consumeBeginTime := time.Now() var err error ctx = statsHandler.TagProcess(ctx, &ConsumeBeginTag{}) defer func() { statsHandler.HandleProcess(ctx, &ConsumeEnd{ BeginTime: consumeBeginTime, EndTime: time.Now(), Error: err, }) statsHandler.HandleProcess(ctx, &End{ MsgCount: 1, BeginTime: start, EndTime: time.Now(), }) } var post Post if err = json.Unmarshal(msg.Data, &post); err != nil { raven.CaptureError(err, nil) log.Printf("could not decode message data: %#v", msg) msg.Ack() return } if err = update(post); err != nil { raven.CaptureError(err, nil) log.Printf("[ID %d] could not update: %v", post.Id, err) msg.Nack() return } msg.Ack() }) ෳ਺ϝοηʔδΛॲཧ͍ͨ͠

No content

raven.SetDSN("https://:@sentry.io/") ctx := context.Background() err := subscription.Receive(ctx, func(ctx context.Context, msg *pubsub.Message) { start := time.Now() log.Println("Receive message") log.Printf("Start consume process") statsHandler.HandleProcess(ctx, &Receive{ BeginTime: start, }) defer func() { log.Printf("End consume process. time: %v", time.Since(start)) if p := recover(); p != nil { raven.CaptureError(err, nil) log.Printf("Revoery from panic: %v", p) } }() consumeBeginTime := time.Now() var err error ctx = statsHandler.TagProcess(ctx, &ConsumeBeginTag{}) defer func() { statsHandler.HandleProcess(ctx, &ConsumeEnd{ BeginTime: consumeBeginTime, EndTime: time.Now(), Error: err, }) statsHandler.HandleProcess(ctx, &End{ MsgCount: 1, BeginTime: start, EndTime: time.Now(), }) } var post Post if err = json.Unmarshal(msg.Data, &post); err != nil { raven.CaptureError(err, nil) log.Printf("could not decode message data: %#v", msg) msg.Ack() return } if err = update(post); err != nil { raven.CaptureError(err, nil) log.Printf("[ID %d] could not update: %v", post.Id, err) msg.Nack() return } msg.Ack() })

raven.SetDSN("https://:@sentry.io/") ctx := context.Background() err := subscription.Receive(ctx, func(ctx context.Context, msg *pubsub.Message) { start := time.Now() log.Println("Receive message") log.Printf("Start consume process") statsHandler.HandleProcess(ctx, &Receive{ BeginTime: start, }) defer func() { log.Printf("End consume process. time: %v", time.Since(start)) if p := recover(); p != nil { raven.CaptureError(err, nil) log.Printf("Revoery from panic: %v", p) } }() consumeBeginTime := time.Now() var err error ctx = statsHandler.TagProcess(ctx, &ConsumeBeginTag{}) defer func() { statsHandler.HandleProcess(ctx, &ConsumeEnd{ BeginTime: consumeBeginTime, EndTime: time.Now(), Error: err, }) statsHandler.HandleProcess(ctx, &End{ MsgCount: 1, BeginTime: start, EndTime: time.Now(), }) } var post Post if err = json.Unmarshal(msg.Data, &post); err != nil { raven.CaptureError(err, nil) log.Printf("could not decode message data: %#v", msg) msg.Ack() return } if err = update(post); err != nil { raven.CaptureError(err, nil) log.Printf("[ID %d] could not update: %v", post.Id, err) msg.Nack() return } msg.Ack() })

ͦ΋ͦ΋୭͕࣮૷͢Δͷʁ

ΞϓϦέʔγϣϯΤϯδχΞʁ

Want to

͍ͪૣ͘ΞϓϦέʔγϣϯΛϢʔβʔʹಧ͚͍ͨ ແବͳࣄʹ࣌ؒΛ͖ͨ͘͞ͳ͍ʢຊ࣭Ͱ͸ͳ͍෦෼ʣ

͍ͪૣ͘ΞϓϦέʔγϣϯΛϢʔβʔʹಧ͚͍ͨ ແବͳࣄʹ࣌ؒΛ͖ͨ͘͞ͳ͍ʢຊ࣭Ͱ͸ͳ͍෦෼ʣ Worker࣮૷
 ʴ
 ΞϓϦέʔγϣϯ࣮૷

͍ͪૣ͘ΞϓϦέʔγϣϯΛϢʔβʔʹಧ͚͍ͨ ແବͳࣄʹ࣌ؒΛ͖ͨ͘͞ͳ͍ʢຊ࣭Ͱ͸ͳ͍෦෼ʣ Worker࣮૷
 ʴ
 ΞϓϦέʔγϣϯ࣮૷ ελʔτμογϡ͕஗͘ͳΔ

ཉ͔ͬͨ͠΋ͷ • ج൫ʢe.g. Workerʣ͕Ͳ͏ͳ͍ͬͯΔ͔ҙࣝͤͣʹ
 ΞϓϦέʔγϣϯίʔυʹूதͰ͖ΔΑ͏ͳ࢓૊Έ
 
 
 
 
 
 


ཉ͔ͬͨ͠΋ͷ • ج൫ʢe.g. Workerʣ͕Ͳ͏ͳ͍ͬͯΔ͔ҙࣝͤͣʹ
 ΞϓϦέʔγϣϯίʔυʹूதͰ͖ΔΑ͏ͳ࢓૊Έ
 
 
 
 
 
 
 ΞϓϦέʔγϣϯ࣮૷ ج൫࣮૷ ج൫ͱΞϓϦέʔγϣϯίʔυ͕෼཭Ͱ͖͍ͯΔੈք

ཉ͔ͬͨ͠΋ͷ • ج൫ʢe.g. Workerʣ͕Ͳ͏ͳ͍ͬͯΔ͔ҙࣝͤͣʹ
 ΞϓϦέʔγϣϯίʔυʹूதͰ͖ΔΑ͏ͳ࢓૊Έ
 
 
 
 
 
 
 ΞϓϦέʔγϣϯ࣮૷ ج൫࣮૷ ج൫ͱΞϓϦέʔγϣϯίʔυ͕෼཭Ͱ͖͍ͯΔੈք

ཉ͔ͬͨ͠΋ͷ • ج൫ʢe.g. Workerʣ͕Ͳ͏ͳ͍ͬͯΔ͔ҙࣝͤͣʹ
 ΞϓϦέʔγϣϯίʔυʹूதͰ͖ΔΑ͏ͳ࢓૊Έ
 
 
 
 
 
 
 ΞϓϦέʔγϣϯ࣮૷ ج൫࣮૷ ج൫ͱΞϓϦέʔγϣϯίʔυ͕෼཭Ͱ͖͍ͯΔੈք ࡞Δͧ

github.com/wantedly/subee

• ಡΈํ͸αϏΟʙʢα΂ΣʙͰ͸ͳ͍…ʣ • Go੡ • WantedlyͰ̑ͭͷϚΠΫϩαʔϏεͰಈ͍͍ͯΔ • Pub/Sub WorkerΛॻ͔ͣʹϝοηʔδॲཧΛ
 Մೳʹ͢ΔWorkerϑϨʔϜϫʔΫ Subscribe + bee = Subee

Subeeಋೖͷൺֱʢલʣ <=====> Worker Code Application Code Application layer ΞϓϦέʔγϣϯίʔυͱج൫͕ີ݁߹ Pub/Sub

Subeeಋೖͷൺֱʢޙʣ Worker Code Application Code Framework Application ΞϓϦέʔγϣϯίʔυͱج൫ͷ੾Γ཭͠ <=====> Pub/Sub

Subeeಋೖͷൺֱʢޙʣ Worker Code Application Code Framework Application ΞϓϦέʔγϣϯίʔυͱج൫ͷ੾Γ཭͠ <=====> Pub/Sub

ಋೖલͷίʔυ

raven.SetDSN("https://:@sentry.io/") ctx := context.Background() err := subscription.Receive(ctx, func(ctx context.Context, msg *pubsub.Message) { start := time.Now() log.Println("Receive message") log.Printf("Start consume process") statsHandler.HandleProcess(ctx, &Receive{ BeginTime: start, }) defer func() { log.Printf("End consume process. time: %v", time.Since(start)) if p := recover(); p != nil { raven.CaptureError(err, nil) log.Printf("Revoery from panic: %v", p) } }() consumeBeginTime := time.Now() var err error ctx = statsHandler.TagProcess(ctx, &ConsumeBeginTag{}) defer func() { statsHandler.HandleProcess(ctx, &ConsumeEnd{ BeginTime: consumeBeginTime, EndTime: time.Now(), Error: err, }) statsHandler.HandleProcess(ctx, &End{ MsgCount: 1, BeginTime: start, EndTime: time.Now(), }) } var post Post if err = json.Unmarshal(msg.Data, &post); err != nil { raven.CaptureError(err, nil) log.Printf("could not decode message data: %#v", msg) msg.Ack() return } if err = update(post); err != nil { raven.CaptureError(err, nil) log.Printf("[ID %d] could not update: %v", post.Id, err) msg.Nack() return } msg.Ack() })

ಋೖޙͷίʔυ

subscriber, err := cloudpubsub.CreateSubscriber(ctx, "projectId", "subscriptionId") if err != nil { panic(err) } engine := subee.New( subscriber, NewConsumer(), NewStatshandler(), subee.WithConsumerInterceptors( subee_zap.ConsumerInterceptor(zap.Nop()), subee_recovery.ConsumerInterceptor( func(ctx context.Context, p interface{}) error { honybadger.Notify(p) return nil }, ), ) subee.WithStatsHandler( NewStatshandler(), ), ) engine.Start(ctx); type consumer struct {} func NewConsumer() subee.Consumer { return new(consumer) } func (c *consumer) Consume(ctx context.Context, msg subee.Message) error { return nil }

subscriber, err := cloudpubsub.CreateSubscriber(ctx, "projectId", "subscriptionId") if err != nil { panic(err) } engine := subee.New( subscriber, NewConsumer(), NewStatshandler(), subee.WithConsumerInterceptors( subee_zap.ConsumerInterceptor(zap.Nop()), subee_recovery.ConsumerInterceptor( func(ctx context.Context, p interface{}) error { honybadger.Notify(p) return nil }, ), ) subee.WithStatsHandler( NewStatshandler(), ), ) engine.Start(ctx); type consumer struct {} func NewConsumer() subee.Consumer { return new(consumer) } func (c *consumer) Consume(ctx context.Context, msg subee.Message) error { return nil } ConsumerΛੜ੒

type consumer struct {} func NewConsumer() subee.Consumer { return new(consumer) } func (c *consumer) Consume(ctx context.Context, msg subee.Message) error { return nil } subscriber, err := cloudpubsub.CreateSubscriber(ctx, "projectId", "subscriptionId") if err != nil { panic(err) } engine := subee.New( subscriber, NewConsumer(), NewStatshandler(), subee.WithConsumerInterceptors( subee_zap.ConsumerInterceptor(zap.Nop()), subee_recovery.ConsumerInterceptor( func(ctx context.Context, p interface{}) error { honybadger.Notify(p) return nil }, ), ) subee.WithStatsHandler( NewStatshandler(), ), ) engine.Start(ctx); Frameworkʹొ࿥͢Δ

subscriber, err := cloudpubsub.CreateSubscriber(ctx, "projectId", "subscriptionId") if err != nil { panic(err) } engine := subee.New( subscriber, NewConsumer(), NewStatshandler(), subee.WithConsumerInterceptors( subee_zap.ConsumerInterceptor(zap.Nop()), subee_recovery.ConsumerInterceptor( func(ctx context.Context, p interface{}) error { honybadger.Notify(p) return nil }, ), ) subee.WithStatsHandler( NewStatshandler(), ), ) engine.Start(ctx);

subscriber, err := cloudpubsub.CreateSubscriber(ctx, "projectId", "subscriptionId") if err != nil { panic(err) } engine := subee.New( subscriber, NewConsumer(), NewStatshandler(), subee.WithConsumerInterceptors( subee_zap.ConsumerInterceptor(zap.Nop()), subee_recovery.ConsumerInterceptor( func(ctx context.Context, p interface{}) error { honybadger.Notify(p) return nil }, ), ) subee.WithStatsHandler( NewStatshandler(), ), ) engine.Start(ctx);

Subscriberͷੜ੒ • Pub/Sub͔ΒϝοηʔδΛऔಘ͢Δ੹຿Λ࣋ͭ • Cloud Pub/SubͷSubscriber࣮૷͸
 ϑϨʔϜϫʔΫʹ༻ҙ • ઃఆ஋ΛຒΊΔ͚ͩͰ
 Subscriberͷར༻͕Մೳ • ॊೈͳઃఆΛ͢ΔͨΊʹ
 Function OptionͰΧελϚΠζՄೳ subscriber, err := cloudpubsub.CreateSubscriber(ctx, “cloudpub-app", "subscription-id") if err != nil { panic(err) }

Pub/Sub Consumer Interceptor Queue Subscriber ack ack receive ack (Business Logic) Engine

subscriber, err := cloudpubsub.CreateSubscriber(ctx, "projectId", "subscriptionId") if err != nil { panic(err) } engine := subee.New( subscriber, NewConsumer(), NewStatshandler(), subee.WithConsumerInterceptors( subee_zap.ConsumerInterceptor(zap.Nop()), subee_recovery.ConsumerInterceptor( func(ctx context.Context, p interface{}) error { honybadger.Notify(p) return nil }, ), ) subee.WithStatsHandler( NewStatshandler(), ), ) engine.Start(ctx);

subscriber, err := cloudpubsub.CreateSubscriber(ctx, "projectId", "subscriptionId") if err != nil { panic(err) } engine := subee.New( subscriber, NewConsumer(), NewStatshandler(), subee.WithConsumerInterceptors( subee_zap.ConsumerInterceptor(zap.Nop()), subee_recovery.ConsumerInterceptor( func(ctx context.Context, p interface{}) error { honybadger.Notify(p) return nil }, ), ) subee.WithStatsHandler( NewStatshandler(), ), ) engine.Start(ctx);

subscriber, err := cloudpubsub.CreateSubscriber(ctx, "projectId", "subscriptionId") if err != nil { panic(err) } engine := subee.New( subscriber, NewConsumer(), NewStatshandler(), subee.WithConsumerInterceptors( subee_zap.ConsumerInterceptor(zap.Nop()), subee_recovery.ConsumerInterceptor( func(ctx context.Context, p interface{}) error { honybadger.Notify(p) return nil }, ), ) subee.WithStatsHandler( NewStatshandler(), ), ) engine.Start(ctx); type consumer struct {} func NewConsumer() subee.Consumer { return new(consumer) } func (c *consumer) Consume(ctx context.Context, msg subee.Message) error { return nil }

Consumerͷੜ੒ • ୯ҰϝοηʔδΛॲཧ͢Δ
 ੹຿Λ࣋ͭ • ΞϓϦέʔγϣϯΤϯδχΞ͕
 ࣮ࡍʹ࣮૷͢Δ෦෼ type consumer struct {} func NewConsumer() subee.Consumer { return new(consumer) } func (c *consumer) Consume(ctx context.Context, msg subee.Message) error { return nil }

Pub/Sub Consumer Interceptor Queue Subscriber ack ack receive ack (Business Logic) Engine

BatchConsumerͷੜ੒ • ෳ਺ϝοηʔδΛॲཧ͢Δ
 ੹຿Λ࣋ͭ • ΞϓϦέʔγϣϯΤϯδχΞ͕
 ࣮ࡍʹ࣮૷͢Δ෦෼ subee.BatchConsumerFunc( func(ctx context.Context, msg []subee.Message) error { return nil }, )

Pub/Sub Consumer Interceptor Queue Subscriber ack ack receive ack (Business Logic) Engine

ACK/NACKͷ੍ޚ • ConsumerɾBatchConsumerͷ໭Γ஋Ͱ
 ACK/NACK͞ΕΔ ‣ ໭Γ஋͕Τϥʔͷ৔߹ɺNACK͞ΕΔ

Pub/Sub Consumer Interceptor Queue Subscriber ack ack receive ack (Business Logic) Engine

subscriber, err := cloudpubsub.CreateSubscriber(ctx, "projectId", "subscriptionId") if err != nil { panic(err) } engine := subee.New( subscriber, NewConsumer(), NewStatshandler(), subee.WithConsumerInterceptors( subee_zap.ConsumerInterceptor(zap.Nop()), subee_recovery.ConsumerInterceptor( func(ctx context.Context, p interface{}) error { honybadger.Notify(p) return nil }, ), ) subee.WithStatsHandler( NewStatshandler(), ), ) engine.Start(ctx);

subscriber, err := cloudpubsub.CreateSubscriber(ctx, "projectId", "subscriptionId") if err != nil { panic(err) } engine := subee.New( subscriber, NewConsumer(), NewStatshandler(), subee.WithConsumerInterceptors( subee_zap.ConsumerInterceptor(zap.Nop()), subee_recovery.ConsumerInterceptor( func(ctx context.Context, p interface{}) error { honybadger.Notify(p) return nil }, ), ) subee.WithStatsHandler( NewStatshandler(), ), ) engine.Start(ctx);

InterceptorɾStatsHandlerͷੜ੒ • Interceptor ‣ Consumeલޙʹհೖ
 Ͱ͖Δ ‣ Logger, Recovery͸
 Frameworkʹ༻ҙ • StatsHandler ‣ ܭଌܥͷͨΊʹར༻ subee.WithConsumerInterceptors( subee_zap.ConsumerInterceptor(zap.Nop()), subee_recovery.ConsumerInterceptor( func(ctx context.Context, p interface{}) error { honybadger.Notify(p) return nil }, ), ), subee.WithStatsHandler( NewStatshandler(), )

ܭଌܥͷͨΊͷStatsHandler Channel͔ΒϝοηʔδΛड͚औΔ ϝοηʔδΛChannelʹྲྀ͢ Pub/Sub͔ΒϝοηʔδΛऔಘ Consume։࢝લ Consume։࢝ Consumeऴྃ औΓग़͔͠ΒConsume·ͰͷҰ࿈͕ऴྃ ࣌ܥྻ

ܭଌܥͷͨΊͷStatsHandler Channel͔ΒϝοηʔδΛड͚औΔ ϝοηʔδΛChannelʹྲྀ͢ Pub/Sub͔ΒϝοηʔδΛऔಘ Consume։࢝લ Consume։࢝ Consumeऴྃ औΓग़͔͠ΒConsume·ͰͷҰ࿈͕ऴྃ ࣌ܥྻ

Pub/Sub Consumer Interceptor Queue Subscriber ack ack receive ack (Business Logic) Engine

subscriber, err := cloudpubsub.CreateSubscriber(ctx, "projectId", "subscriptionId") if err != nil { panic(err) } engine := subee.New( subscriber, NewConsumer(), NewStatshandler(), subee.WithConsumerInterceptors( subee_zap.ConsumerInterceptor(zap.Nop()), subee_recovery.ConsumerInterceptor( func(ctx context.Context, p interface{}) error { honybadger.Notify(p) return nil }, ), ) subee.WithStatsHandler( NewStatshandler(), ), ) engine.Start(ctx);

subscriber, err := cloudpubsub.CreateSubscriber(ctx, "projectId", "subscriptionId") if err != nil { panic(err) } engine := subee.New( subscriber, NewConsumer(), NewStatshandler(), subee.WithConsumerInterceptors( subee_zap.ConsumerInterceptor(zap.Nop()), subee_recovery.ConsumerInterceptor( func(ctx context.Context, p interface{}) error { honybadger.Notify(p) return nil }, ), ) subee.WithStatsHandler( NewStatshandler(), ), ) engine.Start(ctx);

Engine • Subscribeͨ͠ϝοηʔδΛ
 Consumerʹ఻ൖ ‣ ୯ҰϝοηʔδΛ఻ൖ ‣ Bufferingͯ͠ෳ਺ϝοηʔδΛ఻ൖ • Graceful Shutdown engine := subee.New( subscriber, NewConsumer(), NewStatshandler(), subee.WithConsumerInterceptors( subee_zap.ConsumerInterceptor(zap.Nop()), subee_recovery.ConsumerInterceptor( func(ctx context.Context, p interface{}) error { honybadger.Notify(p) return nil }, ), ), subee.WithStatsHandler( NewStatshandler(), ), )

Engine • Subscribeͨ͠ϝοηʔδΛ
 Consumerʹ఻ൖ ‣ ୯ҰϝοηʔδΛ఻ൖ ‣ Bufferingͯ͠ෳ਺ϝοηʔδΛ఻ൖ • Graceful Shutdown engine := subee.New( subscriber, NewConsumer(), NewStatshandler(), subee.WithConsumerInterceptors( subee_zap.ConsumerInterceptor(zap.Nop()), subee_recovery.ConsumerInterceptor( func(ctx context.Context, p interface{}) error { honybadger.Notify(p) return nil }, ), ), subee.WithStatsHandler( NewStatshandler(), ), ) InterfaceΛຬͨ͢ObjectΛొ࿥͢Δ͚ͩʂʂ
 ͋ͱ͸ྑ͍ײ͡ʹ΍ͬͯ͘ΕΔ

Pub/Sub Consumer Interceptor Queue Subscriber ack ack receive ack (Business Logic) Engine

·ͱΊ • Subee: Pub/Sub Worker Framework Implementation ‣ WorkerपΓͷෳࡶ͞ΛӅṭ ‣ ΞϓϦέʔγϣϯΤϯδχΞ͕ຊ࣭తͳ࣮૷ʹूதͰ͖Δ https://github.com/wantedly/subee

github.com/wantedly/subee