Slide 1
Slide 1 text
©2019 Wantedly, Inc.
`cloud.google.com/go/pubsub` internal
golang.tokyo #24
May 20, 2019 - Masayuki Izumi
Slide 2
Slide 2 text
©2019 Wantedly, Inc.
$ whoami
@izumin5210
Wantedly, Inc.
‣ Wantedly People
- Application Engineer (Go, Ruby, Web
Frontend)
‣ Interested in…
- Microservices
- Developer Productivity
- Developer Experience
Slide 3
Slide 3 text
©2019 Wantedly, Inc.
Talk abst and desc
Slide 4
Slide 4 text
©2019 Wantedly, Inc.
HPPHMFDMPVEHPͱఆٛݩͷδϟϯϓ͕͑Δڥ
͕͋ΔͱɺൃදΛΑΓָ͓͠Έ͍͚ͨͩ·͢
git clone https://github.com/googleapis/google-cloud-go.git
cd google-cloud-go
git checkout v0.39.0
export GO111MODULE=on
go mod download
Slide 5
Slide 5 text
©2019 Wantedly, Inc.
1VC4VCɾϝοηʔδΩϡʔΈ͍ͨͳͭ
͔ͭͬͨ͜ͱ͋Δͻͱʔ
Slide 6
Slide 6 text
©2019 Wantedly, Inc.
Pub/Sub
• ඇಉظϝοηʔδϯάύϥμΠϜͷҰछ
• ϝοηʔδૹ৴ऀ͕ड৴ऀΛఆͤͣʹ
ϝοηʔδΛૹΕΔΑ͏ϓϩάϥϜ͞Εͨͷ
Go
Publisher
ʢProducerʣ
Ruby
Python
Go
Go
Subscriber
ʢConsumerʣ
Pub/Sub
ʢBrokerʣ
@hlts2, "Subee: Pub/Sub Worker Framework Implementation", GoCon 2019 Spring
IUUQTTQFBLFSEFDLDPNIMUTTVCXPSLFSGSBNFXPSLJNQMFNFOUBUJPO
Slide 7
Slide 7 text
©2019 Wantedly, Inc.
@hlts2, "Subee: Pub/Sub Worker Framework Implementation", GoCon 2019 Spring
IUUQTTQFBLFSEFDLDPNIMUTTVCXPSLFSGSBNFXPSLJNQMFNFOUBUJPO
ϢʔεέʔεʢඇಉظAPIʣ
API Server Push Server
Push ґཔ × 100ສ
HTTP or gRPC
ϢʔεέʔεʢඇಉظAPIʣ
API Server Push Server
Pub/Sub
ࣗͷϖʔεͰॲཧ
Push ґཔ × 100ສ
"1*αʔό͕ϖʔεΛѲΔͷͰɺ
8PSLFSଆ͕ϫʔΫϩʔυʹ͖͑Εͳ͍
8PSLFSࣗͷϖʔεͰॲཧΛ͢ΔͷͰམͪͳ͍͠ɺ
ඇಉظͳͷͰ"1*αʔόʹӨڹ͕ग़ͳ͍
Slide 8
Slide 8 text
©2019 Wantedly, Inc.
ϢʔεέʔεʢΠϕϯτൖʣ
User Service
A Service
Pub/Sub
B Service
C Service
ProfileΛߋ৽
υϝΠϯڥք͕Ͱ͖Δ
ϢʔεέʔεʢΠϕϯτൖʣ
User Service
A Service
B Service
C Service
ProfileΛߋ৽
HTTP or gRPC
@hlts2, "Subee: Pub/Sub Worker Framework Implementation", GoCon 2019 Spring
IUUQTTQFBLFSEFDLDPNIMUTTVCXPSLFSGSBNFXPSLJNQMFNFOUBUJPO
ج൫αʔϏε͕ͷϚΠΫϩαʔϏεʹ
ґଘ͞Ε͍ͯΔ͜ͱΛΒͳ͍ͱ͍͚ͳ͍
ͷαʔϏε͕ࣗ୭ʹґଘ͍ͯ͠Δ͔Λ͍ͬͯΔ͕ɺ
ج൫αʔϏε୭ʹґଘ͞Ε͍ͯΔ͔ΛΒͳͯ͘ࡁΉ
Slide 9
Slide 9 text
©2019 Wantedly, Inc.
client, err := pubsub.NewClient(ctx, "project-id")
if err != nil {
// ...
}
defer client.Close()
// topic リソースは事前につくっておく
topic := client.Topic("my-topic")
defer topic.Stop()
topic.Publish(ctx, &pubsub.Message{
Data: string("Hello, Workd!"),
})
IUUQTDMPVEHPPHMFDPNQVCTVCEPDTPWFSWJFX
Slide 10
Slide 10 text
©2019 Wantedly, Inc. IUUQTDMPVEHPPHMFDPNQVCTVCEPDTPWFSWJFX
client, err := pubsub.NewClient(ctx, "project-id")
if err != nil {
// ...
}
defer client.Close()
// subscription リソースは事前につくっておく
subscription := client.Subscription("my-subscription")
subscription.Receive(ctx, func(ctx context.Context, msg *pubsub.Message) error {
// ...
return nil
})
Slide 11
Slide 11 text
©2019 Wantedly, Inc.
"1*ɾ͍ํγϯϓϧ
Ͱ෦Ͱ͔ͳΓ͍Ζ͍Ζ͍ͬͯΔ
Slide 12
Slide 12 text
©2019 Wantedly, Inc.
৴པੑͱύϑΥʔϚϯε͕ཁٻ͞ΕΔ
ϛυϧΣΞͷΫϥΠΞϯτ
Slide 13
Slide 13 text
©2019 Wantedly, Inc.
ͦͷ෦Λ࣮ࡍʹݟͯΈ·͠ΐ͏
ϥΠϒίʔυϦʔσΟϯά͢Δͧʂʂ
Slide 14
Slide 14 text
©2019 Wantedly, Inc.
Publishing
Slide 15
Slide 15 text
©2019 Wantedly, Inc.
entrypoint:
`func (*pubsub.Topic) Publish(context.Context, *pubsub.Message)`
QVCTVCUPQJDHP-
1VCMJTIJOH FOUSZQPJOU
Slide 16
Slide 16 text
©2019 Wantedly, Inc.
google.golang.org/api/support/bundler
func NewBundler(itemExample interface{}, handler func(interface{})) *Bundler
func (b *Bundler) Add(item interface{}, size int) error
func (b *Bundler) AddWait(ctx context.Context, item interface{}, size int) error
func (b *Bundler) Flush()
1VCMJTIJOH CVOEMFS
ΞΠςϜΛཷΊࠐΜͰɺմ͝ͱʹॲཧ͢Δ
ʢσʔλ
σʔλ༰ྔ
࣌ؒͳͲͰGMVTI͢Δ݅ΛܾΊΒΕΔʣ
Slide 17
Slide 17 text
©2019 Wantedly, Inc.
google.golang.org/api/support/bundler
t.bundler = bundler.NewBundler(&bundledMessage{}, func(items interface{}) {
// TODO(jba): use a context detached from the one passed to NewClient.
ctx := context.TODO()
if timeout != 0 {
var cancel func()
ctx, cancel = context.WithTimeout(ctx, timeout)
defer cancel()
}
t.publishMessageBundle(ctx, items.([]*bundledMessage))
})
1VCMJTIJOH CVOEMFS
(PPHMF$MPVE1VC4VCͷQVCMJTIFSɺCVOEMFSʹཷΊࠐΜͰඇಉظʹϝοηʔδΛૹ৴͍ͯ͠Δ
./pubsub/topic.go L412
Slide 18
Slide 18 text
©2019 Wantedly, Inc.
github.com/googleapis/gax-go
gax.WithRetry(func() gax.Retryer {
return gax.OnCodes([]codes.Code{
codes.Aborted,
codes.Canceled,
codes.Internal,
codes.ResourceExhausted,
codes.Unavailable,
codes.Unknown,
}, gax.Backoff{
Initial: 100 * time.Millisecond,
Max: 60000 * time.Millisecond,
Multiplier: 1.3,
})
}),
},
1VCMJTIJOH HJUIVCDPNHPPHMFBQJTHBYHP
H31$Λར༻ͨ͠"1*DBMMͷϦτϥΠઓུΛӅṭ͢Δύοέʔδ
QVCTVCBQJWQVCMJTIFIS@DMJFOUHP-
Slide 19
Slide 19 text
©2019 Wantedly, Inc.
Subscribing
Slide 20
Slide 20 text
©2019 Wantedly, Inc.
entrypoint:
`func (*pubsub.Subscription) Receive(context.Context, func(context.Context, *pubsub.Message)) error`
QVCTVCTVCTDSJQUJPOHP-
4VCTDSJCJOH FOUSZQPJOU
Slide 21
Slide 21 text
©2019 Wantedly, Inc.
golang.org/x/sync/errgroup
group, gctx := errgroup.WithContext(ctx)
for i := 0; i < numGoroutines; i++ {
group.Go(func() error {
return s.receive(gctx, po, fc, f)
})
}
return group.Wait()
1VCMJTIJOH FSSHSPVQ
AFSSHSPVQAΛར༻͠ɺϝοηʔδΛड৴͢ΔͨΊͷXPSLFSHPSPVUJOFΛੜΈग़͍ͯ͠Δ
./pubsub/subscription.go L547
Slide 22
Slide 22 text
©2019 Wantedly, Inc.
4VCTDSJCJOH "DL/BDL
A QVCTVC.FTTBHF
"DL
A͕ݺΕΔͱ
"DLͪϝοηʔδͷεϥΠεʹBQQFOEͨ͋͠ͱɺ
GMPXDPOUSPMMFSʹͦΕΛΒͤͯΩϡʔʹۭ͖Λ࡞Δ
msg.doneFunc = func(ackID string, ack bool, receiveTime time.Time) {
defer fc.release(msgLen)
old(ackID, ack, receiveTime)
}
func (m *Message) Ack() {
m.done(true)
}
func (m *Message) Nack() {
m.done(false)
}
func (m *Message) done(ack bool) {
if m.calledDone {
return
}
m.calledDone = true
m.doneFunc(m.ackID, ack, m.receiveTime)
}
func (it *messageIterator) done(ackID string, ack bool, receiveTime time.Time) {
it.ackTimeDist.Record(int(time.Since(receiveTime) / time.Second))
it.mu.Lock()
defer it.mu.Unlock()
delete(it.keepAliveDeadlines, ackID)
if ack {
it.pendingAcks[ackID] = true
} else {
it.pendingNacks[ackID] = true
}
it.checkDrained()
}
QVCTVCJUFSBUPSHP-
QVCTVCTVCTDSJQUJPOHP-
QVCTVCNFTTBHFHP-
Slide 23
Slide 23 text
©2019 Wantedly, Inc.
ʜΈ͍ͨͳײ͡ͷ͜ͱΛɺࣾษڧձͰͬͯ·͢
.JDSPTFSWJDFT.POEBZIUUQTXXXXBOUFEMZDPNQSPKFDUT
(PQIFST$PEF3FBEJOH1BSUZΠϕϯτϖʔδ͕ͳ͍ͷͰ͔͚͍ͯͩ͘͞