Slide 1
Slide 1 text
pt
&Goroutine
- GoCon 2014 spring -
Slide 2
Slide 2 text
MIYAKE Yusuke
(@monochromegane)
Slide 3
Slide 3 text
GMO Pepabo, Inc.
Slide 4
Slide 4 text
grep
ͯ͠·͔͢ʁ
Slide 5
Slide 5 text
grep?
Slide 6
Slide 6 text
ack?
Slide 7
Slide 7 text
ag?
Slide 8
Slide 8 text
pt
The Platinum Searcher
Slide 9
Slide 9 text
Written in
Golang
Slide 10
Slide 10 text
Mac OSX
Linux
Windows
Slide 11
Slide 11 text
UTF-8
EUC-JP
Shift-JIS
Slide 12
Slide 12 text
AND
Slide 13
Slide 13 text
fast !
ack go 6.24s user 1.06s system 99% cpu 7.304 total # ack
ag go 0.88s user 1.39s system 221% cpu 1.027 total # ag
pt go 1.09s user 1.01s system 235% cpu 0.892 total # pt
Slide 14
Slide 14 text
How?
Slide 15
Slide 15 text
Goroutine & Channel
Slide 16
Slide 16 text
͍ͬ͠ΐʹߴԽͯ͠Έ·͠ΐ͏
Slide 17
Slide 17 text
1. ϑΝΠϧΛݕࡧͯ͠(find)
2. จࣈྻΛݕࡧͯ͠(grep)
3. ݁ՌΛදࣔ͢Δ(print)
ύλʔϯݕࡧͱ
Slide 18
Slide 18 text
Approach-0
!
ॱ൪ʹ
Slide 19
Slide 19 text
find
Slide 20
Slide 20 text
find
grep
Slide 21
Slide 21 text
find
grep
print
Slide 22
Slide 22 text
// find
find := find.Find{Option: self.Option}
find.Do(self.Root)
!
// grep
grep := grep.Grep{
Files: find.Files, // result
Pattern: self.Pattern,
Option: self.Option}
grep.Do()
!
// print
print := print.Print{
Matches: grep.Matches, // result
Pattern: self.Pattern,
Option: self.Option}
print.Do()
Slide 23
Slide 23 text
> the_simple_searcher go $GOROOT > /dev/null
Slide 24
Slide 24 text
0.79
seconds
Slide 25
Slide 25 text
Approach-1
!
ฒߦʹ
Slide 26
Slide 26 text
Goroutine
Slide 27
Slide 27 text
• GoݴޠͰฒߦॲཧΛ࣮ݱ͢Δ
• εϨουɺίϧʔνϯͱҧ͏
• Concurrency(ฒߦ)ͱParallelism(ฒྻ)
• ܰྔ
• go f()
Slide 28
Slide 28 text
find
grep
print
go
go
go
Slide 29
Slide 29 text
Channel
Slide 30
Slide 30 text
• Goroutineؒͷϝοηʔδϯά
• ͷૹड৴
• όοϑΝʹΑΔϒϩοΫ
Slide 31
Slide 31 text
find
grep
print
go
go
go
Slide 32
Slide 32 text
$IBOFM
find
grep
print
$IBOFM
go
go
go
Slide 33
Slide 33 text
$IBOFM
find
grep
print
$IBOFM
go
go
go
Slide 34
Slide 34 text
// channel
files := make(chan *string, self.Option.Cap)
matches := make(chan *grep.Match, self.Option.Cap)
done := make(chan bool)
!
// find
find := find.Find{Files: files, Option: self.Option}
go find.Do(self.Root)
!
// grep
grep := grep.Grep{
Files: files,
Matches: matches,
Pattern: self.Pattern,
Option: self.Option}
go grep.Do()
!
// print
print := print.Print{
Done: done,
Matches: matches,
Pattern: self.Pattern,
Option: self.Option}
go print.Do()
!
<-done // block
Slide 35
Slide 35 text
walkFunc := func(path string, info os.FileInfo,
err error) error {
if info.IsDir() {
return nil
}
self.Files <- &path // send
return nil
}
!
filepath.Walk(root, walkFunc)
close(self.Files) // close
Slide 36
Slide 36 text
for file := range self.Files { // receive ( <-self.Files )
fh, err := os.Open(*file)
if err != nil {
panic(err)
}
!
f := bufio.NewReader(fh)
!
var buf []byte
var lineNum = 1
for {
buf, _, err = f.ReadLine()
if err != nil {
break
}
line := string(buf)
if strings.Contains(line, self.Pattern) {
self.Matches <- &Match{*file, lineNum, line} // send
}
lineNum++
}
fh.Close()
}
close(self.Matches) // close
Slide 37
Slide 37 text
for match := range self.Matches { // receive
fmt.Printf("%s:%d:%s\n", match.Path,
match.Num, match.Match)
}
self.Done <- true // send
Slide 38
Slide 38 text
> the_simple_searcher go $GOROOT > /dev/null
Slide 39
Slide 39 text
0.79 -> 0.87
seconds
Slide 40
Slide 40 text
?
Slide 41
Slide 41 text
buffer
Slide 42
Slide 42 text
• Channelͷड༰ྔ
• ch := make(chan ܕ, ༰ྔ)
• ༰ྔ·Ͱड
• ༰ྔ͑Δͱૹ৴ଆडͪ
• ड৴͢Δͱ༰ྔ͕ͻͱۭͭ͘
• ༰ྔ͕0ͷ߹ɺৗʹͭ
Slide 43
Slide 43 text
// channel with buffer
files := make(chan *string, self.Option.Cap)
matches := make(chan *grep.Match, self.Option.Cap)
done := make(chan bool) // always wait
Slide 44
Slide 44 text
> the_simple_searcher go $GOROOT > /dev/null
Slide 45
Slide 45 text
0.79 -> 0.8
seconds
Slide 46
Slide 46 text
Approach-2
!
ͬͱฒߦʹ
Slide 47
Slide 47 text
$IBOFM
find
grep
print
$IBOFM
go
go
go
Slide 48
Slide 48 text
$IBOFM
find
grep
print
$IBOFM
go
go
go
grep
grep
grep
Slide 49
Slide 49 text
var wg sync.WaitGroup
for file := range self.Files {
wg.Add(1) // goroutineͷىಈΛΠϯΫϦϝϯτ
(தུ)
go func(self *Grep, file *string) {
defer wg.Done() // goroutine͕ྃͨ͠ΒىಈΛσΫϦϝϯτ
for {
ɹɹɹɹɹɹɹɹɹɹɹɹɹɹɹɹɹɹɹ(தུ)
}
fh.Close()
!
}(self, file) // ΫϩʔδϟΛgoroutineʹ͢Δͱ͖มͷڞ༗ʹҙ
!
}
wg.Wait() // ෆಛఆͷgoroutine͕શͯऴྃ͢ΔͷΛͭ
close(self.Matches)
Slide 50
Slide 50 text
> the_simple_searcher go $GOROOT > /dev/null
Slide 51
Slide 51 text
panic !
too many open files
Slide 52
Slide 52 text
var wg sync.WaitGroup
sem := make(chan bool, self.Option.Cap) // ىಈ͢ΔgoroutineͷΛ੍ޚ͢Δchannel
for file := range self.Files {
sem <- true // goroutineͷىಈ(channelͷbuffer)͕͍ͬͺ͍ͳΒͭ
wg.Add(1)
(தུ)
go func(self *Grep, file *string) {
defer wg.Done()
for {
ɹɹɹɹɹɹɹɹɹɹɹɹɹɹɹɹɹɹɹ(தུ)
}
fh.Close()
<-sem // ಉ࣌ىಈchannelͷbufferʹۭ͖Λͭ͘Δ
!
}(self, file)
!
}
wg.Wait()
close(self.Matches)
Slide 53
Slide 53 text
> the_simple_searcher go $GOROOT > /dev/null
Slide 54
Slide 54 text
0.79 -> 0.8
seconds
Slide 55
Slide 55 text
Approach-3
!
ฒྻʹ
Slide 56
Slide 56 text
GOMAXPROCS
Slide 57
Slide 57 text
• Goroutineͷฒྻ
• σϑΥϧτ1
• runtime.NumCPU()ͰίΞΛऔಘ
• runtime.GOMAXPROCS()ͰฒྻΛઃఆ
Slide 58
Slide 58 text
> the_simple_searcher go $GOROOT > /dev/null
Slide 59
Slide 59 text
0.79 -> 0.55 !
seconds
Slide 60
Slide 60 text
benchmark
!
• Mac OSX(10.9.3)
• CPU: 2.5GHz Core i5(2Core)
• Memory: 8GB
• Go: 1.2.2
Slide 61
Slide 61 text
#V⒎FS
(0."9130$4
"QQSPBDI
Slide 62
Slide 62 text
ฒߦԽͯ͠ͳ͍ͷͰ
ฒྻԽͯ͠มΘΒͣ ίΞҎ্ͷࢦఆ
ޮՌͳ͠
ଌఆͯ͠ௐ͠ͳ͍ͱ
߹ʹΑͬͯ͘ͳΔ
#V⒎FS
(0."9130$4
"QQSPBDI
Slide 63
Slide 63 text
–Rob Pike
• Concurrency is powerful.
• Concurrency is not parallelism.
• Concurrency enables parallelism.
• Concurrency makes parallelism (and
scaling and everything else) easy.
Slide 64
Slide 64 text
એ
ϖύϘͰΤϯδχΞΛืू͍ͯ͠·͢ɻ
ڞʹαʔϏεΛੜΈग़͠ҭͯͯ͘ΕΔ৽͍ؒ͠
Λ͍ͬͯ·͢ɻ
!
http://pepabo.com/recruit/career/engineer/
Slide 65
Slide 65 text
͓ΘΓ