Slide 1
Slide 1 text
GoݴޠͰͷഉଞ੍ޚ
monochromegane / Pepabo R&D Institute, GMO Pepabo, Inc.
2017.06.28 Geeks Who Drink in Fukuoka -Go Go Golang Edition!-
Mutual exclusion in Golang.
Slide 2
Slide 2 text
ϓϦϯγύϧΤϯδχΞ
ࡾ ༔հ / @monochromegane
2
http://blog.monochromegane.com
Yusuke Miyake
ϖύϘݚڀॴ ݚڀһ
Slide 3
Slide 3 text
1. Motivation
2. Mutual exclusion
3. Resource locking
3
Contents
Slide 4
Slide 4 text
1.
Motivation
Slide 5
Slide 5 text
Gannoy
Approximate nearest neighbor search server
and dynamic index written in Golang.
https://github.com/monochromegane/gannoy
Slide 6
Slide 6 text
Similar images search system using Gannoy
6
Features Similar items
Gannoy
[2048]float64
query
by http
find similar features
mapping
similar features
to items
response
Deep CNN
index
Features
[2048]float64
Deep CNN
register
by http
Slide 7
Slide 7 text
Many goroutines access one resource
concurrently.
Slide 8
Slide 8 text
2.
Mutual exclusion
Slide 9
Slide 9 text
Many goroutines access one variable
concurrently.
Slide 10
Slide 10 text
• sync.Mutex
• A Mutex is a mutual exclusion lock.
• sync.RWMutex
• An RWMutex is a reader/writer mutual exclusion lock. The lock can be held
by an arbitrary number of readers or a single writer.
10
Mutual exclusion in Golang
Slide 11
Slide 11 text
No content
Slide 12
Slide 12 text
12
type Free struct {
mu sync.Mutex
free []int
}
func (f *Free) push(id int) {
f.mu.Lock()
defer f.mu.Unlock()
f.free = append(f.free, id)
}
func (f *Free) pop() (int, error) {
f.mu.Lock()
defer f.mu.Unlock()
if len(f.free) == 0 {
return -1, fmt.Errorf("empty")
}
x, newFree := f.free[len(f.free)-1], f.free[:len(f.free)-1]
f.free = newFree
return x, nil
}
Free nodes stack using sync.Mutex
Exclusive lock
Exclusive lock
Slide 13
Slide 13 text
13
type Maps struct {
mu *sync.RWMutex
keyToId map[int]int
}
func (m *Maps) add(id, key int) {
m.mu.Lock()
defer m.mu.Unlock()
m.keyToId[key] = id
}
func (m Maps) getId(key int) (int, error) {
m.mu.RLock()
defer m.mu.RUnlock()
if id, ok := m.keyToId[key]; !ok {
return -1, fmt.Errorf("not found")
} else {
return id, nil
}
}
Mapping id and key using sync.RWMutex
Exclusive lock
Shared lock
Slide 14
Slide 14 text
• Mutex and RWMutex must not be copied after first use.
14
Caution
type Maps struct {
mu sync.RWMutex
keyToId map[int]int
}
func (m Maps) getId(key int) (int, error) {
fmt.Printf("mu address in getId: %p\n", &m.mu)
m.mu.RLock()
defer m.mu.RUnlock()
if id, ok := m.keyToId[key]; !ok {
return -1, fmt.Errorf("not found")
} else {
return id, nil
}
}
If type of member isn’t a pointer
receiver isn’t a pointer
$ go run -race main.go
mu address in getId: 0xc420010110
mu address in getId: 0xc4200781c0
==================
WARNING: DATA RACE
wg.Add(4)
for i := 0; i < 2; i++ {
go func(i int) {
maps.add(i, i)
wg.Done()
}(i)
go func(i int) {
maps.getId(i)
wg.Done()
}(i)
}
wg.Wait()
and
Slide 15
Slide 15 text
3.
Resource locking
Slide 16
Slide 16 text
Many goroutines access one file
concurrently.
Slide 17
Slide 17 text
Data structure of Gannoy
17
Hyperplane
Logical data structure Data structure on file system
Bucket Bucket
/PEF
'FBUVSFT
/PEF
'FBUVSFT
/PEF
'FBUVSFT
/PEF
'FBUVSFT
Node
(Features)
/PEF
'FBUVSFT
/PEF
'FBUVSFT
/PEF
'FBUVSFT
/PEF
'FBUVSFT
Node
(Features)
query
features
cosine similarity > 0
<= 0
B N
H N
free (1byte)
number of descendants (4byte)
key (4byte)
parents (4byte * tree)
children (4byte * K)
or
hyperplane / features (8byte* dim)
…
Slide 18
Slide 18 text
• syscall.Flock
• apply or remove an advisory lock on an open file.
• syscall.FcntlFlock
• file control. F_SETLK, F_SETLKW, and F_GETLK are used to acquire,
release, and test for the existence of record locks.
18
Resource locking in Golang (using system call)
locks associated with
Flock file open file description
Fcntl byte range process ID and i-node
Slide 19
Slide 19 text
19
// Flock
func flock(fd uintptr) {
syscall.Flock(int(fd), syscall.LOCK_EX) // or LOCK_SH
defer syscall.Flock(int(fd), syscall.LOCK_UN)
}
// Fcntl + F_SETLKW
func fcntl(fd uintptr, start, len int64) {
syscall.FcntlFlock(fd, syscall.F_SETLKW, &syscall.Flock_t{
Start: start,
Len: len,
Type: syscall.F_WRLCK, // or syscall.F_RDLCK
Whence: io.SeekStart,
})
defer syscall.FcntlFlock(fd, syscall.F_SETLKW, &syscall.Flock_t{
Start: start,
Len: len,
Type: syscall.F_UNLCK,
Whence: io.SeekStart,
})
}
Flock
Shared or Exclusive lock
Fcntl + F_SETLKW
Shared or Exclusive lock
byte range
Slide 20
Slide 20 text
20
file descriptor
fd0
fd1
fd2
Open file description
0
1
2
10
20
i-node
1000
1001
1002
2000
3000
file descriptor
fd0
fd1
fd2
fd10
dup
fork
Process System System
Goroutine
Goroutine
OFD
0
2
ProcessID i-node
100 1000
101 1001
Flock Fcntl + F_SETLKW
Locks that associated with same
one don’t block each other.
Slide 21
Slide 21 text
21
file1, _ := os.OpenFile("dummy.txt", os.O_RDWR, 0)
file2, _ := os.OpenFile("dummy.txt", os.O_RDWR, 0)
fd1 := file1.Fd()
fd2 := file2.Fd()
var wg sync.WaitGroup
wg.Add(2)
locker := gannoy.Flock{}
go lockAndProcess("fd1", fd1, locker, &wg)
go lockAndProcess("fd2", fd2, locker, &wg)
wg.Wait()
fd2 processed something (1sec).
fd2 processed something (2sec).
fd2 processed something (3sec).
fd1 processed something (1sec).
fd1 processed something (2sec).
fd1 processed something (3sec).
locker := gannoy.Fcntl{}
fd2 processed something (1sec).
fd1 processed something (1sec).
fd2 processed something (2sec).
fd1 processed something (2sec).
fd2 processed something (3sec).
fd1 processed something (3sec).
No blocking
Blocking
Flock
Fcntl
+
F_SETLKW
Slide 22
Slide 22 text
22
func lockAndProcess(name string, fd uintptr, locker gannoy.Locker, wg *sync.WaitGroup) {
locker.WriteLock(fd, 0, 4)
defer locker.UnLock(fd, 0, 4)
t := time.NewTicker(1 * time.Second)
count := 1
loop:
for {
select {
case <-t.C:
fmt.Printf("%s processed something (%d sec).\n", name, count)
count++
if count > 3 {
break loop
}
}
}
t.Stop()
wg.Done()
}
Slide 23
Slide 23 text
Open file description lock
open file description locks are associated with the open
file description on which they are acquired,
http://man7.org/linux/man-pages/man2/fcntl.2.html
❝
https://www.oreilly.co.jp/community/blog/2014/11/file-lock-and-new-ofd-lock.html
Slide 24
Slide 24 text
24
type Fcntl struct {
}
const F_OFD_SETLKW = 38
func (f Fcntl) ReadLock(fd uintptr, start, len int64) error {
return f.fcntl(syscall.F_RDLCK, fd, start, len)
}
func (f Fcntl) WriteLock(fd uintptr, start, len int64) error {
return f.fcntl(syscall.F_WRLCK, fd, start, len)
}
func (f Fcntl) UnLock(fd uintptr, start, len int64) error {
return f.fcntl(syscall.F_UNLCK, fd, start, len)
}
func (f Fcntl) fcntl(typ int16, fd uintptr, start, len int64) error {
return syscall.FcntlFlock(fd, F_OFD_SETLKW, &syscall.Flock_t{
Start: start,
Len: len,
Type: typ,
Whence: io.SeekStart,
})
}
Command for working with
open file description locks
Exclusive lock
Shared lock
Slide 25
Slide 25 text
25
file1, _ := os.OpenFile("dummy.txt", os.O_RDWR, 0)
file2, _ := os.OpenFile("dummy.txt", os.O_RDWR, 0)
fd1 := file1.Fd()
fd2 := file2.Fd()
var wg sync.WaitGroup
wg.Add(2)
locker := gannoy.Flock{}
go lockAndProcess("fd1", fd1, locker, &wg)
go lockAndProcess("fd2", fd2, locker, &wg)
wg.Wait()
fd2 processed something (1sec).
fd2 processed something (2sec).
fd2 processed something (3sec).
fd1 processed something (1sec).
fd1 processed something (2sec).
fd1 processed something (3sec).
locker := gannoy.Fcntl{}
Blocking!!
Blocking
Flock
Fcntl
+
F_ODF_SETLKW
fd2 processed something (1sec).
fd2 processed something (2sec).
fd2 processed something (3sec).
fd1 processed something (1sec).
fd1 processed something (2sec).
fd1 processed something (3sec).
Slide 26
Slide 26 text
1. non-POSIX (Only Linux)
2. kernel >= 3.15.0 (CentOS7.2 is 3.10.0…)
26
Caution
func newLocker() Locker {
bytes, err := exec.Command("uname", "-sr").Output()
if err != nil {
return Flock{}
}
kernel := strings.Split(strings.TrimRight(string(bytes), "\n"), " ")
if len(kernel) != 2 {
return Flock{}
}
if kernel[0] == "Linux" && !semver.New(kernel[1]).LessThan(*semver.New("3.15.0")) {
return Fcntl{}
}
return Flock{}
}
kernel >= 3.15.0
Slide 27
Slide 27 text
ݚڀһɺੵۃతʹืूதʂ
http://rand.pepabo.com/