Go in production - case of Gannoy -

fukuokago#9
https://connpass.com/event/68358/

Transcript

1. - case of Gannoy -
   ࡾ୐༔հ / Pepabo R&D Institute, GMO Pepabo, Inc.
   2017.10.17 Fukuoka.go#9
   Go in production
   

   View Slide

2. ϓϦϯγύϧΤϯδχΞ
   ࡾ୐ ༔հ / @monochromegane
   2
   http://blog.monochromegane.com
   Yusuke Miyake
   ϖύϘݚڀॴ ݚڀһ
   

   View Slide

3. 1. Goຊ൪ӡ༻ͷཁ݅
   2. Goຊ൪ӡ༻ͷ࣮ݱ
   3. ·ͱΊ
   3
   ໨࣍
   

   View Slide

4. 1.
   GannoyʹΈΔ
   Goຊ൪ӡ༻ͷཁ݅
   

   View Slide

5. Gannoy
   Approximate nearest neighbor search server
   and dynamic index written in Golang.
   https://github.com/monochromegane/gannoy
   

   View Slide

6. Similar images search system using Gannoy
   6
   Features Similar items
   Gannoy
   [2048]float64
   query
   by http
   find similar features
   mapping
   similar features
   to items
   response
   Deep CNN
   index
   Features
   [2048]float64
   Deep CNN
   register
   by http
   

   View Slide

7. • HTTP based Approximate nearest neighbor search server
   • Using labstack/echo
   • Go 1.8+
   7
   Gannoy
   

   View Slide

8. Goຊ൪ӡ༻ͷཁ݅
   

   View Slide

9. • Graceful restart
   • Daemonize
   • Monitoring
   • Log rotation
   9
   Goຊ൪ӡ༻ͷཁ݅
   • Configuration
   • Build
   • Deploy
   

   View Slide

10. 2.
    GannoyʹΈΔ
    Goຊ൪ӡ༻ͷ࣮ݱ
    

    View Slide

11. Graceful restart
    11
    go-server-starter + http.Server#Shutdown()
    import "github.com/lestrrat/go-server-starter/listener"
    e := echo.New()
    listeners, err := listener.ListenAll()
    if err != nil && err != listener.ErrNoListeningTarget {
    fmt.Fprintln(os.Stderr, err)
    os.Exit(1)
    }
    e.Listener = netutil.LimitListener(listeners[0], opts.MaxConnections)
    Listen from start_server socket.
    

    View Slide

12. Graceful restart
    12
    go-server-starter + http.Server#Shutdown()
    go func() {
    if err := e.Start(""); err != nil {
    e.Logger.Info("Shutting down the server")
    }
    }()
    sigCh := make(chan os.Signal, 1)
    signal.Notify(sigCh, sig)
    <-sigCh
    ctx, cancel := context.WithTimeout(context.Background(),
    time.Duration(opts.ShutDownTimeout)*time.Second)
    defer cancel()
    if err := e.Shutdown(ctx); err != nil {
    e.Logger.Fatal(err)
    }
    Start server as goroutines.
    Wait SIGTERM.
    Graceful shutdown using http.Server.
    

    View Slide

13. Daemonize
    13
    Systemd
    [Unit]
    Description=Gannoy DB
    ConditionPathExists=/usr/bin/start_server
    ConditionPathExists=/usr/bin/gannoy-db
    After=network.target
    [Service]
    User=gannoy
    Group=gannoy
    Type=simple
    PIDFile=/var/run/gannoy/server_starter.pid
    ExecStart=/usr/bin/start_server --port 8080 --pid-file /var/run/gannoy/server_starter.pid -- /
    usr/bin/gannoy-db -c /etc/gannoy/gannoy-db.toml
    ExecReload=/bin/kill -HUP $MAINPID
    ExecStop=/bin/kill -TERM $MAINPID
    [Install]
    WantedBy=multi-user.target
    gannoy-db.service
    

    View Slide

14. Monitoring
    14
    health-check endpoint + Consul
    e.GET("/health", func(c echo.Context) error {
    return c.NoContent(http.StatusOK)
    })
    {
    "service": {
    (snip)
    "port": 80,
    "checks": [
    {
    "script": "/usr/bin/check-http -u http://localhost/health",
    "interval": "30s"
    }
    ]
    }
    }
    health-check endpoint
    consul.d/gannoy.json
    

    View Slide

15. Log ratation
    15
    logrotate
    /var/log/gannoy/gannoy-db.log {
    daily
    rotate 7
    missingok
    notifempty
    compress
    postrotate
    kill -HUP `cat /var/run/gannoy/server_starter.pid`
    endscript
    }
    gannoy-db.logrotate
    reopen logfile.
    

    View Slide

16. Configuration
    16
    monochromegane/conflag
    // Parse option from args and configuration file.
    conflag.LongHyphen = true
    conflag.BoolValue = false
    parser := flags.NewParser(&opts, flags.Default)
    _, err := parser.ParseArgs(os.Args[1:])
    if err != nil {
    os.Exit(1)
    }
    if opts.Config != "" {
    if args, err := conflag.ArgsFrom(opts.Config); err == nil {
    if _, err := parser.ParseArgs(args); err != nil {
    os.Exit(1)
    }
    }
    }
    _, err = parser.ParseArgs(os.Args[1:])
    if err != nil {
    os.Exit(1)
    }
    Read option from config.
    Overwrite option from args.
    data-dir = "/var/lib/gannoy"
    log-dir = "/var/log/gannoy"
    lock-dir = "/var/run/gannoy"
    server-starter = true
    auto-save = true
    gannoy-db —data-dir=“data”
    

    View Slide

17. Configuration
    17
    monochromegane/conflag
    http://blog.monochromegane.com/blog/2015/05/30/conflag/
    

    View Slide

18. Build
    18
    rpmbuild + docker-compose
    .
    !"" Dockerfile.rpmbuild
    !"" docker-compose.yml
    !"" rpmbuild
    # !"" RPMS
    # # $"" x86_64
    # # $"" gannoy-0.0.1-8.x86_64.rpm
    # !"" SOURCES
    # # !"" gannoy-db.logrotate
    # # !"" gannoy-db.service
    # # $"" gannoy-db.toml
    # $"" SPECS
    # $"" gannoy.spec
    $"" version.go
    version: '2'
    services:
    gannoy-rpmbuild:
    build:
    context: .
    dockerfile: Dockerfile.rpmbuild
    command: cp /root/rpmbuild/RPMS/x86_64/
    gannoy-0.0.1-8.x86_64.rpm /tmp/rpmbuild/RPMS/
    x86_64/.
    volumes:
    - .:/tmp:rw
    RUN rpmbuild -bb /root/rpmbuild/SPECS/gannoy.spec
    $ docker-compose build gannoy-rpmbuild
    $ docker-compose run gannoy-rpmbuild
    docker-compose.yml
    Dockerfile.rpmbuild
    

    View Slide

19. Deploy (and test)
    19
    Puppet + Serverspec
    class gannoy::install {
    include ::go_start_server
    $rpm = 'gannoy-0.0.1-8.x86_64.rpm'
    file { "/usr/local/src/${rpm}":
    ensure => present,
    source => "puppet:///modules/gannoy/${rpm}",
    mode => '0755',
    owner => 'root',
    notify => Package['gannoy'],
    } ->
    package { 'gannoy':
    ensure => '0.0.1-8',
    source => "/usr/local/src/${rpm}",
    provider => rpm,
    }
    (snip)
    }
    (snip)
    describe service('gannoy-db') do
    it { should be_enabled }
    end
    describe package('gannoy') do
    it { should
    be_installed.with_version('0.0.1-8') }
    end
    (snip)
    

    View Slide

20. ·ͱΊ
    

    View Slide

21. • σʔϞϯԽ͕ඞཁͳGoΞϓϦͷຊ൪ӡ༻͸όΠφϦҎ֎ʹ΋ඞཁͳ
    ϑΝΠϧ͕͋ΔͷͰύοέʔδϯάͨ͠΄͏͕औΓճָ͕͠
    • ΦϓγϣϯͱઃఆϑΝΠϧͷڞ௨Խʹ͸conflag͕ศར
    • ϕετϓϥΫςΟε͸໛ࡧதɻσΟεΧογϣϯ͠·͠ΐ͏ɻ
    21
    ·ͱΊ
    

    View Slide

22. ݚڀһɺੵۃతʹืूதʂ
    http://rand.pepabo.com/
    

    View Slide