Upgrade to Pro — share decks privately, control downloads, hide ads and more …

Bleve - London 2015

16cdfb0c4af5297e261cb36e30fa5c20?s=47 Marty Schoch
November 12, 2015

Bleve - London 2015

Bleve: the Couchbase-sponsored Go full text search and indexing engine.

16cdfb0c4af5297e261cb36e30fa5c20?s=128

Marty Schoch

November 12, 2015
Tweet

More Decks by Marty Schoch

Other Decks in Technology

Transcript

  1. Bleve modern text indexing for Go Meetup

  2. 2 Why? • Lucene/Solr/Elasticsearch are awesome • Could we build

    50% of Lucene's text analysis, combine it with off-the- shelf KV stores and get something interesting?
  3. 3 Bleve Core Ideas • Text Analysis Pipeline • We

    only have to build common core • Users customize for domain/language through interfaces • Pluggable KV storage • No custom file format • Plug-in Bolt, LevelDB, ForestDB, etc • Search • Make term search work • Almost everything else built on top of that...
  4. 4 Bleve Extension Points Bleve Analysis Index Search Tokenizers Token

    Filters KV Storage Searcher
  5. Getting Started

  6. 6 Install Bleve go get github.com/blevesearch/bleve/...

  7. 7 Import import "github.com/blevesearch/bleve" type Person struct { Name string

    } func main() { mapping := bleve.NewIndexMapping() index, err := bleve.New("people.bleve", mapping) if err != nil { log.Fatal(err) } person := Person{"Marty Schoch"} err = index.Index("m1", person) if err != nil { log.Fatal(err) } fmt.Println("Indexed Document") } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
  8. 8 Data Model import "github.com/blevesearch/bleve" type Person struct { Name

    string } func main() { mapping := bleve.NewIndexMapping() index, err := bleve.New("people.bleve", mapping) if err != nil { log.Fatal(err) } person := Person{"Marty Schoch"} err = index.Index("m1", person) if err != nil { log.Fatal(err) } fmt.Println("Indexed Document") } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
  9. 9 Index Mapping import "github.com/blevesearch/bleve" type Person struct { Name

    string } func main() { mapping := bleve.NewIndexMapping() index, err := bleve.New("people.bleve", mapping) if err != nil { log.Fatal(err) } person := Person{"Marty Schoch"} err = index.Index("m1", person) if err != nil { log.Fatal(err) } fmt.Println("Indexed Document") } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
  10. 10 Create New Index import "github.com/blevesearch/bleve" type Person struct {

    Name string } func main() { mapping := bleve.NewIndexMapping() index, err := bleve.New("people.bleve", mapping) if err != nil { log.Fatal(err) } person := Person{"Marty Schoch"} err = index.Index("m1", person) if err != nil { log.Fatal(err) } fmt.Println("Indexed Document") } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
  11. 11 Index Data import "github.com/blevesearch/bleve" type Person struct { Name

    string } func main() { mapping := bleve.NewIndexMapping() index, err := bleve.New("people.bleve", mapping) if err != nil { log.Fatal(err) } person := Person{"Marty Schoch"} err = index.Index("m1", person) if err != nil { log.Fatal(err) } fmt.Println("Indexed Document") } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
  12. 12 Index Data import "github.com/blevesearch/bleve" type Person struct { Name

    string } func main() { mapping := bleve.NewIndexMapping() index, err := bleve.New("people.bleve", mapping) if err != nil { log.Fatal(err) } person := Person{"Marty Schoch"} err = index.Index("m1", person) if err != nil { log.Fatal(err) } fmt.Println("Indexed Document") } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 Indexed Document Program exited.
  13. 13 Open Index func main() { index, err := bleve.Open("people.bleve")

    if err != nil { log.Fatal(err) } query := bleve.NewTermQuery("marty") request := bleve.NewSearchRequest(query) result, err := index.Search(request) if err != nil { log.Fatal(err) } fmt.Println(result) } 1 2 3 4 5 6 7 8 9 10 11 12 13 14
  14. 14 Build Query func main() { index, err := bleve.Open("people.bleve")

    if err != nil { log.Fatal(err) } query := bleve.NewTermQuery("marty") request := bleve.NewSearchRequest(query) result, err := index.Search(request) if err != nil { log.Fatal(err) } fmt.Println(result) } 1 2 3 4 5 6 7 8 9 10 11 12 13 14
  15. 15 Build Request func main() { index, err := bleve.Open("people.bleve")

    if err != nil { log.Fatal(err) } query := bleve.NewTermQuery("marty") request := bleve.NewSearchRequest(query) result, err := index.Search(request) if err != nil { log.Fatal(err) } fmt.Println(result) } 1 2 3 4 5 6 7 8 9 10 11 12 13 14
  16. 16 Search func main() { index, err := bleve.Open("people.bleve") if

    err != nil { log.Fatal(err) } query := bleve.NewTermQuery("marty") request := bleve.NewSearchRequest(query) result, err := index.Search(request) if err != nil { log.Fatal(err) } fmt.Println(result) } 1 2 3 4 5 6 7 8 9 10 11 12 13 14
  17. 17 Search func main() { index, err := bleve.Open("people.bleve") if

    err != nil { log.Fatal(err) } query := bleve.NewTermQuery("marty") request := bleve.NewSearchRequest(query) result, err := index.Search(request) if err != nil { log.Fatal(err) } fmt.Println(result) } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 1 matches, showing 1 through 1, took 24.713µs 1. m1 (0.216978) Program exited.
  18. More Realistic Data

  19. 19 Conference Schedule type Event struct { UID string `json:"uid"`

    Summary string `json:"summary"` Description string `json:"description"` Speaker string `json:"speaker"` Start time.Time `json:"start"` Duration float64 `json:"duration"` } 1 2 3 4 5 6 7 8
  20. 20 Phrase Search func main() { index, err := bleve.Open("gopherconin.bleve")

    if err != nil { log.Fatal(err) } phrase := []string{"quality", "search", "results"} q := bleve.NewPhraseQuery(phrase, "description") req := bleve.NewSearchRequest(q) req.Highlight = bleve.NewHighlightWithStyle("html") req.Fields = []string{"summary", "speaker"} res, err := index.Search(req) if err != nil { log.Fatal(err) } fmt.Println(res) } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
  21. 21 Phrase Search func main() { index, err := bleve.Open("gopherconin.bleve")

    if err != nil { log.Fatal(err) } phrase := []string{"quality", "search", "results"} q := bleve.NewPhraseQuery(phrase, "description") req := bleve.NewSearchRequest(q) req.Highlight = bleve.NewHighlightWithStyle("html") req.Fields = []string{"summary", "speaker"} res, err := index.Search(req) if err != nil { log.Fatal(err) } fmt.Println(res) } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 1 matches, showing 1 through 1, took 46.134µs 1. bleve_-_modern_text_indexing_for_go (1.033644) description …earch component. But delivering high quality search results requires a long list of text analysis and indexing techniques. With the bleve library, we bring advanced text indexing and search to your Go… summary bleve - modern text indexing for Go speaker Martin Schoch Program exited.
  22. 22 Query String func main() { index, err := bleve.Open("gopherconin.bleve")

    if err != nil { log.Fatal(err) } qString := `+description:text ` qString += `summary:"text indexing" ` qString += `summary:believe~2 ` qString += `-description:lucene ` qString += `duration:<30` q := bleve.NewQueryStringQuery(qString) req := bleve.NewSearchRequest(q) req.Highlight = bleve.NewHighlightWithStyle("html") req.Fields = []string{"summary", "speaker", "description", "duration"} res, err := index.Search(req) if err != nil { log.Fatal(err) } fmt.Println(res) } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
  23. 23 Query String func main() { index, err := bleve.Open("gopherconin.bleve")

    if err != nil { log.Fatal(err) } qString := `+description:text ` qString += `summary:"text indexing" ` qString += `summary:believe~2 ` qString += `-description:lucene ` qString += `duration:<30` q := bleve.NewQueryStringQuery(qString) req := bleve.NewSearchRequest(q) req.Highlight = bleve.NewHighlightWithStyle("html") req.Fields = []string{"summary", "speaker", "description", "duration"} res, err := index.Search(req) if err != nil { log.Fatal(err) } fmt.Println(res) } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 1 matches, showing 1 through 1, took 220.08µs 1. bleve_-_modern_text_indexing_for_go (0.338882) summary bleve - modern text indexing for Go description …ist of text analysis and indexing techniques. With the bleve library, we bring advanced text indexing and search to your Go applications. This talk will start with a brief introduction to text search … speaker Martin Schoch duration 25
  24. 24 HTTP Handlers (optional) import "github.com/blevesearch/bleve/http" • All major bleve

    operations mapped • Assume JSON document bodies • See bleve-explorer sample app https://github.com/blevesearch/bleve-explorer
  25. Putting It All Together

  26. 26 Search Hugo Sites

  27. 27 Hugo Site Demo

  28. Join the Community

  29. 29 Community • #bleve is small/quiet room, talk to us

    real time • Discuss your use-case • Plan a feature implementation • Apache License v2.0 • Report Issues, Submit Pull Requests
  30. 30 Contributors

  31. 31 Roadmap • Performance • Prepare for 1.0 Release

  32. 32 Thanks • Marty Schoch • marty@couchbase.com • http://github.com/blevesearch/bleve •

    @mschoch • @blevesearch