Upgrade to Pro — share decks privately, control downloads, hide ads and more …

Bleve - London 2015

Marty Schoch
November 12, 2015

Bleve - London 2015

Bleve: the Couchbase-sponsored Go full text search and indexing engine.

Marty Schoch

November 12, 2015
Tweet

More Decks by Marty Schoch

Other Decks in Technology

Transcript

  1. 2 Why? • Lucene/Solr/Elasticsearch are awesome • Could we build

    50% of Lucene's text analysis, combine it with off-the- shelf KV stores and get something interesting?
  2. 3 Bleve Core Ideas • Text Analysis Pipeline • We

    only have to build common core • Users customize for domain/language through interfaces • Pluggable KV storage • No custom file format • Plug-in Bolt, LevelDB, ForestDB, etc • Search • Make term search work • Almost everything else built on top of that...
  3. 7 Import import "github.com/blevesearch/bleve" type Person struct { Name string

    } func main() { mapping := bleve.NewIndexMapping() index, err := bleve.New("people.bleve", mapping) if err != nil { log.Fatal(err) } person := Person{"Marty Schoch"} err = index.Index("m1", person) if err != nil { log.Fatal(err) } fmt.Println("Indexed Document") } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
  4. 8 Data Model import "github.com/blevesearch/bleve" type Person struct { Name

    string } func main() { mapping := bleve.NewIndexMapping() index, err := bleve.New("people.bleve", mapping) if err != nil { log.Fatal(err) } person := Person{"Marty Schoch"} err = index.Index("m1", person) if err != nil { log.Fatal(err) } fmt.Println("Indexed Document") } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
  5. 9 Index Mapping import "github.com/blevesearch/bleve" type Person struct { Name

    string } func main() { mapping := bleve.NewIndexMapping() index, err := bleve.New("people.bleve", mapping) if err != nil { log.Fatal(err) } person := Person{"Marty Schoch"} err = index.Index("m1", person) if err != nil { log.Fatal(err) } fmt.Println("Indexed Document") } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
  6. 10 Create New Index import "github.com/blevesearch/bleve" type Person struct {

    Name string } func main() { mapping := bleve.NewIndexMapping() index, err := bleve.New("people.bleve", mapping) if err != nil { log.Fatal(err) } person := Person{"Marty Schoch"} err = index.Index("m1", person) if err != nil { log.Fatal(err) } fmt.Println("Indexed Document") } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
  7. 11 Index Data import "github.com/blevesearch/bleve" type Person struct { Name

    string } func main() { mapping := bleve.NewIndexMapping() index, err := bleve.New("people.bleve", mapping) if err != nil { log.Fatal(err) } person := Person{"Marty Schoch"} err = index.Index("m1", person) if err != nil { log.Fatal(err) } fmt.Println("Indexed Document") } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
  8. 12 Index Data import "github.com/blevesearch/bleve" type Person struct { Name

    string } func main() { mapping := bleve.NewIndexMapping() index, err := bleve.New("people.bleve", mapping) if err != nil { log.Fatal(err) } person := Person{"Marty Schoch"} err = index.Index("m1", person) if err != nil { log.Fatal(err) } fmt.Println("Indexed Document") } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 Indexed Document Program exited.
  9. 13 Open Index func main() { index, err := bleve.Open("people.bleve")

    if err != nil { log.Fatal(err) } query := bleve.NewTermQuery("marty") request := bleve.NewSearchRequest(query) result, err := index.Search(request) if err != nil { log.Fatal(err) } fmt.Println(result) } 1 2 3 4 5 6 7 8 9 10 11 12 13 14
  10. 14 Build Query func main() { index, err := bleve.Open("people.bleve")

    if err != nil { log.Fatal(err) } query := bleve.NewTermQuery("marty") request := bleve.NewSearchRequest(query) result, err := index.Search(request) if err != nil { log.Fatal(err) } fmt.Println(result) } 1 2 3 4 5 6 7 8 9 10 11 12 13 14
  11. 15 Build Request func main() { index, err := bleve.Open("people.bleve")

    if err != nil { log.Fatal(err) } query := bleve.NewTermQuery("marty") request := bleve.NewSearchRequest(query) result, err := index.Search(request) if err != nil { log.Fatal(err) } fmt.Println(result) } 1 2 3 4 5 6 7 8 9 10 11 12 13 14
  12. 16 Search func main() { index, err := bleve.Open("people.bleve") if

    err != nil { log.Fatal(err) } query := bleve.NewTermQuery("marty") request := bleve.NewSearchRequest(query) result, err := index.Search(request) if err != nil { log.Fatal(err) } fmt.Println(result) } 1 2 3 4 5 6 7 8 9 10 11 12 13 14
  13. 17 Search func main() { index, err := bleve.Open("people.bleve") if

    err != nil { log.Fatal(err) } query := bleve.NewTermQuery("marty") request := bleve.NewSearchRequest(query) result, err := index.Search(request) if err != nil { log.Fatal(err) } fmt.Println(result) } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 1 matches, showing 1 through 1, took 24.713µs 1. m1 (0.216978) Program exited.
  14. 19 Conference Schedule type Event struct { UID string `json:"uid"`

    Summary string `json:"summary"` Description string `json:"description"` Speaker string `json:"speaker"` Start time.Time `json:"start"` Duration float64 `json:"duration"` } 1 2 3 4 5 6 7 8
  15. 20 Phrase Search func main() { index, err := bleve.Open("gopherconin.bleve")

    if err != nil { log.Fatal(err) } phrase := []string{"quality", "search", "results"} q := bleve.NewPhraseQuery(phrase, "description") req := bleve.NewSearchRequest(q) req.Highlight = bleve.NewHighlightWithStyle("html") req.Fields = []string{"summary", "speaker"} res, err := index.Search(req) if err != nil { log.Fatal(err) } fmt.Println(res) } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
  16. 21 Phrase Search func main() { index, err := bleve.Open("gopherconin.bleve")

    if err != nil { log.Fatal(err) } phrase := []string{"quality", "search", "results"} q := bleve.NewPhraseQuery(phrase, "description") req := bleve.NewSearchRequest(q) req.Highlight = bleve.NewHighlightWithStyle("html") req.Fields = []string{"summary", "speaker"} res, err := index.Search(req) if err != nil { log.Fatal(err) } fmt.Println(res) } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 1 matches, showing 1 through 1, took 46.134µs 1. bleve_-_modern_text_indexing_for_go (1.033644) description …earch component. But delivering high quality search results requires a long list of text analysis and indexing techniques. With the bleve library, we bring advanced text indexing and search to your Go… summary bleve - modern text indexing for Go speaker Martin Schoch Program exited.
  17. 22 Query String func main() { index, err := bleve.Open("gopherconin.bleve")

    if err != nil { log.Fatal(err) } qString := `+description:text ` qString += `summary:"text indexing" ` qString += `summary:believe~2 ` qString += `-description:lucene ` qString += `duration:<30` q := bleve.NewQueryStringQuery(qString) req := bleve.NewSearchRequest(q) req.Highlight = bleve.NewHighlightWithStyle("html") req.Fields = []string{"summary", "speaker", "description", "duration"} res, err := index.Search(req) if err != nil { log.Fatal(err) } fmt.Println(res) } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
  18. 23 Query String func main() { index, err := bleve.Open("gopherconin.bleve")

    if err != nil { log.Fatal(err) } qString := `+description:text ` qString += `summary:"text indexing" ` qString += `summary:believe~2 ` qString += `-description:lucene ` qString += `duration:<30` q := bleve.NewQueryStringQuery(qString) req := bleve.NewSearchRequest(q) req.Highlight = bleve.NewHighlightWithStyle("html") req.Fields = []string{"summary", "speaker", "description", "duration"} res, err := index.Search(req) if err != nil { log.Fatal(err) } fmt.Println(res) } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 1 matches, showing 1 through 1, took 220.08µs 1. bleve_-_modern_text_indexing_for_go (0.338882) summary bleve - modern text indexing for Go description …ist of text analysis and indexing techniques. With the bleve library, we bring advanced text indexing and search to your Go applications. This talk will start with a brief introduction to text search … speaker Martin Schoch duration 25
  19. 24 HTTP Handlers (optional) import "github.com/blevesearch/bleve/http" • All major bleve

    operations mapped • Assume JSON document bodies • See bleve-explorer sample app https://github.com/blevesearch/bleve-explorer
  20. 29 Community • #bleve is small/quiet room, talk to us

    real time • Discuss your use-case • Plan a feature implementation • Apache License v2.0 • Report Issues, Submit Pull Requests