Bleve modern text indexing for Go Meetup

2 Why? • Lucene/Solr/Elasticsearch are awesome • Could we build 50% of Lucene's text analysis, combine it with off-the- shelf KV stores and get something interesting?

3 Bleve Core Ideas • Text Analysis Pipeline • We only have to build common core • Users customize for domain/language through interfaces • Pluggable KV storage • No custom file format • Plug-in Bolt, LevelDB, ForestDB, etc • Search • Make term search work • Almost everything else built on top of that...

4 Bleve Extension Points Bleve Analysis Index Search Tokenizers Token Filters KV Storage Searcher

Getting Started

6 Install Bleve go get github.com/blevesearch/bleve/...

7 Import import "github.com/blevesearch/bleve" type Person struct { Name string } func main() { mapping := bleve.NewIndexMapping() index, err := bleve.New("people.bleve", mapping) if err != nil { log.Fatal(err) } person := Person{"Marty Schoch"} err = index.Index("m1", person) if err != nil { log.Fatal(err) } fmt.Println("Indexed Document") } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20

8 Data Model import "github.com/blevesearch/bleve" type Person struct { Name string } func main() { mapping := bleve.NewIndexMapping() index, err := bleve.New("people.bleve", mapping) if err != nil { log.Fatal(err) } person := Person{"Marty Schoch"} err = index.Index("m1", person) if err != nil { log.Fatal(err) } fmt.Println("Indexed Document") } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20

9 Index Mapping import "github.com/blevesearch/bleve" type Person struct { Name string } func main() { mapping := bleve.NewIndexMapping() index, err := bleve.New("people.bleve", mapping) if err != nil { log.Fatal(err) } person := Person{"Marty Schoch"} err = index.Index("m1", person) if err != nil { log.Fatal(err) } fmt.Println("Indexed Document") } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20

10 Create New Index import "github.com/blevesearch/bleve" type Person struct { Name string } func main() { mapping := bleve.NewIndexMapping() index, err := bleve.New("people.bleve", mapping) if err != nil { log.Fatal(err) } person := Person{"Marty Schoch"} err = index.Index("m1", person) if err != nil { log.Fatal(err) } fmt.Println("Indexed Document") } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20

11 Index Data import "github.com/blevesearch/bleve" type Person struct { Name string } func main() { mapping := bleve.NewIndexMapping() index, err := bleve.New("people.bleve", mapping) if err != nil { log.Fatal(err) } person := Person{"Marty Schoch"} err = index.Index("m1", person) if err != nil { log.Fatal(err) } fmt.Println("Indexed Document") } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20

12 Index Data import "github.com/blevesearch/bleve" type Person struct { Name string } func main() { mapping := bleve.NewIndexMapping() index, err := bleve.New("people.bleve", mapping) if err != nil { log.Fatal(err) } person := Person{"Marty Schoch"} err = index.Index("m1", person) if err != nil { log.Fatal(err) } fmt.Println("Indexed Document") } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 Indexed Document Program exited.

13 Open Index func main() { index, err := bleve.Open("people.bleve") if err != nil { log.Fatal(err) } query := bleve.NewTermQuery("marty") request := bleve.NewSearchRequest(query) result, err := index.Search(request) if err != nil { log.Fatal(err) } fmt.Println(result) } 1 2 3 4 5 6 7 8 9 10 11 12 13 14

14 Build Query func main() { index, err := bleve.Open("people.bleve") if err != nil { log.Fatal(err) } query := bleve.NewTermQuery("marty") request := bleve.NewSearchRequest(query) result, err := index.Search(request) if err != nil { log.Fatal(err) } fmt.Println(result) } 1 2 3 4 5 6 7 8 9 10 11 12 13 14

15 Build Request func main() { index, err := bleve.Open("people.bleve") if err != nil { log.Fatal(err) } query := bleve.NewTermQuery("marty") request := bleve.NewSearchRequest(query) result, err := index.Search(request) if err != nil { log.Fatal(err) } fmt.Println(result) } 1 2 3 4 5 6 7 8 9 10 11 12 13 14

16 Search func main() { index, err := bleve.Open("people.bleve") if err != nil { log.Fatal(err) } query := bleve.NewTermQuery("marty") request := bleve.NewSearchRequest(query) result, err := index.Search(request) if err != nil { log.Fatal(err) } fmt.Println(result) } 1 2 3 4 5 6 7 8 9 10 11 12 13 14

17 Search func main() { index, err := bleve.Open("people.bleve") if err != nil { log.Fatal(err) } query := bleve.NewTermQuery("marty") request := bleve.NewSearchRequest(query) result, err := index.Search(request) if err != nil { log.Fatal(err) } fmt.Println(result) } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 1 matches, showing 1 through 1, took 24.713µs 1. m1 (0.216978) Program exited.

More Realistic Data

19 Conference Schedule type Event struct { UID string `json:"uid"` Summary string `json:"summary"` Description string `json:"description"` Speaker string `json:"speaker"` Start time.Time `json:"start"` Duration float64 `json:"duration"` } 1 2 3 4 5 6 7 8

20 Phrase Search func main() { index, err := bleve.Open("gopherconin.bleve") if err != nil { log.Fatal(err) } phrase := []string{"quality", "search", "results"} q := bleve.NewPhraseQuery(phrase, "description") req := bleve.NewSearchRequest(q) req.Highlight = bleve.NewHighlightWithStyle("html") req.Fields = []string{"summary", "speaker"} res, err := index.Search(req) if err != nil { log.Fatal(err) } fmt.Println(res) } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18

21 Phrase Search func main() { index, err := bleve.Open("gopherconin.bleve") if err != nil { log.Fatal(err) } phrase := []string{"quality", "search", "results"} q := bleve.NewPhraseQuery(phrase, "description") req := bleve.NewSearchRequest(q) req.Highlight = bleve.NewHighlightWithStyle("html") req.Fields = []string{"summary", "speaker"} res, err := index.Search(req) if err != nil { log.Fatal(err) } fmt.Println(res) } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 1 matches, showing 1 through 1, took 46.134µs 1. bleve_-_modern_text_indexing_for_go (1.033644) description …earch component. But delivering high quality search results requires a long list of text analysis and indexing techniques. With the bleve library, we bring advanced text indexing and search to your Go… summary bleve - modern text indexing for Go speaker Martin Schoch Program exited.

22 Query String func main() { index, err := bleve.Open("gopherconin.bleve") if err != nil { log.Fatal(err) } qString := `+description:text ` qString += `summary:"text indexing" ` qString += `summary:believe~2 ` qString += `-description:lucene ` qString += `duration:<30` q := bleve.NewQueryStringQuery(qString) req := bleve.NewSearchRequest(q) req.Highlight = bleve.NewHighlightWithStyle("html") req.Fields = []string{"summary", "speaker", "description", "duration"} res, err := index.Search(req) if err != nil { log.Fatal(err) } fmt.Println(res) } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22

23 Query String func main() { index, err := bleve.Open("gopherconin.bleve") if err != nil { log.Fatal(err) } qString := `+description:text ` qString += `summary:"text indexing" ` qString += `summary:believe~2 ` qString += `-description:lucene ` qString += `duration:<30` q := bleve.NewQueryStringQuery(qString) req := bleve.NewSearchRequest(q) req.Highlight = bleve.NewHighlightWithStyle("html") req.Fields = []string{"summary", "speaker", "description", "duration"} res, err := index.Search(req) if err != nil { log.Fatal(err) } fmt.Println(res) } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 1 matches, showing 1 through 1, took 220.08µs 1. bleve_-_modern_text_indexing_for_go (0.338882) summary bleve - modern text indexing for Go description …ist of text analysis and indexing techniques. With the bleve library, we bring advanced text indexing and search to your Go applications. This talk will start with a brief introduction to text search … speaker Martin Schoch duration 25

24 HTTP Handlers (optional) import "github.com/blevesearch/bleve/http" • All major bleve operations mapped • Assume JSON document bodies • See bleve-explorer sample app https://github.com/blevesearch/bleve-explorer

Putting It All Together

26 Search Hugo Sites

27 Hugo Site Demo

Join the Community

29 Community • #bleve is small/quiet room, talk to us real time • Discuss your use-case • Plan a feature implementation • Apache License v2.0 • Report Issues, Submit Pull Requests

30 Contributors

31 Roadmap • Performance • Prepare for 1.0 Release

32 Thanks • Marty Schoch • marty@couchbase.com • http://github.com/blevesearch/bleve • @mschoch • @blevesearch