Share
Embed
Bleve - modern text indexing for Go
An introduction to Bleve, a Go library for full text indexing and search, presented at GopherCon India 2015.
Marty Schoch
February 21, 2015
Transcript
41 slides
-
3 Marty Schoch • NoSQL Document Database • Official Go
SDK • Projects Using Go • N1QL Query Language • Secondary Indexing • Cross Data-Center Replication -
4 Why? • Lucene/Solr/Elasticsearch are awesome • Could we build
50% of Lucene's text analysis, combine it with off-the-shelf KV stores and get something interesting? • Pluggable text analysis • Pluggable KV storage -
12 Import import "github.com/blevesearch/bleve" type Person struct {
Name string } func main() { mapping := bleve.NewIndexMapping() index, err := bleve.New("people.bleve", mapping) if err != nil { log.Fatal(err) } person := Person{"Marty Schoch"} err = index.Index("m1", person) if err != nil { log.Fatal(err) } fmt.Println("Indexed Document") } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 -
13 Data Model import "github.com/blevesearch/bleve" type Person struct {
Name string } func main() { mapping := bleve.NewIndexMapping() index, err := bleve.New("people.bleve", mapping) if err != nil { log.Fatal(err) } person := Person{"Marty Schoch"} err = index.Index("m1", person) if err != nil { log.Fatal(err) } fmt.Println("Indexed Document") } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 -
14 Index Mapping import "github.com/blevesearch/bleve" type Person struct {
Name string } func main() { mapping := bleve.NewIndexMapping() index, err := bleve.New("people.bleve", mapping) if err != nil { log.Fatal(err) } person := Person{"Marty Schoch"} err = index.Index("m1", person) if err != nil { log.Fatal(err) } fmt.Println("Indexed Document") } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 -
15 Create New Index import "github.com/blevesearch/bleve" type Person struct
{ Name string } func main() { mapping := bleve.NewIndexMapping() index, err := bleve.New("people.bleve", mapping) if err != nil { log.Fatal(err) } person := Person{"Marty Schoch"} err = index.Index("m1", person) if err != nil { log.Fatal(err) } fmt.Println("Indexed Document") } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 -
16 Index Data import "github.com/blevesearch/bleve" type Person struct {
Name string } func main() { mapping := bleve.NewIndexMapping() index, err := bleve.New("people.bleve", mapping) if err != nil { log.Fatal(err) } person := Person{"Marty Schoch"} err = index.Index("m1", person) if err != nil { log.Fatal(err) } fmt.Println("Indexed Document") } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 -
17 Index Data import "github.com/blevesearch/bleve" type Person struct {
Name string } func main() { mapping := bleve.NewIndexMapping() index, err := bleve.New("people.bleve", mapping) if err != nil { log.Fatal(err) } person := Person{"Marty Schoch"} err = index.Index("m1", person) if err != nil { log.Fatal(err) } fmt.Println("Indexed Document") } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 Indexed Document Program exited. -
18 Open Index func main() {
index, err := bleve.Open("people.bleve") if err != nil { log.Fatal(err) } query := bleve.NewTermQuery("marty") request := bleve.NewSearchRequest(query) result, err := index.Search(request) if err != nil { log.Fatal(err) } fmt.Println(result) } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 -
19 Build Query func main() {
index, err := bleve.Open("people.bleve") if err != nil { log.Fatal(err) } query := bleve.NewTermQuery("marty") request := bleve.NewSearchRequest(query) result, err := index.Search(request) if err != nil { log.Fatal(err) } fmt.Println(result) } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 -
20 Build Request func main() {
index, err := bleve.Open("people.bleve") if err != nil { log.Fatal(err) } query := bleve.NewTermQuery("marty") request := bleve.NewSearchRequest(query) result, err := index.Search(request) if err != nil { log.Fatal(err) } fmt.Println(result) } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 -
21 Search func main() { index,
err := bleve.Open("people.bleve") if err != nil { log.Fatal(err) } query := bleve.NewTermQuery("marty") request := bleve.NewSearchRequest(query) result, err := index.Search(request) if err != nil { log.Fatal(err) } fmt.Println(result) } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 -
22 Search func main() { index,
err := bleve.Open("people.bleve") if err != nil { log.Fatal(err) } query := bleve.NewTermQuery("marty") request := bleve.NewSearchRequest(query) result, err := index.Search(request) if err != nil { log.Fatal(err) } fmt.Println(result) } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 1 matches, showing 1 through 1, took 24.713µs 1. m1 (0.216978) Program exited. -
24 Conference Schedule type Event struct {
UID string `json:"uid"` Summary string `json:"summary"` Description string `json:"description"` Speaker string `json:"speaker"` Start time.Time `json:"start"` Duration float64 `json:"duration"` } 1 2 3 4 5 6 7 8 -
25 Phrase Search func main() {
index, err := bleve.Open("gopherconin.bleve") if err != nil { log.Fatal(err) } phrase := []string{"quality", "search", "results"} q := bleve.NewPhraseQuery(phrase, "description") req := bleve.NewSearchRequest(q) req.Highlight = bleve.NewHighlightWithStyle("html") req.Fields = []string{"summary", "speaker"} res, err := index.Search(req) if err != nil { log.Fatal(err) } fmt.Println(res) } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 -
26 Phrase Search func main() {
index, err := bleve.Open("gopherconin.bleve") if err != nil { log.Fatal(err) } phrase := []string{"quality", "search", "results"} q := bleve.NewPhraseQuery(phrase, "description") req := bleve.NewSearchRequest(q) req.Highlight = bleve.NewHighlightWithStyle("html") req.Fields = []string{"summary", "speaker"} res, err := index.Search(req) if err != nil { log.Fatal(err) } fmt.Println(res) } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 1 matches, showing 1 through 1, took 46.134µs 1. bleve_-‐_modern_text_indexing_for_go (1.033644) description …earch component. But delivering high quality search results requires a long list of text analysis and indexing techniques. With the bleve library, we bring advanced text indexing and search to your Go… summary bleve -‐ modern text indexing for Go speaker Martin Schoch Program exited. -
27 Query String func main() {
index, err := bleve.Open("gopherconin.bleve") if err != nil { log.Fatal(err) } qString := `+description:text ` qString += `summary:"text indexing" ` qString += `summary:believe~2 ` qString += `-‐description:lucene ` qString += `duration:<30` q := bleve.NewQueryStringQuery(qString) req := bleve.NewSearchRequest(q) req.Highlight = bleve.NewHighlightWithStyle("html") req.Fields = []string{"summary", "speaker", "description", "duration"} res, err := index.Search(req) if err != nil { log.Fatal(err) } fmt.Println(res) } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 -
28 Query String func main() {
index, err := bleve.Open("gopherconin.bleve") if err != nil { log.Fatal(err) } qString := `+description:text ` qString += `summary:"text indexing" ` qString += `summary:believe~2 ` qString += `-‐description:lucene ` qString += `duration:<30` q := bleve.NewQueryStringQuery(qString) req := bleve.NewSearchRequest(q) req.Highlight = bleve.NewHighlightWithStyle("html") req.Fields = []string{"summary", "speaker", "description", "duration"} res, err := index.Search(req) if err != nil { log.Fatal(err) } fmt.Println(res) } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 1 matches, showing 1 through 1, took 220.08µs 1. bleve_-‐_modern_text_indexing_for_go (0.338882) summary bleve -‐ modern text indexing for Go description …ist of text analysis and indexing techniques. With the bleve library, we bring advanced text indexing and search to your Go applications. This talk will start with a brief introduction to text search … speaker Martin Schoch duration 25 Program exited. -
29 HTTP Handlers (optional) import "github.com/blevesearch/bleve/http" • All major bleve
operations mapped • Assume JSON document bodies • See bleve-explorer sample app https://github.com/blevesearch/bleve-‐explorer -
36 Community • #bleve is small/quiet room, talk to us
real time • Discuss your use-case • Plan a feature implementation • Apache License v2.0 • Report Issues, Submit Pull Requests -
38 Roadmap • Result Sorting (other than score) • Better
Spell Suggest/Fuzzy Search • Performance • Prepare for 1.0 Release -
41 Thanks • Marty Schoch • [email protected] • http://github.com/blevesearch/bleve •
@mschoch • @blevesearch