Share
Embed
ElasticSearch In Action - FOWA15
Slides for my ElasticSearch talk at FOWA 2015 in London.
Thijs Feryn PRO
October 06, 2015
Transcript
77 slides
-
•Full-text search database •NoSQL database •Analytics engine •Written in Java
•Lucene based ( ~Solr) •Inverted indices •Easy to scale (~Elastic) •RESTFul interface (HTTP/JSON) •Schemaless •Real-time •ELK stack -
{ "status" : 200, "name" :
"War V", "cluster_name" : "elasticsearch", "version" : { "number" : "1.5.2", "build_hash" : "62ff9868b4c8a0c45860bebb259e21980778ab1c", "build_timestamp" : "2015-‐04-‐27T09:21:06Z", "build_snapshot" : false, "lucene_version" : "4.10.4" }, "tagline" : "You Know, for Search" } http://localhost:9200 -
POST /my-‐index {"acknowledged":true} POST/my-‐index/my-‐type {
"key" : "value", "date" : "2015-‐05-‐10", "counter" : 1, "tags" : ["tag1","tag2","tag3"] } { "_index": "my-‐index", "_type": "my-‐type", "_id": "AU089olr9oI99a_rK9fi", "_version": 1, "created": true } CREATE DATABASE my-‐index INSERT INTO my-‐index.my-‐type (key,date,counter,tags) VALUES ("value","2015-‐05-‐10",1,"tag1,tag2,tag3") Confirmation -
GET/my-‐index/my-‐type/AU089olr9oI99a_rK9fi?pretty { "_index": "my-‐index",
"_type": "my-‐type", "_id": "AU089olr9oI99a_rK9fi", "_version": 1, "found": true, "_source": { "key": "value", "date": "2015-‐05-‐10", "counter": 1, "tags": [ "tag1", "tag2", "tag3" ] } } Retrieve document by id Document & meta data SELECT * FROM my-‐index.my-‐type WHERE id="AU089olr9oI99a_rK9fi" -
GET /my-‐index/_mapping?pretty { "my-‐index": {
"mappings": { "my-‐type": { "properties": { "counter": { "type": "long" }, "date": { "type": "date", "format": "dateOptionalTime" }, "key": { "type": "string" }, "tags": { "type": "string" } } } } } } Schemaless? Not really … “Guesses” mapping on insert -
mysql> desc catalog_product_flat_1; +-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐+-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐+-‐-‐-‐-‐-‐-‐+-‐-‐-‐-‐-‐+-‐-‐-‐-‐-‐-‐-‐-‐-‐+-‐-‐-‐-‐-‐-‐-‐+ | Field
| Type | Null | Key | Default | Extra | +-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐+-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐-‐+-‐-‐-‐-‐-‐-‐+-‐-‐-‐-‐-‐+-‐-‐-‐-‐-‐-‐-‐-‐-‐+-‐-‐-‐-‐-‐-‐-‐+ | entity_id | int(10) unsigned | NO | PRI | NULL | | | attribute_set_id | smallint(5) unsigned | NO | MUL | 0 | | | type_id | varchar(32) | NO | MUL | simple | | | allow_open_amount | int(11) | YES | | NULL | | | cost | decimal(12,4) | YES | | NULL | | | created_at | timestamp | YES | | NULL | | | email_template | varchar(255) | YES | | NULL | | | enable_googlecheckout | smallint(6) | YES | | NULL | | | giftcard_amounts | decimal(12,4) | YES | | NULL | | | giftcard_type | smallint(5) unsigned | YES | | NULL | | | gift_message_available | smallint(6) | YES | | NULL | | | gift_wrapping_available | smallint(6) | YES | | NULL | | | gift_wrapping_price | decimal(12,4) | YES | | NULL | | | has_options | smallint(6) | NO | | 0 | | | image_label | varchar(255) | YES | | NULL | | | is_recurring | smallint(6) | YES | | NULL | | | is_redeemable | int(11) | YES | | NULL | | | lifetime | int(11) | YES | | NULL | | | links_exist | int(11) | YES | | NULL | | | links_purchased_separately | int(11) | YES | | NULL | | | links_title | varchar(255) | YES | | NULL | | | msrp | decimal(12,4) | YES | | NULL | | | msrp_display_actual_price_type | varchar(255) | YES | | NULL | | | msrp_enabled | smallint(6) | YES | | NULL | | | name | varchar(255) | YES | MUL | NULL | | | news_from_date | datetime | YES | | NULL | | | news_to_date | datetime | YES | | NULL | | | open_amount_max | decimal(12,4) | YES | | NULL | | | open_amount_min | decimal(12,4) | YES | | NULL | | | price | decimal(12,4) | YES | MUL | NULL | | | price_type | int(11) | YES | | NULL | | | price_view | int(11) | YES | | NULL | | | recurring_profile | text | YES | | NULL | | | required_options | smallint(5) unsigned | NO | | 0 | | | shipment_type | int(11) | YES | | NULL | | | short_description | text | YES | | NULL | | | sku | varchar(64) | YES | | NULL | | | sku_type | int(11) | YES | | NULL | | | small_image | varchar(255) | YES | | NULL | | | small_image_label | varchar(255) | YES | | NULL | | Magento’s flattened product table -
{ "category": [
"Default Category", "Men", "Shirts" ], "entity_id": "231", "created_at": "2013-‐03-‐05 06:48:12", "updated_at": "2013-‐03-‐05 09:27:15", "name": "French Cuff Cotton Twill Oxford", "price": "190.0000", "description": "Made with wrinkle resistant cotton twill, this French-‐cuffed luxury dress shirt is perfect for Business Class frequent flyers.", "sku": "msj000" } SELECT GROUP_CONCAT(distinct c.name) as category,p.entity_id, p.created_at, p.updated_at, p.name, p.price, p.short_description as description,p.sku FROM catalog_product_flat_1 p JOIN `catalog_category_product_index` AS `cat_index` ON cat_index.product_id=p.entity_id JOIN `catalog_category_flat_store_1` AS `c` ON cat_index.category_id=c.entity_id WHERE cat_index.store_id = 1 AND cat_index.visibility=1 GROUP BY p.name ORDER BY p.entity_id -
POST /products { "mappings": {
"product" : { "_id" : { "path" : "entity_id" }, "properties" : { "entity_id" : {"type" : "integer"}, "name" : { "type" : "string", "index" : "not_analyzed", "fields" : { "raw" : { "type" : "string", "analyzer": "english" } } }, "description" : { "type" : "string", "index" : "not_analyzed", "fields" : { "raw" : { "type" : "string", "analyzer": "english" } } }, "price" : {"type" : "double"}, "sku" : {"type" : "string", "index" : "not_analyzed"}, "created_at" : {"type" : "date", "format" : "YYYY-‐MM-‐dd HH:mm:ss"}, "updated_at" : {"type" : "date", "format" : "YYYY-‐MM-‐dd HH:mm:ss"} , "category" : { "type" : "string", "index" : "not_analyzed" } } } } } Explicit mapping at index creation time -
"entity_id" : {"type" : "integer"},
"name" : { "type" : "string", "index" : "not_analyzed", "fields" : { "raw" : { "type" : "string", "analyzer": "english" } } }, "description" : { "type" : "string", "index" : "not_analyzed", "fields" : { "raw" : { "type" : "string", "analyzer": "english" } } }, "price" : {"type" : "double"}, "sku" : {"type" : "string", "index" : "not_analyzed"}, "created_at" : {"type" : "date", "format" : "YYYY-‐MM-‐dd HH:mm:ss"}, "updated_at" : {"type" : "date", "format" : "YYYY-‐MM-‐dd HH:mm:ss"} , "category" : { "type" : "string", "index" : "not_analyzed" } Not analyzed? English analyzer? -
Analyzer •Character filters •Tokenizers •Token filters Replaces characters for analyzed
text Break text down into terms Add/modify/ delete tokens -
Built-in analyzers •Standard •Simple •Whitespace •Stop •Keyword •Pattern •Language •Snowball
•Custom Standard tokenizer Lowercase token filter English stop word token filter -
Set the shape to semi-transparent by calling set_trans(5) set, the,
shape, to, semi, transparent, by, calling, set_trans, 5 set, the, shape, to, semi, transparent, by, calling, set, trans Set, the, shape, to, semi-transparent, by, calling, set_trans(5) set, shape, semi, transpar, call, set_tran, 5 Standard Simple Whitespace English -
Building a custom analyzer PUT /my_index {
"settings": { "analysis": { "char_filter": { "&_to_and": { "type": "mapping", "mappings": [ "&=> and "] }}, "filter": { "my_stopwords": { "type": "stop", "stopwords": [ "the", "a" ] }}, "analyzer": { "my_analyzer": { "type": "custom", "char_filter": [ "html_strip", "&_to_and" ], "tokenizer": "standard", "filter": [ "lowercase", "my_stopwords" ] }} }}} -
GET /products/product/_search?pretty { "took": 2,
"timed_out": false, "_shards": { "total": 1, "successful": 1, "failed": 0 }, "hits": { "total": 192, "max_score": 1, "hits": [ { "_index": "products", "_type": "product", "_id": "231", "_score": 1, "_source": { "category": [ "Default Category", "Men", "Shirts" ], "entity_id": "231", "created_at": "2013-‐03-‐05 06:48:12", "updated_at": "2013-‐03-‐05 09:27:15", "name": "French Cuff Cotton Twill Oxford", "price": "190.0000", "description": "Made with wrinkle resistant cotton twill, this French-‐cuffed luxury dress shirt is perfect for Business Class frequent flyers.", "sku": "msj000" } }, … SELECT * FROM products.product -
GET /products/product/_search?pretty POST /products/product/_search?pretty {
"query": { "match_all": {} } } Search “lite” vs full query DSL SELECT * FROM products.product -
GET /products/product/_search? pretty&q=name.raw:Blazer POST /products/product/_search?pretty {
"query": { "match": { "name.raw": "Blazer" } } } SELECT * FROM products.product WHERE name LIKE "%Blazer%" Search “lite” vs full query DSL -
SELECT * FROM products.product WHERE
name = "Linen Blazer" POST /products/product/_search? pretty { "query": { "match": { "name.raw": "Linen Blazer" } } } POST /products/product/_search? pretty { "query": { "filtered": { "query": { "match_all": {} }, "filter": { "term": { "name": "Linen Blazer" } } } } } Matches 2 products Matches 1 product SELECT * FROM products.product WHERE name LIKE "%Linen%" OR name LIKE "%Blazer%" -
Filter •Does it match? Yes or no •When relevance doesn’t
matter •Faster & cacheable •For non-analyzed data Query •How well does it match? •For full-text search •On analyzed/tokenized data -
Match Query Multi Match Query Bool Query Boosting Query Common
Terms Query Constant Score Query Dis Max Query Filtered Query Fuzzy Like This Query Fuzzy Like This Field Query Function Score Query Fuzzy Query GeoShape Query Has Child Query Has Parent Query Ids Query Indices Query Match All Query More Like This Query Nested Query Prefix Query Query String Query Simple Query String Query Range Query Regexp Query Span First Query Span Multi Term Query Span Near Query Span Not Query Span Or Query Span Term Query Term Query Terms Query Top Children Query Wildcard Query Minimum Should Match Multi Term Query Rewrite Template Query -
And Filter Bool Filter Exists Filter Geo Bounding Box Filter
Geo Distance Filter Geo Distance Range Filter Geo Polygon Filter GeoShape Filter Geohash Cell Filter Has Child Filter Has Parent Filter Ids Filter Indices Filter Limit Filter Match All Filter Missing Filter Nested Filter Not Filter Or Filter Prefix Filter Query Filter Range Filter Regexp Filter Script Filter Term Filter Terms Filter Type Filter -
POST /products/product/_search?pretty { "query": {
"filtered": { "filter": { "ids": { "values": [231,234,258] } } } } } SELECT * FROM products.product WHERE entity_id IN (231,234,258) -
POST /products/product/_search?pretty { "query": {
"filtered": { "filter": { "bool": { "must": [ { "range": { "price": { "gte": 100, "lte": 400 } } } ], "must_not": [ { "term": { "name": "Convertible Dress" } } ], "should": [ { "term": { "category": "Women" } }, { "term": { "category": "New Arrivals" } } ] } } } } } SELECT * FROM products.product WHERE price BETWEEN 100 AND 400 AND name != "Convertible Dress" AND (category = "Women" OR category = "New Arrivals") -
POST /products/product/_search?pretty { "query": {
"filtered": { "filter": { "prefix": { "name": "Slim" } } } } } SELECT * FROM products.product WHERE name LIKE "Slim%" -
{ "filtered" : {
"filter" : { "geo_distance_range" : { "from" : "200km", "to" : "400km", "pin.location" : { "lat" : 40, "lon" : -‐70 } } } } } Requires “geo point” typed field -
{ "took": 1, "timed_out":
false, "_shards": { "total": 1, "successful": 1, "failed": 0 }, "hits": { "total": 2, "max_score": 3.3350093, "hits": [ { "_index": "products", "_type": "product", "_id": "243", "_score": 3.3350093, "fields": { "name": [ "Linen Blazer" ] } }, { "_index": "products", "_type": "product", "_id": "246", "_score": 0.5954286, "fields": { "name": [ "Stretch Cotton Blazer" ] } } ] } } POST /products/product/_search?pretty { "fields": ["name"], "query": { "match": { "name.raw": "Linen Blazer" } } } Hits both terms. More relevant -
{ "took": 1, "timed_out":
false, "_shards": { "total": 1, "successful": 1, "failed": 0 }, "hits": { "total": 1, "max_score": 1, "hits": [ { "_index": "products", "_type": "product", "_id": "243", "_score": 1, "fields": { "name": [ "Linen Blazer" ] } } ] } } POST /products/product/_search?pretty { "query": { "filtered": { "query": { "match_all": {} }, "filter": { "term": { "name": "Linen Blazer" } } } } } No relevance on filters Score is always 1 -
POST /products/product/_search?pretty { "fields": ["name","description"],
"query": { "bool": { "must": [ { "match": { "description.raw": "suit" } } ], "should": [ { "match": { "name.raw": "coat" } } ] } } } Only search for “suit” Increase relevance if name contains “coat” -
POST /products/product/_search?pretty { "fields": ["price","name","description","category"],
"query": { "bool": { "must": [ { "filtered": { "filter": { "bool": { "must": [ { "range": { "price": { "gte": 100, "lte": 400 } } }, { "term": { "category": "Men" } } ] } } } } ], "should": [ { "match": { "category": "New Arrivals" } } ] } } } Increase relevance Combined filters -
POST /products/product/_search?pretty { "query": {
"bool": { "must": [ { "match_all": {} } ], "should": [ { "match": { "description.raw": { "query": "comfort", "boost": 2 } } }, { "match": { "name.raw": { "query": "tee", "boost": 3 } } } ] } } } Increase relevance Query- time boosting -
SELECT * FROM *.tbl_a* WHERE
key LIKE ‘%val%’ •Cross-database queries •Wildcard database queries •Wildcard table queries Not supported in SQL -
/_search /products/_search /products/product/_search /products,clients/_search /pro*/_search
/pro*,cli*/_search /products/product,invoice/_search /products/pro*/_search /_all/product/_search /_all/product,invoice/_search /_all/pro*/_search -
SELECT COUNT(category) FROM products.product GROUP BY category
Aggregations in SQL Metric Bucket -
SELECT COUNT(category) FROM products.product GROUP BY category
POST /products/product/_search? pretty&search_type=count { "aggs": { "number_of_categories" : { "cardinality": { "field": "category" } } } } Metric Bucket Only aggs, no docs -
POST /products/product/_search?pretty { "fields": ["category","price","name"],
"query": { "match": { "name.raw": "blazer" } }, "aggs": { "avg_price": { "avg": { "field": "price" } }, "min_price" : { "min": { "field": "price" } }, "max_price" : { "max": { "field": "price" } }, "number_of_products_per_category" : { "terms": { "field": "category", "size": 10 } } } } Multi-group by besides query -
"aggregations": {
"min_price": { "value": 455 }, "number_of_products_per_category": { "doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, "buckets": [ { "key": "Blazers", "doc_count": 2 }, { "key": "Default Category", "doc_count": 2 }, { "key": "Men", "doc_count": 2 } ] }, "max_price": { "value": 490 }, "avg_price": { "value": 472.5 } } Aggregation output Drill down search -
Min Aggregation Max Aggregation Sum Aggregation Avg Aggregation Stats Aggregation
Extended Stats Aggregation Value Count Aggregation Percentiles Aggregation Percentile Ranks Aggregation Cardinality Aggregation Geo Bounds Aggregation Top hits Aggregation Scripted Metric Aggregation Global Aggregation Filter Aggregation Filters Aggregation Missing Aggregation Nested Aggregation Reverse nested Aggregation Children Aggregation Terms Aggregation Significant Terms Aggregation Range Aggregation Date Range Aggregation IPv4 Range Aggregation Histogram Aggregation Date Histogram Aggregation Geo Distance Aggregation GeoHash grid Aggregation -
Example config settings node.rack: my-‐location node.master: true
node.data: true http.enabled: true cluster.name: my-‐cluster node.name: my-‐node index.number_of_shards: 5 index.number_of_replicas: 1 discovery.zen.minimum_master_nodes: 2 -
GET /_cat =^.^= /_cat/allocation /_cat/shards /_cat/shards/{index}
/_cat/master /_cat/nodes /_cat/indices /_cat/indices/{index} /_cat/segments /_cat/segments/{index} /_cat/count /_cat/count/{index} /_cat/recovery /_cat/recovery/{index} /_cat/health /_cat/pending_tasks /_cat/aliases /_cat/aliases/{alias} /_cat/thread_pool /_cat/plugins /_cat/fielddata /_cat/fielddata/{fields} Non-JSON output -
GET /_cat/shards?v index shard prirep state
docs store ip node my-‐index 2 r STARTED 6 7.2kb 192.168.10.142 node3 my-‐index 2 p STARTED 6 9.5kb 192.168.10.142 node2 my-‐index 0 p STARTED 4 7.1kb 192.168.10.142 node3 my-‐index 0 r STARTED 4 4.8kb 192.168.10.142 node2 my-‐index 3 r STARTED 5 7.1kb 192.168.10.142 node1 my-‐index 3 p STARTED 5 7.2kb 192.168.10.142 node3 my-‐index 1 p STARTED 1 2.4kb 192.168.10.142 node1 my-‐index 1 r STARTED 1 2.4kb 192.168.10.142 node2 my-‐index 4 p STARTED 5 9.5kb 192.168.10.142 node1 my-‐index 4 r STARTED 5 9.4kb 192.168.10.142 node3 5 shards & a single replica by default -
GET /_cat/health? v&h=cluster,status,node.total,shards,pri,unassign,init cluster status node.total shards
pri unassign init mycluster green 3 12 6 0 0 Cluster health -
It’s REST, deal with it! Directly from Javascript? Don’t forget
to “restrict” access AJAX #FTW Behind a proxy No backend code
thijsferyn
inariku
kmotohas
sansantech
cscengineer
amixedcolor
minorun365
wa6sn
kazzpapa3
.jpg){kind=avatar}
ryouto715
oracle4engineer
erikaheidi
ianozsvald
aleyda
gurzu
raygrieselhuber
baasie
cassininazir
khoart
eileencodes
popppiees
hannesfritz
signer
![POST /products/product/_search?pretty { "fields": ["name","description"],](https://files.speakerdeck.com/presentations/fb5c53cf579b470e888b3ffa5e06c529/slide_44.jpg){kind=link}
![POST /products/product/_search?pretty { "fields": ["price","name","description","category"],](https://files.speakerdeck.com/presentations/fb5c53cf579b470e888b3ffa5e06c529/slide_45.jpg){kind=link}
![POST /products/product/_search?pretty { "fields": ["category","price","name"],](https://files.speakerdeck.com/presentations/fb5c53cf579b470e888b3ffa5e06c529/slide_55.jpg){kind=link}
