Related
i am having trouble forming query to fetch all values with sql group by kind of thing.
so below is my data structure:
product index:
{
"createdBy" : "61c1fcdd88dbad1920da8caf",
"creationTime" : "2021-12-22T11:58:53.576932Z",
"lastModifiedBy" : "61c1fcdd88dbad1920da8caf",
"lastModificationTime" : "2021-12-22T11:58:53.576932Z",
"id" : "61c312fdc6aa620a609db0b2",
"title" : "string",
"brand" : "string",
"longDesc" : "string",
"categoryId" : "string",
"imageUrls" : [
"string",
"string"
],
"keySpecs" : [
"string",
"string",
],
"facets" : [
{
"name" : "color",
"value" : "red"
},
{
"name" : "storage",
"value" : "16 GB"
},
{
"name" : "brand",
"value" : "Intex"
}
],
"categoryName" : "handsets"
}
Now, i want to fetch all the facets with their different values and count as well. Let's say
productA has color blue, productB has color red
productA has brand ABC, productB has brand XYZ
so, i want data which list all facets like:
color: blue(200 count), red (12 count)
brand: ABC(13 count), XYZ (99 count)
Also, different product will have different type of facet, like iphone will have color memory brand size, but a pen will have color and brand only (not memory/size).
Note: i'm using latest version of elastic
=================
UPDATE 1:
Below is the es mapping details
{
"settings": {
"analysis": {
"filter": {
"english_stop": {
"type": "stop",
"stopwords": "_english_"
},
"english_keywords": {
"type": "keyword_marker",
"keywords": [
"example"
]
},
"english_stemmer": {
"type": "stemmer",
"language": "english"
},
"english_possessive_stemmer": {
"type": "stemmer",
"language": "possessive_english"
}
},
"analyzer": {
"lalashree_standard_analyzer": {
"tokenizer": "standard",
"filter": [
"english_possessive_stemmer",
"lowercase",
"english_stop",
"english_keywords",
"english_stemmer"
]
},
"html_standard_analyzer": {
"char_filter": [
"html_strip"
],
"tokenizer": "standard",
"filter": [
"english_possessive_stemmer",
"lowercase",
"english_stop",
"english_keywords",
"english_stemmer"
]
}
}
}
},
"mappings": {
"properties": {
"id": {
"type": "keyword"
},
"createdBy": {
"type": "keyword"
},
"creationTime": {
"type": "date"
},
"lastModifiedBy": {
"type": "keyword"
},
"lastModificationTime": {
"type": "date"
},
"deleted": {
"type": "boolean"
},
"deletedBy": {
"type": "keyword"
},
"deletionTime": {
"type": "date"
},
"title": {
"type": "text",
"analyzer": "lalashree_standard_analyzer",
"fields": {
"suggest": {
"type": "completion"
}
}
},
"shortDesc": {
"type": "text",
"analyzer": "lalashree_standard_analyzer"
},
"longDesc": {
"type": "text",
"analyzer": "lalashree_standard_analyzer"
},
"categoryId": {
"type": "keyword"
},
"searchDetails": {
"type": "object",
"properties": {
"desc": {
"type": "text",
"analyzer": "lalashree_standard_analyzer"
},
"keywords": {
"type": "text",
"analyzer": "lalashree_standard_analyzer",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"imageUrls": {
"type": "keyword",
"index": false
},
"keySpecs": {
"type": "text",
"analyzer": "lalashree_standard_analyzer"
},
"sections": {
"type": "object",
"properties": {
"name": {
"type": "text",
"index": false
},
"shortDesc": {
"type": "text",
"analyzer": "lalashree_standard_analyzer"
},
"longDesc": {
"type": "text",
"analyzer": "lalashree_standard_analyzer"
},
"htmlContent": {
"type": "text",
"analyzer": "html_standard_analyzer"
}
}
},
"facets": {
"type": "nested",
"properties": {
"name": {
"type": "keyword"
},
"value": {
"type": "keyword"
}
}
},
"specificationItems": {
"type": "object",
"properties": {
"key": {
"type": "text",
"analyzer": "lalashree_standard_analyzer",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"values": {
"type": "text",
"analyzer": "lalashree_standard_analyzer"
}
}
},
"categoryName": {
"type": "keyword"
},
"productFamily": {
"type": "nested",
"properties": {
"id": {
"type": "keyword"
},
"familyVariantOptions": {
"type": "nested",
"properties": {
"name": {
"type": "keyword"
},
"values": {
"type": "keyword"
}
}
},
"productFamilyItems": {
"type": "nested",
"properties": {
"baseProductId": {
"type": "keyword"
},
"itemVariantInfoSet": {
"type": "nested",
"properties": {
"name": {
"type": "keyword"
},
"value": {
"type": "keyword"
}
}
}
}
}
}
},
"rating": {
"type": "float"
},
"totalReviewsCount": {
"type": "long"
},
"stores": {
"type": "nested",
"properties": {
"id": {
"type": "keyword"
},
"logo": {
"type": "keyword",
"index": false
},
"active": {
"type": "boolean"
},
"name": {
"type": "text"
},
"quantity": {
"type": "long"
},
"rating": {
"type": "float"
},
"totalReviewsCount": {
"type": "long"
},
"price.mrp": {
"type": "float"
},
"price.sp": {
"type": "float"
},
"location.geoPoint": {
"type": "geo_point"
},
"oos": {
"type": "boolean"
}
}
}
}
}
}
This query first group by names then groups each name's values. By setting sizes, you can arrange number of facets you want and number of items in each facet. I think it does what you need.
Note that if you have too many documents and if performance matters, this query may perform bad.
{
"size": 0,
"aggs": {
"facets": {
"nested": {
"path": "facets"
},
"aggs": {
"names": {
"terms": {
"field": "facets.name",
"size": 10
},
"aggs": {
"values": {
"terms": {
"field": "facets.value",
"size": 10
}
}
}
}
}
}
}
}
I am currently trying to update an index template on Elastic Search 6.7/6.8.
Templates are stored in the code and are applied each time my API starts.
There are no errors, the request returns 200.
For example, here is a template i am currently using:
{
"index_patterns": [ "*-ec2-reports" ],
"version": 11,
"mappings": {
"ec2-report": {
"properties": {
"account": {
"type": "keyword"
},
"reportDate": {
"type": "date"
},
"reportType": {
"type": "keyword"
},
"instance": {
"properties": {
"id": {
"type": "keyword"
},
"region": {
"type": "keyword"
},
"state": {
"type": "keyword"
},
"purchasing": {
"type": "keyword"
},
"keyPair": {
"type": "keyword"
},
"type": {
"type": "keyword"
},
"platform": {
"type": "keyword"
},
"tags": {
"type": "nested",
"properties": {
"key": {
"type": "keyword"
},
"value": {
"type": "keyword"
}
}
},
"costs": {
"type": "object"
},
"stats": {
"type": "object",
"properties": {
"cpu": {
"type": "object",
"properties": {
"average": {
"type": "double"
},
"peak": {
"type": "double"
}
}
},
"network": {
"type": "object",
"properties": {
"in": {
"type": "double"
},
"out": {
"type": "double"
}
}
},
"volumes": {
"type": "nested",
"properties": {
"id": {
"type": "keyword"
},
"read": {
"type": "double"
},
"write": {
"type": "double"
}
}
}
}
},
"recommendation": {
"type": "object",
"properties": {
"instancetype": {
"type": "keyword"
},
"reason": {
"type": "keyword"
},
"newgeneration": {
"type": "keyword"
}
}
}
}
}
},
"_all": {
"enabled": false
},
"numeric_detection": false,
"date_detection": false
}
}
}
I'd like to add a new keyword field under the properties object like this :
"exampleField": {
"type": "keyword"
}
but it seems the template is not applied to existing indexes.
When data is inserted into a specific index which use the template, it is stored like this:
"exampleField": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
because the template has not been updated beforehand.
I would expect it to be like:
"exampleField": {
"type": "keyword"
}
in the index and in the template.
Does someone have any idea on how to have this result?
Thank you, Alexandre.
I am creating this Index, and I got an error.
I want to create an index of cities with names, people living, number of villages, facts about cities, etc.
My code is:
IMAGE OF MY CODE
PUT City-mk
{
"mappings": {
"properties": {
"CityID": {
"type": "integer"
},
"CityName": {
"type": "text",
"fields": {
"type": "keyword"
}
}
},
"People": {
"type":"integer"
},
"Fact": {
"type": "text",
"fields": {
"type": "keyword"
}
}
},
"Villages": {
"type": "integer"
},
"CallNum": {
"type": "integer"
}
}
You need to make following corrections:
Index name must be in lowercase, so change City-mk to city-mk
Braces count was wrong
Subfields were defined wrongly- "fields": { "keyword": { "type": "keyword" } }
PUT city-mk
{
"mappings": {
"properties": {
"CityID": {
"type": "integer"
},
"CityName": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"People": {
"type": "integer"
},
"Fact": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"Villages": {
"type": "integer"
},
"CallNum": {
"type": "integer"
}
}
}
}
I am trying to create a search query that allows to search by name and type.
I have indexed the values, and my record in Elasticsearch look like this:
{
_index: "assets",
_type: "asset",
_id: "eAOEN28BcFmQazI-nngR",
_score: 1,
_source: {
name: "test.png",
mediaType: "IMAGE",
meta: {
content-type: "image/png",
width: 3348,
height: 1890,
},
createdAt: "2019-12-24T10:47:15.727Z",
updatedAt: "2019-12-24T10:47:15.727Z",
}
}
so how would I create for example, a query that finds all assets that have the name "test' and are images?
I tried multi_mach query but that did not return the correct results:
{
"query": {
"multi_match" : {
"query": "*test* IMAGE",
"type": "cross_fields",
"fields": [ "name", "mediaType" ],
"operator": "and"
}
}
}
The query above returns 0 results, and if I change the operator to "or" it returns all this assets of type IMAGE.
Any suggestions would be greatly appreciated. TIA!
EDIT: Added Mapping
Below is the mapping:
{
"assets": {
"aliases": {},
"mappings": {
"properties": {
"__v": {
"type": "long"
},
"createdAt": {
"type": "date"
},
"deleted": {
"type": "date"
},
"mediaType": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"meta": {
"properties": {
"content-type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"width": {
"type": "long"
},
"height": {
"type": "long"
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"originalName": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"updatedAt": {
"type": "date"
}
}
},
"settings": {
"index": {
"creation_date": "1575884312237",
"number_of_shards": "1",
"number_of_replicas": "1",
"uuid": "nSiAoIIwQJqXQRTyqw9CSA",
"version": {
"created": "7030099"
},
"provided_name": "assets"
}
}
}
}
You are unnecessary using the wildcard expression for this simple query.
First, change your analyzer on name field.
You need to create a custom analyzer which replaces . with space as default standard analyzer doesn't do that, so that you when searching for test you get test.png as there will be both test and png in the inverted index. The main benefit of doing this is to avoid the regex queries which are very costly.
Updated mapping with custom analyzer which would do the work for you. Just update your mapping and re-index again all the doc.
{
"aliases": {},
"mappings": {
"properties": {
"__v": {
"type": "long"
},
"createdAt": {
"type": "date"
},
"deleted": {
"type": "date"
},
"mediaType": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"meta": {
"properties": {
"content-type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"width": {
"type": "long"
},
"height": {
"type": "long"
}
}
},
"name": {
"type": "text",
"analyzer" : "my_analyzer"
},
"originalName": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"updatedAt": {
"type": "date"
}
}
},
"settings": {
"analysis": {
"analyzer": {
"my_analyzer": {
"tokenizer": "standard",
"char_filter": [
"replace_dots"
]
}
},
"char_filter": {
"replace_dots": {
"type": "mapping",
"mappings": [
". => \\u0020"
]
}
}
},
"index": {
"number_of_shards": "1",
"number_of_replicas": "1"
}
}
}
Second, you should change your query to bool query as below:
{
"query": {
"bool": {
"must": [
{
"match": {
"name": "test"
}
},
{
"match": {
"mediaType.keyword": "IMAGE"
}
}
]
}
}
}
Which is using must with 2 match queries means, that it would return docs only when there is a match in all the clauses of must query.
I already tested my solution by creating the index, inserting a few sample docs and query them, let me know if you need any help.
Did you tried with best_fields ?
{
"query": {
"multi_match" : {
"query": "Will Smith",
"type": "best_fields",
"fields": [ "name", "mediaType" ],
"operator": "and"
}
}
}
We have an elastic search 5.5 setup. We use nest to perform our queries through C#.
When executing the following query:
{
"query": {
"bool": {
"must": [
{
"query_string": {
"query": "00917751"
}
}
]
}
}
}
We get the desired result: one result with that the number as identifier.
When using the following query:
{
"query": {
"bool": {
"must": [
{
"query_string": {
"query": "00917751",
"fields": [
"searchReference",
"searchIdentifier",
"searchObjectNo",
"searchBrand",
"searchExtSerNo"
]
}
}
]
}
}
}
We get no results.
The value we are searching for is in the field searchIndentifier, and has the value "1-00917751".
We have a custom analyzer called "final"
.Custom("final", cu => cu
.Tokenizer("keyword").Filters(new List() { "lowercase" }))
The field searchIndentifier has no custom analyzer set on it. We tried adding the whitespace tokenizer in it but that made no difference.
Another field called "searchObjectNo" does work, when we try to search for the value "S328-25" with the query "S328". These fields are exactly the same.
Any ideas here?
Another question. In the first query, when we search for 1-00917751 (without the quotes) we get a lot of results. But we think that is because of the keyword tokenizer?
Thank you
Schoof
Index settings and mappings:
{
"inventoryitems": {
"aliases": {},
"mappings": {
"inventoryobject": {
"properties": {
"articleGroups": {
"type": "nested",
"properties": {
"id": {
"type": "long"
}
}
},
"articleId": {
"type": "long"
},
"articleNumber": {
"type": "text",
"boost": 1.5,
"analyzer": "final"
},
"brand": {
"type": "text",
"analyzer": "final"
},
"catalogues": {
"type": "nested",
"properties": {
"articleGroupId": {
"type": "long"
},
"articleGroupName": {
"type": "text",
"analyzer": "final",
"fielddata": true
},
"id": {
"type": "long"
},
"name": {
"type": "text",
"analyzer": "final",
"fielddata": true
}
}
},
"details": {
"type": "nested",
"properties": {
"actualState": {
"type": "double"
},
"allocation": {
"type": "text",
"analyzer": "final",
"fielddata": true
},
"available": {
"type": "double"
},
"batch": {
"type": "text",
"analyzer": "final"
},
"calibrationDate": {
"type": "date"
},
"expected": {
"type": "double"
},
"externalSerialNumber": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"inReturn": {
"type": "double"
},
"inventory": {
"type": "double"
},
"isInMobileCarrier": {
"type": "boolean"
},
"locationDetail": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"locationId": {
"type": "long"
},
"locationName": {
"type": "text",
"analyzer": "final",
"fielddata": true
},
"locationType": {
"type": "text",
"analyzer": "final",
"fielddata": true
},
"lotId": {
"type": "long"
},
"mobileCarrierCode": {
"type": "text",
"analyzer": "final",
"fielddata": true
},
"mobileCarrierId": {
"type": "long"
},
"ownerCode": {
"type": "text",
"analyzer": "final"
},
"requested": {
"type": "double"
},
"reserved": {
"type": "double"
},
"storeLocationId": {
"type": "long"
},
"thicknessCode": {
"type": "text",
"analyzer": "final"
},
"weldedMark": {
"type": "text",
"analyzer": "final"
}
}
},
"docNo": {
"type": "long"
},
"hasStock": {
"type": "boolean"
},
"id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"identifier": {
"type": "text",
"boost": 1.5,
"analyzer": "final"
},
"inventoryItemType": {
"properties": {
"name": {
"type": "text",
"analyzer": "final",
"fielddata": true
}
}
},
"mobileCarrierId": {
"type": "long"
},
"name": {
"type": "text",
"boost": 1.5,
"analyzer": "final"
},
"objectNumber": {
"type": "text",
"boost": 1.5,
"analyzer": "final"
},
"quantity": {
"type": "double"
},
"reference": {
"type": "text",
"boost": 1.5,
"analyzer": "final"
},
"searchBrand": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"searchExtSerNo": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"searchIndentifier": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"searchName": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"searchObjectNo": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"searchReference": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"sortNumber": {
"type": "long"
},
"stockUnit": {
"type": "text",
"boost": 1.5,
"analyzer": "final"
}
}
}
},
"settings": {
"index": {
"number_of_shards": "3",
"provided_name": "inventoryitems",
"creation_date": "1539253308319",
"analysis": {
"analyzer": {
"final": {
"filter": [
"lowercase"
],
"type": "custom",
"tokenizer": "keyword"
}
}
},
"number_of_replicas": "1",
"uuid": "Kb5KuYEiR5GQqgBPVYjJfA",
"version": {
"created": "5050299"
}
}
}
}
}
The answer is pretty simple: in your mapping your field is named searchIndentifier and in your query you're using a field called searchIdentifier which doesn't exist ;-)