I'm using Kibana 6 to display aggregations from terms indexed in ElasticSearch 6.
I'm encoutering an issue while displaying a bucket aggregating terms (bidRequest.site.domain.keyword) by keywords whereas the records are effectively present in the Index and date range is okay:
I don't understand why there is no result despite having those in the Index:
Here's my mapping:
{
"mapping": {
"BidRequestRecord": {
"properties": {
"bidRequest": {
"type": "nested",
"properties": {
"allimps": {
"type": "integer"
},
"app": {
"type": "nested",
"properties": {
"bundle": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"cat": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"domain": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ext": {
"type": "object"
},
"id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"keywords": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"pagecat": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"publisher": {
"type": "nested",
"properties": {
"ext": {
"type": "object"
},
"id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"storeurl": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ver": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"at": {
"type": "integer"
},
"badv": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"bcat": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"cur": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"device": {
"type": "nested",
"properties": {
"carrier": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"connectiontype": {
"type": "integer"
},
"devicetype": {
"type": "integer"
},
"didmd5": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"didsha1": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"dnt": {
"type": "integer"
},
"dpidmd5": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"dpidsha1": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ext": {
"type": "nested",
"properties": {
"is_app": {
"type": "integer"
}
}
},
"geo": {
"type": "nested",
"properties": {
"city": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"country": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"lat": {
"type": "float"
},
"lon": {
"type": "float"
},
"region": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"type": {
"type": "integer"
},
"zip": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"ifa": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ip": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"js": {
"type": "integer"
},
"language": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"make": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"model": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"os": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"osv": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ua": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"ext": {
"type": "object"
},
"id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"imp": {
"type": "nested",
"properties": {
"banner": {
"type": "nested",
"properties": {
"api": {
"type": "integer"
},
"battr": {
"type": "integer"
},
"btype": {
"type": "integer"
},
"expdir": {
"type": "integer"
},
"ext": {
"type": "object"
},
"format": {
"type": "nested",
"properties": {
"ext": {
"type": "object"
},
"h": {
"type": "integer"
},
"w": {
"type": "integer"
}
}
},
"h": {
"type": "integer"
},
"hmax": {
"type": "integer"
},
"hmin": {
"type": "integer"
},
"id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"mimes": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"pos": {
"type": "integer"
},
"topframe": {
"type": "integer"
},
"w": {
"type": "integer"
},
"wmax": {
"type": "integer"
},
"wmin": {
"type": "integer"
}
}
},
"bidFloorFloat": {
"type": "float"
},
"bidfloor": {
"type": "double"
},
"bidfloorInEur": {
"type": "double"
},
"bidfloorcur": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"displaymanager": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"displaymanagerver": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ext": {
"type": "object"
},
"id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"iframebuster": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"instl": {
"type": "integer"
},
"native": {
"type": "object"
},
"pmp": {
"type": "nested",
"properties": {
"deals": {
"type": "nested",
"properties": {
"at": {
"type": "integer"
},
"bidFloorFloat": {
"type": "float"
},
"bidFloorInEuro": {
"type": "double"
},
"bidFloorInEuroFloat": {
"type": "float"
},
"bidfloor": {
"type": "double"
},
"bidfloorcur": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ext": {
"type": "object"
},
"id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"wadomain": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"wseat": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"ext": {
"type": "object"
},
"private_auction": {
"type": "integer"
}
}
},
"secure": {
"type": "integer"
},
"tagid": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"video": {
"type": "nested",
"properties": {
"api": {
"type": "integer"
},
"battr": {
"type": "integer"
},
"boxingallowed": {
"type": "integer"
},
"companionad": {
"type": "nested",
"properties": {
"api": {
"type": "integer"
},
"battr": {
"type": "integer"
},
"btype": {
"type": "integer"
},
"expdir": {
"type": "integer"
},
"ext": {
"type": "object"
},
"format": {
"type": "nested",
"properties": {
"ext": {
"type": "object"
},
"h": {
"type": "integer"
},
"w": {
"type": "integer"
}
}
},
"h": {
"type": "integer"
},
"hmax": {
"type": "integer"
},
"hmin": {
"type": "integer"
},
"id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"mimes": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"pos": {
"type": "integer"
},
"topframe": {
"type": "integer"
},
"w": {
"type": "integer"
},
"wmax": {
"type": "integer"
},
"wmin": {
"type": "integer"
}
}
},
"companiontype": {
"type": "integer"
},
"ext": {
"type": "object"
},
"h": {
"type": "integer"
},
"linearity": {
"type": "integer"
},
"maxbitrate": {
"type": "integer"
},
"maxduration": {
"type": "integer"
},
"mimes": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"minbitrate": {
"type": "integer"
},
"minduration": {
"type": "integer"
},
"playbackmethod": {
"type": "integer"
},
"pos": {
"type": "integer"
},
"protocols": {
"type": "integer"
},
"startdelay": {
"type": "integer"
},
"w": {
"type": "integer"
}
}
}
}
},
"regs": {
"type": "nested",
"properties": {
"coppa": {
"type": "integer"
},
"ext": {
"type": "object"
}
}
},
"site": {
"type": "nested",
"properties": {
"cat": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"domain": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ext": {
"type": "object"
},
"id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"keywords": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"page": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"pagecat": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"source": {
"type": "nested",
"properties": {
"ext": {
"type": "object"
},
"fd": {
"type": "integer"
},
"pchain": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"tid": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
},
"bidRequestDate": {
"type": "date"
}
}
}
}
}
Well apologies for this one, I was using a not up-to-date index and at that time mapping was wrong. Works perfect with the correctly set mapping.
Related
I am trying to find a way to match the below line with either KQL or Lucene query from Kibana discovery. The value I am trying to match is in field "message" which is of type "text".
message: Starting <app_name> v1.7.0-SNAPSHOT on ...
Tried with below query:
message: /Starting\s[a-ZA-Z] v/
Application Stack:
logstash-7.10.0
Kibana 7.6.1
elasticsearch 7.6.1
Index Name: logstash-filebeat-7.10.0
Index Mappings:
{
"mapping": {
"_doc": {
"dynamic": "true",
"_meta": {},
"_source": {
"includes": [],
"excludes": []
},
"dynamic_date_formats": [
"strict_date_optional_time",
"yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z"
],
"dynamic_templates": [
{
"message_field": {
"path_match": "message",
"match_mapping_type": "string",
"mapping": {
"norms": false,
"type": "text"
}
}
},
{
"string_fields": {
"match": "*",
"match_mapping_type": "string",
"mapping": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"norms": false,
"type": "text"
}
}
}
],
"date_detection": true,
"numeric_detection": false,
"properties": {
"#timestamp": {
"type": "date"
},
"#version": {
"type": "keyword"
},
"agent": {
"properties": {
"hostname": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"name": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"type": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"version": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"classname": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"container": {
"properties": {
"id": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"ecs": {
"properties": {
"version": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"geoip": {
"dynamic": "true",
"properties": {
"ip": {
"type": "ip"
},
"latitude": {
"type": "half_float"
},
"location": {
"type": "geo_point"
},
"longitude": {
"type": "half_float"
}
}
},
"input": {
"properties": {
"type": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"log": {
"properties": {
"file": {
"properties": {
"path": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"flags": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"offset": {
"type": "long"
}
}
},
"loglevel": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"message": {
"type": "text",
"norms": false
},
"thread": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}
I am performing elastic search full indexing using Bulk request. I have an issue during the indexing the results are coming as empty. As I am deleting the index during the full index, How I can handle this situation.
I have done the these steps:
delete index
Create index
Create Mapping
bulk request
Index properties and Mapping:
{
"products": {
"aliases": {},
"mappings": {
"properties": {
"assemblyrequired": {
"type": "boolean"
},
"australianmade": {
"type": "boolean"
},
"australiasellable": {
"type": "boolean"
},
"avgRating": {
"type": "float"
},
"category": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"categorylevel1": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"categorylevel2": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"categorylevel3": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"categoryname": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"categoryname_old": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"clearance": {
"type": "boolean"
},
"commercialuse": {
"type": "boolean"
},
"customisable": {
"type": "boolean"
},
"depth": {
"type": "float"
},
"freedelivery": {
"type": "boolean"
},
"genericcolourcode": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"height": {
"type": "float"
},
"hideprice": {
"type": "boolean"
},
"listprice": {
"type": "float"
},
"materialcode": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"moneybackguarantee": {
"type": "boolean"
},
"newrelease": {
"type": "boolean"
},
"numberOfRating": {
"type": "long"
},
"online": {
"type": "boolean"
},
"outdooruse": {
"type": "boolean"
},
"predictivecategorydata": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"pricematchguarantee": {
"type": "boolean"
},
"productcode": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"productid": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"productimageurl": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"productname": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"producttypecode": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"promotedprice": {
"type": "float"
},
"sale": {
"type": "integer"
},
"saleprice": {
"type": "float"
},
"sellable": {
"type": "boolean"
},
"sellercode": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"shortdescription": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"sku": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"sortweight": {
"type": "long"
},
"state": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"stylecode": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"warrantycode": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"weight": {
"type": "float"
},
"width": {
"type": "float"
}
}
},
"settings": {
"index": {
"number_of_shards": "1",
"provided_name": "products",
"max_result_window": "500000",
"creation_date": "1595814303422",
"number_of_replicas": "1",
"uuid": "sGJxwr73Rkyu7-JekWFYsw",
"version": {
"created": "7060199"
}
}
}
}
}
I have around 75k documents.
Thanks,
Sree.
If you want the full index to be available during the reindex, your only option is to not delete the original index until after the indexing is done. In that case, I would probably work with aliases. For example, let's assume products-2020.07.28 was your current index, you would then create a new index for today and change the alias as soon as the indexing is done.
Create Index
PUT /products-2020.07.28
{
"settings": {
... your settings ...
},
"mappings": {
... your mappings ...
}
}
Bulk Index Request
Change Alias to new Index
POST /_aliases
{
"actions" : [
{ "remove" : { "index" : "products-2020.07.27", "alias" : "products" } },
{ "add" : { "index" : "products-2020.07.28", "alias" : "products" } }
]
}
Delete old Index
DELETE /products-2020.07.27
Any requests can then go directly to the alias, instead of the index.
GET /products/_search
That way you can reindex without the user noticing anything.
how to find all the unique records in elastic search based on an attribute in it..
Is there a way?
GET questiondetails/question/_search
{
"size": 0,
"aggs" : {
"Name" : {
"terms" : {
"field" : "owner.user_id.keyword"
}
}
}
}
I tried this way.. But I want it to give all the docs wherever user id is present in the whole record.
Whether it is in inner loop also....
Is there a way to get this in a single query instead of making in multiple queries..
Here is my sample record.
{
"_index": "questiondetails",
"_type": "question",
"_id": "BOktPnABW1evGoA0CqDZ",
"_score": 1,
"_source": {
"tags": [
"android",
"kotlin"
],
"owner": {
"user_id": "10633771"
},
"reply_count": "1",
"reply": [
{
"owner": {
"user_id": "12632101",
"user_type": "registered",
}
}
]
}
}
Index
{
"questiondetails": {
"aliases": {},
"mappings": {
"question": {
"properties": {
"accepted_answer_id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"answer_count": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"answers": {
"properties": {
"answer_id": {
"type": "long"
},
"body": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"body_markdown": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"comment_count": {
"type": "long"
},
"comments": {
"properties": {
"comment_id": {
"type": "long"
},
"creation_date": {
"type": "long"
},
"edited": {
"type": "boolean"
},
"owner": {
"properties": {
"accept_rate": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"display_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"link": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"profile_image": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"reputation": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"post_id": {
"type": "long"
},
"reply_to_user": {
"properties": {
"display_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"link": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"profile_image": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"reputation": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"score": {
"type": "long"
}
}
},
"creation_date": {
"type": "long"
},
"down_vote_count": {
"type": "long"
},
"is_accepted": {
"type": "boolean"
},
"last_activity_date": {
"type": "long"
},
"last_edit_date": {
"type": "long"
},
"link": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"owner": {
"properties": {
"accept_rate": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"display_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"link": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"profile_image": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"reputation": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"question_id": {
"type": "long"
},
"score": {
"type": "long"
},
"share_link": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"title": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"up_vote_count": {
"type": "long"
}
}
},
"body": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"body_markdown": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"close_vote_count": {
"type": "long"
},
"comment_count": {
"type": "long"
},
"comments": {
"properties": {
"body": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"body_markdown": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"comment_id": {
"type": "long"
},
"creation_date": {
"type": "long"
},
"edited": {
"type": "boolean"
},
"owner": {
"properties": {
"accept_rate": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"display_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"link": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"profile_image": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"reputation": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"post_id": {
"type": "long"
},
"reply_to_user": {
"properties": {
"accept_rate": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"display_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"link": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"profile_image": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"reputation": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"score": {
"type": "long"
}
}
},
"creation_date": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"delete_vote_count": {
"type": "long"
},
"down_vote_count": {
"type": "long"
},
"is_answered": {
"type": "boolean"
},
"last_activity_date": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"last_edit_date": {
"type": "long"
},
"last_editor": {
"properties": {
"accept_rate": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"display_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"link": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"profile_image": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"reputation": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"link": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"owner": {
"properties": {
"accept_rate": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"display_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"link": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"profile_image": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"reputation": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"question_id": {
"type": "long"
},
"reopen_vote_count": {
"type": "long"
},
"score": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"share_link": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"tags": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"title": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"up_vote_count": {
"type": "long"
},
"view_count": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
},
"settings": {
"index": {
"creation_date": "1581585452123",
"number_of_shards": "5",
"number_of_replicas": "1",
"uuid": "_nvo_G7ZRvqMIwLmddUXPw",
"version": {
"created": "6020399"
},
"provided_name": "questiondetails"
}
}
}
}
You can achieve what you want with a scripted terms aggregation, like this:
POST questions/_search
{
"size": 0,
"aggs": {
"ids": {
"terms": {
"script": "[doc['owner.user_id.keyword'].value, doc['reply.owner.user_id.keyword'].value]",
"size": 10
}
}
}
}
I have an index with objects named "DynamicFields" and each of them have inner objects named "Fields" like this:
{
"DynamicFields": [
{
"Fields": [
{
"DFieldVal": "Value1",
"Owned": 0,
"DFieldRelCode": 181254,
"DFieldCode": 1835
},
{
"DFieldVal": "Value2",
"Owned": 0,
"DFieldRelCode": 181255,
"DFieldCode": 1836
},
{
"DFieldVal": "Value3",
"Owned": 1,
"DFieldRelCode": 181256,
"DFieldCode": 1837
},
{
"DFieldVal": "Value4",
"Owned": 0,
"DFieldRelCode": 181257,
"DFieldCode": 1838
}
]
}
]
}
I need to find objects "DynamicFields" that has inner objects "Fields" with this exact values:
"DFieldCode": 1837
and
"Owned": 0
Im using this query for it, but it gives me wrong result, it should return an empty result because there isn't any inner object "Fields" having both of the values:
{
"from":0,
"size":10,
"query": {
"bool":{
"must":[
{ "terms": { "DynamicFields.Fields.Owned" : [0] } },
{ "terms": { "DynamicFields.Fields.DFieldCode" : [1837] } }
]
}
}
}
I think the problem is that Elastic search sees the inner objects properties as normal property for the Root Object so it returns the objects that have the mentioned fields in all inner objects no matter in the same inner object.
EDIT:
i have summarized the data to make it simpler
the mapping is full map of the data:
{
"marketplace": {
"mappings": {
"object": {
"properties": {
"Addresses": {
"properties": {
"AddrID": {
"type": "long"
},
"AddressText": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"AddressTree": {
"properties": {
"AddrFieldRelID": {
"type": "long"
},
"AddrTitleName": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"AddrTitlePersianName": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"AddrValName": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"Latitude": {
"type": "float"
},
"Longitude": {
"type": "float"
}
}
},
"Latitude": {
"type": "float"
},
"Longitude": {
"type": "float"
},
"Tel": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"DelFlag": {
"type": "long"
},
"DynamicFields": {
"properties": {
"DynamicDefCode": {
"type": "long"
},
"DynamicDefDataTypeName": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"DynamicDefName": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"DynamicValKind": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"Fields": {
"properties": {
"DFieldCode": {
"type": "long"
},
"DFieldRelCode": {
"type": "long"
},
"DFieldVal": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"Owned": {
"type": "boolean"
}
}
}
}
},
"GFRefCode": {
"type": "long"
},
"GoodsDesc": {
"properties": {
"FName": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"GoodsFullName": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"Supplier": {
"properties": {
"Barcode": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"GPackDayPrice": {
"type": "long"
},
"GoodsEnterDate": {
"type": "date"
},
"GoodsFinalCode": {
"type": "long"
},
"GoodsFullName": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"GoodsWHStock": {
"type": "long"
},
"StoreName": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"UserName": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"WHName": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"WareHouseCode": {
"type": "long"
}
}
},
"UserName": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"GoodsFinalCode": {
"type": "long"
},
"Images": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"IsMainObject": {
"type": "boolean"
},
"ObjectDetailPackID": {
"type": "long"
},
"ObjectKind": {
"type": "long"
},
"Prices": {
"properties": {
"Barcode": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"GPWeight": {
"type": "float"
},
"GpackDayPrice": {
"type": "long"
},
"PackingName": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"RefGoodsFinalCode": {
"type": "long"
},
"TreePath": {
"properties": {
"DFieldCode": {
"type": "long"
},
"DFieldRelCode": {
"type": "long"
},
"DFieldVal": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}
}
}
}
thanks.
As your index shows you saved your array as an object,
read more about this here
Basically unless specified otherwise elasticsearch flattens arrays when being saved, making objects in arrays lose their structure.
you should define the type of Fields as nested to avoid this.
I am trying to write a query that returns all docs which don't have a particular field which is a nested property but it doesn't work.
I am using ES 5.4
{
"query": {
"bool": {
"must_not": [
{
"exists": {
"field": "shares"
}
}
]
}
}
}
What am I doing wrong?
This is my mapping
{
"test": {
"aliases": {},
"mappings": {
"vendor": {
"properties": {
"address": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"categories": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"city": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"displayTypes": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"emailId": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"facebookAppId": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"foursquareType": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"googleTypes": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"hours": {
"properties": {
"day": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"from": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"to": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"isOnBoarded": {
"type": "boolean"
},
"kiaskCategories": {
"type": "keyword"
},
"latitude": {
"type": "float"
},
"location": {
"type": "geo_point"
},
"longitude": {
"type": "float"
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"phoneNumber": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"pictures": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"placeId": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"price": {
"type": "float"
},
"rating": {
"type": "float"
},
"shares": {
"type": "nested",
"properties": {
"_id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"content": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"createdDate": {
"type": "date"
},
"endDate": {
"type": "date"
},
"facebookLikes": {
"type": "long"
},
"images": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"price": {
"type": "long"
},
"quantity": {
"type": "long"
},
"shareType": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"startDate": {
"type": "date"
},
"tags": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"source": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"state": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"timezone": {
"type": "float"
},
"url": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"website": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"yelpTypes": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
},
"settings": {
"index": {
"creation_date": "1496319860700",
"number_of_shards": "5",
"number_of_replicas": "1",
"uuid": "0S0A9YN-S-SrW4eeOHX06w",
"version": {
"created": "5040099"
},
"provided_name": "test"
}
}
}
}
this should work
You have enforce nested data type mappings and use nested query.
{
"query" : {
"nested": {
"path": "shares",
"query": {
"bool": {
"must_not": [
{
"exists" : {
"field" : "shares._id"
}
}
]
}
}
}
}
}
Note: The following setup works for me.
PUT test_index
{
"mappings": {
"document_type" : {
"properties": {
"name" : {
"type": "text"
},
"shares" : {
"type": "nested"
}
}
}
}
}
POST test_index/document_type
{
"name" : "vicky"
}
POST test_index/_search
{
"query": {
"nested": {
"path": "shares",
"query": {
"bool": {
"must_not": [
{
"exists" : {
"field" : "shares.city"
}
}
]
}
}
}
}
}