I am getting unexpected result from Elasticsearch while searching though object property which are inside nested property. I am using elasticsearch-dsl python library for creating document and querying. Is this bug or I am missing something in querying and mapping parts? Below are the elasticsearch json mapping, query, unexpected result and expected result
Mapping:
{
"deal_acls": {
"type": "nested",
"properties": {
"created_at": {
"type": "date"
},
"created_by": {
"properties": {
"id": {
"type": "keyword"
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"status": {
"type": "keyword",
"normalizer": "lowercase"
}
}
},
"permission": {
"properties": {
"CRM": {
"properties": {
"description": {
"properties": {
"created_by": {
"type": "keyword",
"normalizer": "lowercase"
},
"object": {
"type": "keyword",
"normalizer": "lowercase"
},
"object_id": {
"type": "keyword",
"normalizer": "lowercase"
},
"timestamp": {
"type": "date"
}
}
},
"permission": {
"properties": {
"delete": {
"type": "integer"
},
"edit": {
"type": "integer"
},
"manage": {
"type": "long"
},
"read": {
"type": "integer"
},
"write": {
"type": "integer"
}
}
}
}
},
"deal": {
"properties": {
"description": {
"properties": {
"created_by": {
"type": "keyword",
"normalizer": "lowercase"
},
"object": {
"type": "keyword",
"normalizer": "lowercase"
},
"object_id": {
"type": "keyword",
"normalizer": "lowercase"
},
"timestamp": {
"type": "date"
}
}
},
"permission": {
"properties": {
"delete": {
"type": "integer"
},
"edit": {
"type": "integer"
},
"manage": {
"type": "long"
},
"read": {
"type": "integer"
},
"write": {
"type": "integer"
}
}
}
}
},
"document": {
"properties": {
"description": {
"properties": {
"created_by": {
"type": "keyword",
"normalizer": "lowercase"
},
"object": {
"type": "keyword",
"normalizer": "lowercase"
},
"object_id": {
"type": "keyword",
"normalizer": "lowercase"
},
"timestamp": {
"type": "date"
}
}
},
"permission": {
"properties": {
"delete": {
"type": "integer"
},
"edit": {
"type": "integer"
},
"manage": {
"type": "long"
},
"read": {
"type": "integer"
},
"write": {
"type": "integer"
}
}
}
}
},
"external_deal_team": {
"properties": {
"description": {
"properties": {
"created_by": {
"type": "keyword",
"normalizer": "lowercase"
},
"object": {
"type": "keyword",
"normalizer": "lowercase"
},
"object_id": {
"type": "keyword",
"normalizer": "lowercase"
},
"timestamp": {
"type": "date"
}
}
},
"permission": {
"properties": {
"delete": {
"type": "integer"
},
"edit": {
"type": "integer"
},
"manage": {
"type": "long"
},
"read": {
"type": "integer"
},
"write": {
"type": "integer"
}
}
}
}
},
"internal_deal_team": {
"properties": {
"description": {
"properties": {
"created_by": {
"type": "keyword",
"normalizer": "lowercase"
},
"object": {
"type": "keyword",
"normalizer": "lowercase"
},
"object_id": {
"type": "keyword",
"normalizer": "lowercase"
},
"timestamp": {
"type": "date"
}
}
},
"permission": {
"properties": {
"delete": {
"type": "integer"
},
"edit": {
"type": "integer"
},
"manage": {
"type": "long"
},
"read": {
"type": "integer"
},
"write": {
"type": "integer"
}
}
}
}
}
}
},
"status": {
"type": "keyword",
"normalizer": "lowercase"
},
"updated_at": {
"type": "date"
},
"updated_by": {
"properties": {
"id": {
"type": "keyword"
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"status": {
"type": "keyword",
"normalizer": "lowercase"
}
}
},
"user": {
"properties": {
"id": {
"type": "keyword"
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"status": {
"type": "keyword",
"normalizer": "lowercase"
}
}
}
}
}
}
Query:
{
"query": {
"bool": {
"must": [
{
"nested": {
"path": "deal_acls",
"query": {
"term": {
"deal_acls.permission.deal.permission.read": 1
}
}
}
},
{
"nested": {
"path": "deal_acls",
"query": {
"terms": {
"deal_acls.user.id": [
"5f7cea05-6562-4bdd-8448-19cfbe11783a"
]
}
}
}
}
]
}
}
}
Unexpected result: Since the deal permission of user with id=5f7cea05-6562-4bdd-8448-19cfbe11783a is 0, it should be returning empty hits.
{
"took": 2,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 1,
"relation": "eq"
},
"max_score": 2,
"hits": [
{
"_index": "dev.crecentric.dealvault.deals",
"_type": "_doc",
"_id": "a928838d-3422-41db-b40e-28f5c793f806",
"_score": 2,
"_source": {
"id": "a928838d-3422-41db-b40e-28f5c793f806",
"deal_acls": [
{
"user": {
"id": "5f7cea05-6562-4bdd-8448-19cfbe11783a",
"name": "testerrrrs testesssss",
"status": "active"
},
"permission": {
"deal": {
"permission": {
"edit": 0,
"read": 0,
"write": 0,
"delete": 0,
"manage": 0
},
"description": {
"object": "workspace",
"object_id": "fbc840b1-8727-4945-a070-fa1c105f9550",
"timestamp": "2022-02-10T05:46:05.140867+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
},
"external_deal_team": {
"permission": {
"edit": 0,
"read": 1,
"write": 1,
"delete": 0,
"manage": 0
},
"description": {
"object": "workspace",
"object_id": "fbc840b1-8727-4945-a070-fa1c105f9550",
"timestamp": "2022-02-10T05:46:05.140902+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
},
"internal_deal_team": {
"permission": {
"edit": 0,
"read": 1,
"write": 1,
"delete": 0,
"manage": 0
},
"description": {
"object": "workspace",
"object_id": "fbc840b1-8727-4945-a070-fa1c105f9550",
"timestamp": "2022-02-10T05:46:05.140910+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
}
},
"status": "active",
"created_at": "2022-02-10T05:40:15.727598+05:45",
"updated_at": "2022-02-10T05:46:05.177076+05:45"
},
{
"user": {
"id": "d78411e5-6645-4b95-a98c-db6db8748580",
"name": "Ramesh Pradhan",
"status": "active"
},
"permission": {
"CRM": {
"permission": {
"edit": 1,
"read": 1,
"write": 1,
"delete": 1,
"manage": 1
},
"description": {
"object": "owner",
"object_id": "d78411e5-6645-4b95-a98c-db6db8748580",
"timestamp": "2022-02-10T05:35:41.453881+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
},
"deal": {
"permission": {
"edit": 1,
"read": 1,
"write": 1,
"delete": 1,
"manage": 1
},
"description": {
"object": "owner",
"object_id": "d78411e5-6645-4b95-a98c-db6db8748580",
"timestamp": "2022-02-10T05:35:41.453881+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
},
"document": {
"permission": {
"edit": 1,
"read": 1,
"write": 1,
"delete": 1,
"manage": 1
},
"description": {
"object": "owner",
"object_id": "d78411e5-6645-4b95-a98c-db6db8748580",
"timestamp": "2022-02-10T05:35:41.453881+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
},
"external_deal_team": {
"permission": {
"edit": 1,
"read": 1,
"write": 1,
"delete": 1,
"manage": 1
},
"description": {
"object": "owner",
"object_id": "d78411e5-6645-4b95-a98c-db6db8748580",
"timestamp": "2022-02-10T05:35:41.453881+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
},
"internal_deal_team": {
"permission": {
"edit": 1,
"read": 1,
"write": 1,
"delete": 1,
"manage": 1
},
"description": {
"object": "owner",
"object_id": "d78411e5-6645-4b95-a98c-db6db8748580",
"timestamp": "2022-02-10T05:35:41.453881+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
}
},
"status": "active",
"created_at": "2022-02-10T05:35:41.453913+05:45",
"updated_at": "2022-02-10T05:35:41.462956+05:45"
}
]
}
}
]
}
}
Expected result:
{
"took": 2,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 1,
"relation": "eq"
},
"max_score": 2,
"hits": []
}
}
You are using two nested queries: so the result is a document which contains "deal_acls.permission.deal.permission.read": 1 and "deal_acls.user.id": ["5f7cea05-6562-4bdd-8448-19cfbe11783a"], but not necessary in the same subobject of the nested field. Try using a terms query inside a single nested query:
{
"query": {
"nested": {
"path": "deal_acls",
"query": {
"bool": {
"must": [
{ "match": { "deal_acls.user.id": "5f7cea05-6562-4bdd-8448-19cfbe11783a" }},
{ "match": { "deal_acls.permission.deal.permission.read": 1 }}
]
}
}
}
}
}
In case if someone is searching for answer, I have fixed this using this query:
{
"query": {
"nested": {
"path": "deal_acls",
"query": {
"bool": {
"must": [
{
"term": {
"deal_acls.permission.deal.permission.read": 1
}
},
{
"terms": {
"deal_acls.user.id": [
"5f7cea05-6562-4bdd-8448-19cfbe11783a"
]
}
}
]
}
}
}
}
}
Related
I am pretty new to elasticsearch and have been trying to create a query which would return me a record that matches all the must conditions of a bool-query. The bool-query is wrapped inside a constant_score: filter.
My mapping for the object is as below:
{
"mappings": {
"doc": {
"properties": {
"available_qty": {
"type": "long"
},
"brand": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"components": {
"type": "nested",
"properties": {
"available_qty": {
"type": "long"
},
"brand": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"id": {
"type": "long"
},
"incoming_qty": {
"type": "long"
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"on_hand_qty": {
"type": "long"
},
"outgoing_qty": {
"type": "long"
},
"parent_id": {
"type": "long"
},
"product_stores": {
"type": "nested",
"properties": {
"channel_id": {
"type": "long"
},
"price": {
"type": "float"
},
"store_id": {
"type": "long"
}
}
},
"product_warehouses": {
"type": "nested",
"properties": {
"available_qty": {
"type": "long"
},
"incoming_qty": {
"type": "long"
},
"outgoing_qty": {
"type": "long"
},
"warehouse_id": {
"type": "long"
}
}
},
"quantity_in_bundle": {
"type": "long"
},
"sku": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"status": {
"type": "long"
},
"tenantId": {
"type": "long"
},
"type": {
"type": "long"
}
}
},
"id": {
"type": "long"
},
"image": {
"properties": {
"id": {
"type": "long"
},
"isDefault": {
"type": "boolean"
},
"thumbnail": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"url": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"incoming_qty": {
"type": "long"
},
"tags": {
"type": "nested",
"properties": {
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"color": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"id": {
"type": "long"
}
}
},
"members": {
"type": "nested",
"properties": {
"available_qty": {
"type": "long"
},
"brand": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"id": {
"type": "long"
},
"image": {
"properties": {
"id": {
"type": "long"
},
"isDefault": {
"type": "boolean"
},
"url": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"incoming_qty": {
"type": "long"
},
"tags": {
"type": "nested",
"properties": {
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"color": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"id": {
"type": "long"
}
}
},
"master_id": {
"type": "long"
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"on_hand_qty": {
"type": "long"
},
"outgoing_qty": {
"type": "long"
},
"parent_id": {
"type": "long"
},
"product_stores": {
"type": "nested",
"properties": {
"channel_id": {
"type": "long"
},
"price": {
"type": "float"
},
"product_url": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"store_id": {
"type": "long"
}
}
},
"product_warehouses": {
"type": "nested",
"properties": {
"available_qty": {
"type": "long"
},
"incoming_qty": {
"type": "long"
},
"outgoing_qty": {
"type": "long"
},
"warehouse_id": {
"type": "long"
}
}
},
"sku": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"status": {
"type": "long"
},
"tenantId": {
"type": "long"
},
"type": {
"type": "long"
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"on_hand_qty": {
"type": "long"
},
"outgoing_qty": {
"type": "long"
},
"product_stores": {
"type": "nested",
"properties": {
"channel_id": {
"type": "long"
},
"price": {
"type": "float"
},
"product_url": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"store_id": {
"type": "long"
}
}
},
"product_warehouses": {
"type": "nested",
"properties": {
"available_qty": {
"type": "long"
},
"incoming_qty": {
"type": "long"
},
"outgoing_qty": {
"type": "long"
},
"warehouse_id": {
"type": "long"
}
}
},
"sku": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"status": {
"type": "long"
},
"tenantId": {
"type": "long"
},
"type": {
"type": "long"
}
}
}
}
}
I am querying this index using the below query:
{
"from": 0,
"size": 20,
"query": {
"constant_score": {
"filter": {
"bool": {
"must": [
{
"term": {
"tenantId": {
"value": 88,
"boost": 1
}
}
},
{
"terms": {
"type": [
2
],
"boost": 1
}
},
{
"bool": {
"should": [
{
"terms": {
"status": [
2
],
"boost": 1
}
},
{
"nested": {
"query": {
"terms": {
"members.status": [
2
],
"boost": 1
}
},
"path": "members",
"ignore_unmapped": false,
"score_mode": "avg",
"boost": 1,
"inner_hits": {
"name": "members",
"ignore_unmapped": false,
"from": 0,
"size": 100,
"version": false,
"explain": false,
"track_scores": false,
"_source": false
}
}
}
],
"adjust_pure_negative": true,
"minimum_should_match": "1",
"boost": 1
}
},
{
"bool": {
"should": [
{
"nested": {
"query": {
"terms": {
"product_stores.store_id": [
20889
],
"boost": 1
}
},
"path": "product_stores",
"ignore_unmapped": false,
"score_mode": "avg",
"boost": 1
}
},
{
"nested": {
"query": {
"nested": {
"query": {
"terms": {
"members.product_stores.store_id": [
20889
],
"boost": 1
}
},
"path": "members.product_stores",
"ignore_unmapped": false,
"score_mode": "avg",
"boost": 1
}
},
"path": "members",
"ignore_unmapped": false,
"score_mode": "avg",
"boost": 1,
"inner_hits": {
"name": "members",
"ignore_unmapped": false,
"from": 0,
"size": 100,
"version": false,
"explain": false,
"track_scores": false,
"_source": false
}
}
}
],
"adjust_pure_negative": true,
"minimum_should_match": "1",
"boost": 1
}
},
{
"bool": {
"should": [
{
"nested": {
"query": {
"terms": {
"tags.id": [
1001
],
"boost": 1
}
},
"path": "tags",
"ignore_unmapped": false,
"score_mode": "avg",
"boost": 1
}
},
{
"nested": {
"query": {
"nested": {
"query": {
"terms": {
"members.tags.id": [
1001
],
"boost": 1
}
},
"path": "members.tags",
"ignore_unmapped": false,
"score_mode": "avg",
"boost": 1
}
},
"path": "members",
"ignore_unmapped": false,
"score_mode": "avg",
"boost": 1,
"inner_hits": {
"name": "members",
"ignore_unmapped": false,
"from": 0,
"size": 100,
"version": false,
"explain": false,
"track_scores": false,
"_source": false
}
}
}
],
"adjust_pure_negative": true,
"minimum_should_match": "1",
"boost": 1
}
}
],
"adjust_pure_negative": true,
"boost": 1
}
},
"boost": 1
}
},
"sort": [
{
"_id": {
"order": "desc"
}
}
]
}
What I am trying to achieve with this is to fetch the object which has at least one nested object that matches all the must conditions in the main bool query. But I am getting records even if there is not a single object that matches all 3. For example, if there's a record that only matches status and tags but not the store_ids, that elastic search will return it despite the all the conditions are part of must. Can anyone please explain me the behaviour of this query? I tried reading documents but I am at loss.
Any pointer or guidance will be much appreciated.
Thank you!
UPDATE:
I fixed this issue by merging all the nested shoulds under a single nested must.
In your query, you're using a should clause which will return results even if they don't match all of the should clause conditions. Your must clause takes precedence over the should clause.
According to the Bool Query documentation, you could adjust the minimum should match parameter.
I need to query elasticsearch & filter the result to be in a range of dates.
the thing is the date property is mapped as a string.
is it possible to do so ?
this is the search query i'm using:
{
"size": 1,
"from": 0,
"query": {
"bool": {
"must": [
{ "match": { "status": "active" }},
{ "match": { "last_action_state": "accepted" }}
],
"filter": [
{"missing" : { "field" : "store_id" }},
{ "range": { "list_time": { "gte": "2017/01/01 00:00:00", "lte": "2017/03/01 23:59:59", "format": "yyyy/MM/dd HH:mm:ss"}}}
]
}
}
}
the thing is i have no control over the mapping since it's created automatically by another program which index the documents, and i can't change the mapping once it's created.
ps: elasticsearch version: 2.3
UPDATE:
index info:
{
"avindex_v3": {
"aliases": {
"avindex": {}
},
"mappings": {
"ads": {
"properties": {
"account_id": {
"type": "long"
},
"ad_id": {
"type": "long"
},
"ad_params": {
"type": "string"
},
"body": {
"type": "string"
},
"category": {
"type": "long"
},
"city": {
"type": "long"
},
"company_ad": {
"type": "boolean"
},
"email": {
"type": "string"
},
"images": {
"type": "string"
},
"lang": {
"type": "string"
},
"last_action_state": {
"type": "string"
},
"list_date": {
"type": "long"
},
"list_id": {
"type": "long"
},
"list_time": {
"type": "string"
},
"modified_at": {
"type": "string"
},
"modified_ts": {
"type": "double"
},
"name": {
"type": "string"
},
"orig_date": {
"type": "long"
},
"orig_list_time": {
"type": "string"
},
"phone": {
"type": "string"
},
"phone_hidden": {
"type": "boolean"
},
"price": {
"type": "long"
},
"region": {
"type": "long"
},
"status": {
"type": "string"
},
"store_id": {
"type": "long"
},
"subject": {
"type": "string"
},
"type": {
"type": "string"
},
"user_id": {
"type": "long"
}
}
}
},
"settings": {
"index": {
"creation_date": "1493216710928",
"number_of_shards": "5",
"number_of_replicas": "1",
"uuid": "WEHGLF8iRyGk3Xgbmo7H8Q",
"version": {
"created": "2040499"
}
}
},
"warmers": {}
}
}
You can try to give it as a keyword like this :
{
"range": {
"list_time.keyword": {
"gte": "2020-08-12 22:24:55.56",
"lte": "2020-08-12 22:24:56.56"
}
}
}
I have mapping as follow
{
"dsi2": {
"mappings": {
"dsi2": {
"_all": {
"enabled": true
},
"properties": {
"instance": {
"properties": {
"assignee": {
"type": "string"
},
"createdTs": {
"type": "long"
},
"dataSourceId": {
"type": "integer"
},
"dsTypeId": {
"type": "integer"
},
"entitlements": {
"properties": {
"create": {
"type": "boolean"
},
"delete": {
"type": "boolean"
},
"edit": {
"type": "boolean"
},
"read": {
"type": "boolean"
},
"roleId": {
"type": "integer"
}
},
"type": "nested"
},
"formDefinitionId": {
"type": "long"
},
"formTypeId": {
"type": "long"
},
"id": {
"fields": {
"raw": {
"type": "integer"
}
},
"type": "integer"
},
"instFields": {
"properties": {
"fieldBoolean": {
"type": "boolean"
},
"fieldDate": {
"format": "strict_date_optional_time||epoch_millis",
"type": "date"
},
"fieldDouble": {
"fields": {
"raw": {
"type": "double"
}
},
"type": "double"
},
"fieldLong": {
"fields": {
"raw": {
"type": "long"
}
},
"type": "long"
},
"fieldString": {
"fields": {
"raw": {
"index": "not_analyzed",
"type": "string"
}
},
"type": "string"
},
"fieldValue": {
"fields": {
"raw": {
"index": "not_analyzed",
"type": "string"
}
},
"type": "string"
},
"isKey": {
"type": "boolean"
},
"referredInstKeyData": {
"properties": {
"fieldValue": {
"fields": {
"raw": {
"index": "not_analyzed",
"type": "string"
}
},
"type": "string"
},
"sortOrder": {
"type": "integer"
},
"sourceFieldId": {
"type": "integer"
}
},
"type": "nested"
},
"sortOrder": {
"type": "long"
},
"sourceFieldId": {
"type": "integer"
}
},
"type": "nested"
},
"instanceId": {
"type": "integer"
},
"modifiedTs": {
"type": "long"
},
"parentEntityId": {
"type": "integer"
},
"processId": {
"type": "string"
},
"processName": {
"type": "string"
},
"relatedInstances": {
"properties": {
"createdTs": {
"type": "long"
},
"dataSourceId": {
"type": "integer"
},
"dsTypeId": {
"type": "integer"
},
"entitlements": {
"properties": {
"create": {
"type": "boolean"
},
"delete": {
"type": "boolean"
},
"edit": {
"type": "boolean"
},
"read": {
"type": "boolean"
},
"roleId": {
"type": "long"
}
}
},
"formDefinitionId": {
"type": "long"
},
"formTypeId": {
"type": "long"
},
"id": {
"type": "integer"
},
"instFields": {
"properties": {
"fieldBoolean": {
"type": "boolean"
},
"fieldDate": {
"format": "strict_date_optional_time||epoch_millis",
"type": "date"
},
"fieldDouble": {
"fields": {
"raw": {
"type": "double"
}
},
"type": "double"
},
"fieldLong": {
"fields": {
"raw": {
"type": "long"
}
},
"type": "long"
},
"fieldString": {
"fields": {
"raw": {
"index": "not_analyzed",
"type": "string"
}
},
"type": "string"
},
"fieldValue": {
"fields": {
"raw": {
"index": "not_analyzed",
"type": "string"
}
},
"type": "string"
},
"isKey": {
"type": "boolean"
},
"referredInstKeyData": {
"properties": {
"fieldString": {
"fields": {
"raw": {
"index": "not_analyzed",
"type": "string"
}
},
"type": "string"
},
"fieldValue": {
"fields": {
"raw": {
"index": "not_analyzed",
"type": "string"
}
},
"type": "string"
},
"sortOrder": {
"type": "integer"
},
"sourceFieldId": {
"type": "integer"
}
},
"type": "nested"
},
"sortOrder": {
"type": "long"
},
"sourceFieldId": {
"type": "integer"
}
},
"type": "nested"
},
"instanceId": {
"type": "integer"
},
"modifiedTs": {
"type": "long"
},
"parentEntityId": {
"type": "integer"
},
"status": {
"type": "long"
},
"tenantId": {
"type": "long"
}
},
"type": "nested"
},
"status": {
"type": "long"
},
"taskAssignees": {
"properties": {
"role": {
"type": "integer"
},
"user": {
"type": "integer"
}
},
"type": "nested"
},
"taskCompleted": {
"format": "strict_date_optional_time||epoch_millis",
"type": "date"
},
"taskCompletedBy": {
"type": "string"
},
"taskCreated": {
"format": "strict_date_optional_time||epoch_millis",
"type": "date"
},
"taskName": {
"type": "string"
},
"tenantId": {
"type": "long"
}
}
},
"status": {
"type": "integer"
},
"tenantId": {
"type": "integer"
}
}
}
}
}
}
when using following query
{
"query": {
"nested": {
"path": "instance.instFields",
"query": {
"bool": {
"must": [
{
"term": {
"instance.instFields.fieldValue": "120656"
}
},
{
"exists": {
"field": "instance.instFields.referredInstKeyData"
}
}
]
}
}
}
}
}
one of the example documents in the index is as follow
{
"took": 1,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"failed": 0
},
"hits": {
"total": 1,
"max_score": 10.130359,
"hits": [
{
"_shard": 0,
"_node": "_FohF7-YRyCSh82NmYVslQ",
"_index": "dsi2",
"_type": "dsi2",
"_id": "120655",
"_score": 10.130359,
"_source": {
"instance": {
"entitlements": [
{
"read": true,
"edit": false,
"roleId": 169,
"create": false,
"delete": false
}
],
"formTypeId": 3,
"parentEntityId": 23589,
"createdTs": 1487701846526,
"dataSourceId": 22,
"dsTypeId": 5,
"modifiedTs": 1487783663308,
"instanceId": 17790,
"instFields": [
{
"fieldDouble": 120658,
"sourceFieldId": 1594,
"sortOrder": 0,
"referredInstKeyData": [
{
"sourceFieldId": 1233,
"sortOrder": 0,
"fieldValue": "Depot Transport"
},
{
"sourceFieldId": 1232,
"sortOrder": 1,
"fieldValue": "LONDON"
},
{
"sourceFieldId": 1236,
"sortOrder": 4,
"fieldValue": "513069"
},
{
"sourceFieldId": 1658,
"sortOrder": 5,
"fieldValue": "7378"
}
],
"isKey": false,
"fieldValue": "120658",
"fieldString": "120658"
},
{
"fieldDouble": 120656,
"sourceFieldId": 1593,
"sortOrder": 0,
"referredInstKeyData": [
{
"sourceFieldId": 1233,
"sortOrder": 0,
"fieldValue": "Joe Cadillac Ii Ltd"
},
{
"sourceFieldId": 1236,
"sortOrder": 4,
"fieldValue": "518508"
},
{
"sourceFieldId": 1658,
"sortOrder": 5,
"fieldValue": "7376"
}
],
"isKey": false,
"fieldValue": "120656",
"fieldString": "120656"
},
{
"sourceFieldId": 2580,
"sortOrder": 0,
"isKey": false,
"fieldValue": "52fe850d-fc17-4dd9-a3a2-257485a9eb27",
"fieldString": "52fe850d-fc17-4dd9-a3a2-257485a9eb27"
}
],
"processName": "Vehicle Delivery",
"tenantId": 3,
"formDefinitionId": 729,
"id": 120655,
"assignee": "none",
"status": 4
},
"formTypeId": 3,
"tenantId": 3,
"status": 4
}
}
]
}
}
it's not returning documents where fieldValue": "120656" and referredInstKeyData element is in there. any idea what could be causing exists to not work? Elastic search version is 2.3.1
this is a known issue in Elastic search. exists isn't supported for nested properties. confirmed here. https://github.com/elastic/elasticsearch/issues/23325
I have an index which contains CustomerProfile documents. Each of this document in the CustomerInsightTargets(with the properties Source,Value) property can be an array with x items. What I am trying to achieve is an autocomplete (of top 5) on CustomerInsightTargets.Value grouped by CustomerInisghtTarget.Source.
It will be helpful if anyone gives me hint about how to select only a subset of nested objects from each document and use that nested obj in aggregations.
{
"customerinsights": {
"aliases": {},
"mappings": {
"customerprofile": {
"properties": {
"CreatedById": {
"type": "long"
},
"CreatedDateTime": {
"type": "date"
},
"CustomerInsightTargets": {
"type": "nested",
"properties": {
"CustomerInsightSource": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"CustomerInsightValue": {
"type": "text",
"term_vector": "yes",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
},
"analyzer": "ngram_tokenizer_analyzer"
},
"CustomerProfileId": {
"type": "long"
},
"Guid": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"Id": {
"type": "long"
}
}
},
"DisplayName": {
"type": "text",
"term_vector": "yes",
"analyzer": "ngram_tokenizer_analyzer"
},
"Email": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"Id": {
"type": "long"
},
"ImageUrl": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
},
"settings": {
"index": {
"number_of_shards": "1",
"provided_name": "customerinsights",
"creation_date": "1484860145041",
"analysis": {
"analyzer": {
"ngram_tokenizer_analyzer": {
"type": "custom",
"tokenizer": "ngram_tokenizer"
}
},
"tokenizer": {
"ngram_tokenizer": {
"type": "nGram",
"min_gram": "1",
"max_gram": "10"
}
}
},
"number_of_replicas": "2",
"uuid": "nOyI0O2cTO2JOFvqIoE8JQ",
"version": {
"created": "5010199"
}
}
}
}
}
Having as example a document:
{
{
"Id": 9072856,
"CreatedDateTime": "2017-01-12T11:26:58.413Z",
"CreatedById": 9108469,
"DisplayName": "valentinos",
"Email": "valentinos#mail.com",
"CustomerInsightTargets": [
{
"Id": 160,
"CustomerProfileId": 9072856,
"CustomerInsightSource": "Tags",
"CustomerInsightValue": "Tag1",
"Guid": "00000000-0000-0000-0000-000000000000"
},
{
"Id": 160,
"CustomerProfileId": 9072856,
"CustomerInsightSource": "ProfileName",
"CustomerInsightValue": "valentinos",
"Guid": "00000000-0000-0000-0000-000000000000"
},
{
"Id": 160,
"CustomerProfileId": 9072856,
"CustomerInsightSource": "Playground",
"CustomerInsightValue": "Wiki",
"Guid": "00000000-0000-0000-0000-000000000000"
}
]
}
}
If i ran an aggregation on the top_hits the result will include all targets from a document -> if one of them match my search text.
Example
GET customerinsights/_search
{
"query": {
"bool": {
"must": [
{
"nested": {
"path": "CustomerInsightTargets",
"query": {
"bool": {
"must": [
{
"match": {
"CustomerInsightTargets.CustomerInsightValue": {
"query": "2017",
"operator": "AND",
"fuzziness": 2
}
}
}
]
}
}
}
}
]
}
} ,
"aggs": {
"root": {
"nested": {
"path": "CustomerInsightTargets"
},
"aggs": {
"top_tags": {
"terms": {
"field": "CustomerInsightTargets.CustomerInsightSource.keyword"
},
"aggs": {
"top_tag_hits": {
"top_hits": {
"sort": [
{
"_score": {
"order": "desc"
}
}
],
"size": 5,
"_source": "CustomerInsightTargets"
}
}
}
}
}
}
},
"size": 0,
"_source": "CustomerInsightTargets"
}
My question is how I should use the aggregation to get the "autocomplete" Values grouped by Source and order by the _score. I tried to use a significant_terms aggregation but doesn't work so well, also terms aggs doesn't sort by score (and by _count) and having fuzzy also adds complexity.
I am having trouble with a simple exist filter. What I am trying to achieve is , return all documents which have specific field.
following is he mapping for my index -
"book": {
"dynamic": "true",
"properties": {
"currencies": {
"dynamic": "true",
"properties": {
"bookChallenge000": {
"type": "long"
},
"achievements": {
"type": "long"
},
"bookChallenge001": {
"type": "long"
},
"giftsSent": {
"type": "long"
},
"stars": {
"type": "long"
},
"knightsDonated": {
"type": "long"
}
}
},
"level": {
"type": "long"
},
"description": {
"index_analyzer": "str_index_analyzer",
"search_analyzer": "str_search_analyzer",
"type": "string"
},
"additionalMaxMembers": {
"type": "long"
},
"name": {
"index_analyzer": "str_index_analyzer",
"search_analyzer": "str_search_analyzer",
"type": "string"
},
"lastUpdated": {
"type": "long"
},
"active": {
"type": "boolean"
},
"bookClubId": {
"type": "string"
},
"attributes": {
"dynamic": "true",
"properties": {
"level": {
"type": "long"
},
"description": {
"index_analyzer": "str_index_analyzer",
"search_analyzer": "str_search_analyzer",
"type": "string"
},
"additionalMaxMembers": {
"type": "long"
},
"name": {
"index_analyzer": "str_index_analyzer",
"search_analyzer": "str_search_analyzer",
"type": "string"
},
"lastUpdated": {
"type": "long"
},
"active": {
"type": "boolean"
},
"memberCount": {
"type": "long"
},
"inviteStatus": {
"type": "string"
}
}
},
"meta": {
"dynamic": "false",
"properties": {
"bookRewards": {
"dynamic": "false",
"type": "object"
},
"challenges": {
"dynamic": "false",
"type": "object"
},
"grantedRewardInfo": {
"dynamic": "false",
"type": "object"
},
"levelRequirement": {
"type": "long"
},
"membersData": {
"dynamic": "false",
"type": "object"
},
"language": {
"type": "string"
},
"conversations": {
"dynamic": "false",
"type": "object"
},
"bookGoals": {
"dynamic": "false",
"type": "object"
},
"banner": {
"type": "string"
}
}
},
"memberCount": {
"type": "long"
},
"inviteStatus": {
"type": "string"
},
"version": {
"type": "long"
}
}
}
here is a document I am searching for -
{
attributes: {
active: true,
additionalMaxMembers: 0,
description: "famous five",
inviteStatus: "OPEN",
lastUpdated: 1452119518547,
level: 1,
memberCount: 39,
name: "zero"
},
currencies: {
bookChallenge000: 74316,
bookChallenge001: 142580,
bookChallenge002: 165526,
achievements: 582,
giftsSent: 0,
knightsDonated: 161,
stars: 1104
},
meta: {
banner: "enidBlyton_17",
challenges: {
Event014_Challenge_A: {
Currency: "bookChallenge000",
finishtime: 20550267,
goalscore: 30240,
memberscores: {
37339561405: 10,
37349022668: 2000,
37432453846: 20,
37437075798: 0
},
playerCurrency: "UserChallenge000"
},
Event014_Challenge_B: {
Currency: "bookChallenge001",
finishtime: 20290830,
goalscore: 38003,
memberscores: {
37339561405: 20,
37349022668: 38,
37432453846: 590
},
playerCurrency: "UserChallenge000"
}
},
language: "en_US",
levelRequirement: 14,
membersData: {
37442021220: {
achievements: 9,
giftsSent: 0,
knightsDonated: 0,
playerLikes: 28,
stars: 20
},
37493332413: {
achievements: 4,
giftsSent: 0,
knightsDonated: 0,
playerLikes: 0,
stars: 20
}
}
},
active: true,
bookClubId: "6106890",
inviteStatus: "OPEN",
memberCount: 39,
additionalMaxMembers: 0,
lastUpdated: 1452119518547,
description: "famous five",
name: "zero",
level: 1
}
My query is pretty straight forward , I need all documents which have challenges field in them.
curl -X'GET' localhost:9200/book_index/_search -d '{"filter": {"exists" : { "field" : "challenges" }}}'
but it doesn't return any document, i have 100s of documents with challenge info.
I tried meta.challenges, and also challenges.membersData but it doesn't work.
though if I directly look for banner or language field, I get expected results, but not for object type fields.
How can I get all documents with challenges or membersData?
Please advise what am I missing.
That is the query you are looking for:
{
"query": {
"filtered": {
"filter": {
"bool": {
"should": [
{
"exists": {
"field": "meta.challenges"
}
},
{
"exists": {
"field": "meta.membersData"
}
}
]
}
}
}
}
}