Related
I am getting unexpected result from Elasticsearch while searching though object property which are inside nested property. I am using elasticsearch-dsl python library for creating document and querying. Is this bug or I am missing something in querying and mapping parts? Below are the elasticsearch json mapping, query, unexpected result and expected result
Mapping:
{
"deal_acls": {
"type": "nested",
"properties": {
"created_at": {
"type": "date"
},
"created_by": {
"properties": {
"id": {
"type": "keyword"
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"status": {
"type": "keyword",
"normalizer": "lowercase"
}
}
},
"permission": {
"properties": {
"CRM": {
"properties": {
"description": {
"properties": {
"created_by": {
"type": "keyword",
"normalizer": "lowercase"
},
"object": {
"type": "keyword",
"normalizer": "lowercase"
},
"object_id": {
"type": "keyword",
"normalizer": "lowercase"
},
"timestamp": {
"type": "date"
}
}
},
"permission": {
"properties": {
"delete": {
"type": "integer"
},
"edit": {
"type": "integer"
},
"manage": {
"type": "long"
},
"read": {
"type": "integer"
},
"write": {
"type": "integer"
}
}
}
}
},
"deal": {
"properties": {
"description": {
"properties": {
"created_by": {
"type": "keyword",
"normalizer": "lowercase"
},
"object": {
"type": "keyword",
"normalizer": "lowercase"
},
"object_id": {
"type": "keyword",
"normalizer": "lowercase"
},
"timestamp": {
"type": "date"
}
}
},
"permission": {
"properties": {
"delete": {
"type": "integer"
},
"edit": {
"type": "integer"
},
"manage": {
"type": "long"
},
"read": {
"type": "integer"
},
"write": {
"type": "integer"
}
}
}
}
},
"document": {
"properties": {
"description": {
"properties": {
"created_by": {
"type": "keyword",
"normalizer": "lowercase"
},
"object": {
"type": "keyword",
"normalizer": "lowercase"
},
"object_id": {
"type": "keyword",
"normalizer": "lowercase"
},
"timestamp": {
"type": "date"
}
}
},
"permission": {
"properties": {
"delete": {
"type": "integer"
},
"edit": {
"type": "integer"
},
"manage": {
"type": "long"
},
"read": {
"type": "integer"
},
"write": {
"type": "integer"
}
}
}
}
},
"external_deal_team": {
"properties": {
"description": {
"properties": {
"created_by": {
"type": "keyword",
"normalizer": "lowercase"
},
"object": {
"type": "keyword",
"normalizer": "lowercase"
},
"object_id": {
"type": "keyword",
"normalizer": "lowercase"
},
"timestamp": {
"type": "date"
}
}
},
"permission": {
"properties": {
"delete": {
"type": "integer"
},
"edit": {
"type": "integer"
},
"manage": {
"type": "long"
},
"read": {
"type": "integer"
},
"write": {
"type": "integer"
}
}
}
}
},
"internal_deal_team": {
"properties": {
"description": {
"properties": {
"created_by": {
"type": "keyword",
"normalizer": "lowercase"
},
"object": {
"type": "keyword",
"normalizer": "lowercase"
},
"object_id": {
"type": "keyword",
"normalizer": "lowercase"
},
"timestamp": {
"type": "date"
}
}
},
"permission": {
"properties": {
"delete": {
"type": "integer"
},
"edit": {
"type": "integer"
},
"manage": {
"type": "long"
},
"read": {
"type": "integer"
},
"write": {
"type": "integer"
}
}
}
}
}
}
},
"status": {
"type": "keyword",
"normalizer": "lowercase"
},
"updated_at": {
"type": "date"
},
"updated_by": {
"properties": {
"id": {
"type": "keyword"
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"status": {
"type": "keyword",
"normalizer": "lowercase"
}
}
},
"user": {
"properties": {
"id": {
"type": "keyword"
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"status": {
"type": "keyword",
"normalizer": "lowercase"
}
}
}
}
}
}
Query:
{
"query": {
"bool": {
"must": [
{
"nested": {
"path": "deal_acls",
"query": {
"term": {
"deal_acls.permission.deal.permission.read": 1
}
}
}
},
{
"nested": {
"path": "deal_acls",
"query": {
"terms": {
"deal_acls.user.id": [
"5f7cea05-6562-4bdd-8448-19cfbe11783a"
]
}
}
}
}
]
}
}
}
Unexpected result: Since the deal permission of user with id=5f7cea05-6562-4bdd-8448-19cfbe11783a is 0, it should be returning empty hits.
{
"took": 2,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 1,
"relation": "eq"
},
"max_score": 2,
"hits": [
{
"_index": "dev.crecentric.dealvault.deals",
"_type": "_doc",
"_id": "a928838d-3422-41db-b40e-28f5c793f806",
"_score": 2,
"_source": {
"id": "a928838d-3422-41db-b40e-28f5c793f806",
"deal_acls": [
{
"user": {
"id": "5f7cea05-6562-4bdd-8448-19cfbe11783a",
"name": "testerrrrs testesssss",
"status": "active"
},
"permission": {
"deal": {
"permission": {
"edit": 0,
"read": 0,
"write": 0,
"delete": 0,
"manage": 0
},
"description": {
"object": "workspace",
"object_id": "fbc840b1-8727-4945-a070-fa1c105f9550",
"timestamp": "2022-02-10T05:46:05.140867+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
},
"external_deal_team": {
"permission": {
"edit": 0,
"read": 1,
"write": 1,
"delete": 0,
"manage": 0
},
"description": {
"object": "workspace",
"object_id": "fbc840b1-8727-4945-a070-fa1c105f9550",
"timestamp": "2022-02-10T05:46:05.140902+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
},
"internal_deal_team": {
"permission": {
"edit": 0,
"read": 1,
"write": 1,
"delete": 0,
"manage": 0
},
"description": {
"object": "workspace",
"object_id": "fbc840b1-8727-4945-a070-fa1c105f9550",
"timestamp": "2022-02-10T05:46:05.140910+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
}
},
"status": "active",
"created_at": "2022-02-10T05:40:15.727598+05:45",
"updated_at": "2022-02-10T05:46:05.177076+05:45"
},
{
"user": {
"id": "d78411e5-6645-4b95-a98c-db6db8748580",
"name": "Ramesh Pradhan",
"status": "active"
},
"permission": {
"CRM": {
"permission": {
"edit": 1,
"read": 1,
"write": 1,
"delete": 1,
"manage": 1
},
"description": {
"object": "owner",
"object_id": "d78411e5-6645-4b95-a98c-db6db8748580",
"timestamp": "2022-02-10T05:35:41.453881+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
},
"deal": {
"permission": {
"edit": 1,
"read": 1,
"write": 1,
"delete": 1,
"manage": 1
},
"description": {
"object": "owner",
"object_id": "d78411e5-6645-4b95-a98c-db6db8748580",
"timestamp": "2022-02-10T05:35:41.453881+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
},
"document": {
"permission": {
"edit": 1,
"read": 1,
"write": 1,
"delete": 1,
"manage": 1
},
"description": {
"object": "owner",
"object_id": "d78411e5-6645-4b95-a98c-db6db8748580",
"timestamp": "2022-02-10T05:35:41.453881+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
},
"external_deal_team": {
"permission": {
"edit": 1,
"read": 1,
"write": 1,
"delete": 1,
"manage": 1
},
"description": {
"object": "owner",
"object_id": "d78411e5-6645-4b95-a98c-db6db8748580",
"timestamp": "2022-02-10T05:35:41.453881+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
},
"internal_deal_team": {
"permission": {
"edit": 1,
"read": 1,
"write": 1,
"delete": 1,
"manage": 1
},
"description": {
"object": "owner",
"object_id": "d78411e5-6645-4b95-a98c-db6db8748580",
"timestamp": "2022-02-10T05:35:41.453881+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
}
},
"status": "active",
"created_at": "2022-02-10T05:35:41.453913+05:45",
"updated_at": "2022-02-10T05:35:41.462956+05:45"
}
]
}
}
]
}
}
Expected result:
{
"took": 2,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 1,
"relation": "eq"
},
"max_score": 2,
"hits": []
}
}
You are using two nested queries: so the result is a document which contains "deal_acls.permission.deal.permission.read": 1 and "deal_acls.user.id": ["5f7cea05-6562-4bdd-8448-19cfbe11783a"], but not necessary in the same subobject of the nested field. Try using a terms query inside a single nested query:
{
"query": {
"nested": {
"path": "deal_acls",
"query": {
"bool": {
"must": [
{ "match": { "deal_acls.user.id": "5f7cea05-6562-4bdd-8448-19cfbe11783a" }},
{ "match": { "deal_acls.permission.deal.permission.read": 1 }}
]
}
}
}
}
}
In case if someone is searching for answer, I have fixed this using this query:
{
"query": {
"nested": {
"path": "deal_acls",
"query": {
"bool": {
"must": [
{
"term": {
"deal_acls.permission.deal.permission.read": 1
}
},
{
"terms": {
"deal_acls.user.id": [
"5f7cea05-6562-4bdd-8448-19cfbe11783a"
]
}
}
]
}
}
}
}
}
How do I always return the documents with the lowest value in the "url_length" field regardless of (from) that I sent to search?
in the query below, I request the results that have the word (netflix) and that the field (pgrk) is between 9 and 10 and that the field (url_length) is less than 4, but when I put it ("from": 1, "size ": 1) does not return the doc of (_id 15) that has the word (netflix) the field pgrk = 10 and the field (url_length) = 2. Returns the doc of (_id 14) that has the word (netflix) the field pgrk = 10 and the field (url_length) = 3
just return the doc of (_id 15) that has the field (url_length) = 2 if I put it in the query from ZERO ("from": 0, "size": 1)
because I had it searched ("from": 1, "size": 1,) and didn't bring the record of (_id 15) that has the ("url_length" = 2) brought the record of (_id 14) that has the ("url_length" = 3)
{
"from": 1,
"size": 1,
"sort": [
{
"pgrk": {
"order": "desc"
}
},
{
"url_length": {
"order": "asc"
}
}
],
"query": {
"bool": {
"must": {
"multi_match": {
"query": "netflix",
"type": "cross_fields",
"fields": [
"tittle",
"description",
"url"
],
"operator": "and"
}
},
"filter": [
{
"range": {
"pgrk": {
"gte": 9,
"lte" : 10
}
}
},
{
"range": {
"url_length": {
"lt" : 4
}
}
}
]
}
}
}
if I put ("from": 1, "size": 1,) it does not return the record (_id 15) that has "url_length = 2" returns the doc of _id 14 that has "url_length = 3" as shown below:
{
"took": 0,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 2,
"relation": "eq"
},
"max_score": null,
"hits": [
{
"_index": "teste",
"_type": "_doc",
"_id": "14",
"_score": null,
"_source": {
"url": "www.333.com",
"title": "netflix netflix netflix netflix netflix netflix netflix netflix netflix netflix",
"description": "tudo sobre netflix netflix netflix netflix netflix netflix",
"pgrk": "10",
"url_length": "3"
},
"sort": [
10,
3
]
}
]
}
}
if I put ("from": 0, "size": 1,) then it returns the record (_id 15) that has "url_length = 2"
{
"took": 0,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 2,
"relation": "eq"
},
"max_score": null,
"hits": [
{
"_index": "teste",
"_type": "_doc",
"_id": "15",
"_score": null,
"_source": {
"url": "www.netflix.yahoo.com",
"title": "melhor filme",
"description": "tudo sobre series",
"pgrk": "10",
"url_length": "2"
},
"sort": [
10,
2
]
}
]
}
}
how do I always return the documents with the lowest value in the "url_length" field regardless of (from) that I sent to search?
follows my mapping:
{
"settings": {
"index": {
"number_of_shards": "5",
"number_of_replicas": "0",
"analysis": {
"filter": {
"stemmer_plural_portugues": {
"name": "minimal_portuguese",
"stopwords" : ["http", "https", "ftp", "www"],
"type": "stemmer"
}
},
"analyzer": {
"analyzer_customizado": {
"filter": [
"lowercase",
"stemmer_plural_portugues",
"asciifolding"
],
"tokenizer": "lowercase"
}
}
}
}
},
"mappings": {
"properties": {
"q": {
"type": "text",
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
}
},
"id": {
"type": "long"
},
"#timestamp": {
"type": "date"
},
"data": {
"type": "date"
},
"#version": {
"type": "text",
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
}
},
"quebrado": {
"type": "byte"
},
"pgrk": {
"type": "integer"
},
"url_length": {
"type": "integer"
},
"term": {
"type": "text",
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
}
},
"titulo": {
"analyzer": "analyzer_customizado",
"type": "text",
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
}
},
"descricao": {
"analyzer": "analyzer_customizado",
"type": "text",
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
}
},
"url": {
"analyzer": "analyzer_customizado",
"type": "text",
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
}
}
}
}
}
I'm just trying to sync my mongodb with ElasticSearch. I've done the sync with river, and river already worked. But the ES won't show all fields, it only show "_ts" field in the object of "_source":
Request:
GET localhost:9200/test/orders/_search
Response:
{
"took": 11,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"failed": 0
},
"hits": {
"total": 137,
"max_score": 1,
"hits": [
{
"_index": "test",
"_type": "orders",
"_id": "58a3251f761f35a107724add",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
},
{
"_index": "test",
"_type": "orders",
"_id": "58a340467f39c50f3a54c614",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
},
{
"_index": "test",
"_type": "orders",
"_id": "58b8ec806f34179d7c7b2431",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
},
{
"_index": "test",
"_type": "orders",
"_id": "58b8eff56f3417670f7b244a",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
},
{
"_index": "test",
"_type": "orders",
"_id": "58b8f0af6f3417fb207b244c",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
},
{
"_index": "test",
"_type": "orders",
"_id": "58b8f19a6f341761337b23da",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
},
{
"_index": "test",
"_type": "orders",
"_id": "58b9320c6f3417bc1c7b23c7",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
},
{
"_index": "test",
"_type": "orders",
"_id": "58b9339f6f341777237b23c6",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
},
{
"_index": "test",
"_type": "orders",
"_id": "58b934ab6f341778237b23c7",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
},
{
"_index": "test",
"_type": "orders",
"_id": "58b96ef76f34174a4b7b23c8",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
}
]
}
}
I already did the remapping, and It shows the updated mapping. See this:
request:
GET localhost:9200/test/orders/_mapping
response:
{
"test": {
"mappings": {
"orders": {
"properties": {
"_ts": {
"type": "long"
},
"activeDate": {
"type": "text"
},
"awbNumber": {
"type": "text"
},
"batchID": {
"type": "text"
},
"consignee": {
"properties": {
"id": {
"type": "text"
},
"name": {
"type": "text"
},
"phoneNumber": {
"type": "text"
}
}
},
"consigner": {
"properties": {
"id": {
"type": "text"
},
"name": {
"type": "text"
},
"phoneNumber": {
"type": "text"
}
}
},
"courier": {
"properties": {
"actualRate": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"id": {
"type": "integer"
},
"max_day": {
"type": "integer"
},
"min_day": {
"type": "integer"
},
"name": {
"type": "text"
},
"rate": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"rate_id": {
"type": "integer"
},
"rate_name": {
"type": "text"
},
"shipmentType": {
"type": "integer"
}
}
},
"creationDate": {
"type": "text"
},
"destination": {
"properties": {
"address": {
"type": "text"
},
"cityID": {
"type": "integer"
},
"cityName": {
"type": "text"
},
"id": {
"type": "integer"
},
"provinceID": {
"type": "integer"
},
"provinceName": {
"type": "text"
}
}
},
"driver": {
"properties": {
"feedback": {
"properties": {
"comment": {
"type": "text"
},
"score": {
"type": "long"
}
}
},
"id": {
"type": "long"
},
"isPaymentCollected": {
"type": "integer"
},
"name": {
"type": "text"
},
"phoneNumber": {
"type": "text"
},
"vehicleNumber": {
"type": "text"
},
"vehicleType": {
"type": "text"
}
}
},
"externalID": {
"type": "text"
},
"groupID": {
"type": "integer"
},
"id": {
"type": "text"
},
"isActive": {
"type": "long"
},
"isAutoTrack": {
"type": "integer"
},
"isCustomAWB": {
"type": "integer"
},
"isEscrow": {
"type": "integer"
},
"isLabelPrinted": {
"type": "integer"
},
"lastUpdatedDate": {
"type": "text"
},
"origin": {
"properties": {
"address": {
"type": "text"
},
"cityID": {
"type": "integer"
},
"cityName": {
"type": "text"
},
"id": {
"type": "integer"
},
"provinceID": {
"type": "integer"
},
"provinceName": {
"type": "text"
}
}
},
"package": {
"properties": {
"content": {
"type": "text"
},
"contents": {
"type": "integer"
},
"cubicalWeight": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"dimension": {
"properties": {
"height": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"length": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"width": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
}
}
},
"fragile": {
"type": "integer"
},
"isConfirmed": {
"type": "integer"
},
"itemName": {
"type": "text"
},
"itemSubtype": {
"type": "integer"
},
"itemType": {
"type": "integer"
},
"pictureURL": {
"type": "text"
},
"price": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"type": {
"type": "integer"
},
"weight": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
}
}
},
"paymentType": {
"type": "text"
},
"pickUpTime": {
"type": "text"
},
"rates": {
"properties": {
"actualInsurance": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "long"
}
}
},
"actualShipment": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"escrowCost": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"fulfillmentCost": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"insurance": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "long"
}
}
},
"itemPrice": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"liability": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"shipment": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
}
}
},
"readyTime": {
"type": "text"
},
"shipmentStatus": {
"properties": {
"description": {
"type": "text"
},
"name": {
"type": "text"
},
"statusCode": {
"type": "integer"
},
"updateDate": {
"type": "text"
},
"updatedBy": {
"type": "text"
}
}
},
"source": {
"type": "text"
},
"specialID": {
"type": "text"
},
"stickerNumber": {
"type": "text"
},
"useInsurance": {
"type": "integer"
}
}
}
}
}
}
I'm expecting I can get the whole fields (not just a single "_ts" field) in the "_source" field which it should be the same with what I've mapped.
I've tried to delete the index and recreate one, still not worked. Any clue of this kind of issue. I really really need help, thank you so much.
ElasticSearch has a behavior of creating fields on its own. So if you do
PUT document/index/1
{
"id" : "1",
"name" : "kashish",
"phoneNumber" : "9740683281"
}
This will automatically create fields for your index. What I am illustrating is if explicitly defining of fields is not working for you for some reason , you can empty the index(if contains dummy data) and then just put your json which ES will pickup automatically.
I have mapping as follow
{
"dsi2": {
"mappings": {
"dsi2": {
"_all": {
"enabled": true
},
"properties": {
"instance": {
"properties": {
"assignee": {
"type": "string"
},
"createdTs": {
"type": "long"
},
"dataSourceId": {
"type": "integer"
},
"dsTypeId": {
"type": "integer"
},
"entitlements": {
"properties": {
"create": {
"type": "boolean"
},
"delete": {
"type": "boolean"
},
"edit": {
"type": "boolean"
},
"read": {
"type": "boolean"
},
"roleId": {
"type": "integer"
}
},
"type": "nested"
},
"formDefinitionId": {
"type": "long"
},
"formTypeId": {
"type": "long"
},
"id": {
"fields": {
"raw": {
"type": "integer"
}
},
"type": "integer"
},
"instFields": {
"properties": {
"fieldBoolean": {
"type": "boolean"
},
"fieldDate": {
"format": "strict_date_optional_time||epoch_millis",
"type": "date"
},
"fieldDouble": {
"fields": {
"raw": {
"type": "double"
}
},
"type": "double"
},
"fieldLong": {
"fields": {
"raw": {
"type": "long"
}
},
"type": "long"
},
"fieldString": {
"fields": {
"raw": {
"index": "not_analyzed",
"type": "string"
}
},
"type": "string"
},
"fieldValue": {
"fields": {
"raw": {
"index": "not_analyzed",
"type": "string"
}
},
"type": "string"
},
"isKey": {
"type": "boolean"
},
"referredInstKeyData": {
"properties": {
"fieldValue": {
"fields": {
"raw": {
"index": "not_analyzed",
"type": "string"
}
},
"type": "string"
},
"sortOrder": {
"type": "integer"
},
"sourceFieldId": {
"type": "integer"
}
},
"type": "nested"
},
"sortOrder": {
"type": "long"
},
"sourceFieldId": {
"type": "integer"
}
},
"type": "nested"
},
"instanceId": {
"type": "integer"
},
"modifiedTs": {
"type": "long"
},
"parentEntityId": {
"type": "integer"
},
"processId": {
"type": "string"
},
"processName": {
"type": "string"
},
"relatedInstances": {
"properties": {
"createdTs": {
"type": "long"
},
"dataSourceId": {
"type": "integer"
},
"dsTypeId": {
"type": "integer"
},
"entitlements": {
"properties": {
"create": {
"type": "boolean"
},
"delete": {
"type": "boolean"
},
"edit": {
"type": "boolean"
},
"read": {
"type": "boolean"
},
"roleId": {
"type": "long"
}
}
},
"formDefinitionId": {
"type": "long"
},
"formTypeId": {
"type": "long"
},
"id": {
"type": "integer"
},
"instFields": {
"properties": {
"fieldBoolean": {
"type": "boolean"
},
"fieldDate": {
"format": "strict_date_optional_time||epoch_millis",
"type": "date"
},
"fieldDouble": {
"fields": {
"raw": {
"type": "double"
}
},
"type": "double"
},
"fieldLong": {
"fields": {
"raw": {
"type": "long"
}
},
"type": "long"
},
"fieldString": {
"fields": {
"raw": {
"index": "not_analyzed",
"type": "string"
}
},
"type": "string"
},
"fieldValue": {
"fields": {
"raw": {
"index": "not_analyzed",
"type": "string"
}
},
"type": "string"
},
"isKey": {
"type": "boolean"
},
"referredInstKeyData": {
"properties": {
"fieldString": {
"fields": {
"raw": {
"index": "not_analyzed",
"type": "string"
}
},
"type": "string"
},
"fieldValue": {
"fields": {
"raw": {
"index": "not_analyzed",
"type": "string"
}
},
"type": "string"
},
"sortOrder": {
"type": "integer"
},
"sourceFieldId": {
"type": "integer"
}
},
"type": "nested"
},
"sortOrder": {
"type": "long"
},
"sourceFieldId": {
"type": "integer"
}
},
"type": "nested"
},
"instanceId": {
"type": "integer"
},
"modifiedTs": {
"type": "long"
},
"parentEntityId": {
"type": "integer"
},
"status": {
"type": "long"
},
"tenantId": {
"type": "long"
}
},
"type": "nested"
},
"status": {
"type": "long"
},
"taskAssignees": {
"properties": {
"role": {
"type": "integer"
},
"user": {
"type": "integer"
}
},
"type": "nested"
},
"taskCompleted": {
"format": "strict_date_optional_time||epoch_millis",
"type": "date"
},
"taskCompletedBy": {
"type": "string"
},
"taskCreated": {
"format": "strict_date_optional_time||epoch_millis",
"type": "date"
},
"taskName": {
"type": "string"
},
"tenantId": {
"type": "long"
}
}
},
"status": {
"type": "integer"
},
"tenantId": {
"type": "integer"
}
}
}
}
}
}
when using following query
{
"query": {
"nested": {
"path": "instance.instFields",
"query": {
"bool": {
"must": [
{
"term": {
"instance.instFields.fieldValue": "120656"
}
},
{
"exists": {
"field": "instance.instFields.referredInstKeyData"
}
}
]
}
}
}
}
}
one of the example documents in the index is as follow
{
"took": 1,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"failed": 0
},
"hits": {
"total": 1,
"max_score": 10.130359,
"hits": [
{
"_shard": 0,
"_node": "_FohF7-YRyCSh82NmYVslQ",
"_index": "dsi2",
"_type": "dsi2",
"_id": "120655",
"_score": 10.130359,
"_source": {
"instance": {
"entitlements": [
{
"read": true,
"edit": false,
"roleId": 169,
"create": false,
"delete": false
}
],
"formTypeId": 3,
"parentEntityId": 23589,
"createdTs": 1487701846526,
"dataSourceId": 22,
"dsTypeId": 5,
"modifiedTs": 1487783663308,
"instanceId": 17790,
"instFields": [
{
"fieldDouble": 120658,
"sourceFieldId": 1594,
"sortOrder": 0,
"referredInstKeyData": [
{
"sourceFieldId": 1233,
"sortOrder": 0,
"fieldValue": "Depot Transport"
},
{
"sourceFieldId": 1232,
"sortOrder": 1,
"fieldValue": "LONDON"
},
{
"sourceFieldId": 1236,
"sortOrder": 4,
"fieldValue": "513069"
},
{
"sourceFieldId": 1658,
"sortOrder": 5,
"fieldValue": "7378"
}
],
"isKey": false,
"fieldValue": "120658",
"fieldString": "120658"
},
{
"fieldDouble": 120656,
"sourceFieldId": 1593,
"sortOrder": 0,
"referredInstKeyData": [
{
"sourceFieldId": 1233,
"sortOrder": 0,
"fieldValue": "Joe Cadillac Ii Ltd"
},
{
"sourceFieldId": 1236,
"sortOrder": 4,
"fieldValue": "518508"
},
{
"sourceFieldId": 1658,
"sortOrder": 5,
"fieldValue": "7376"
}
],
"isKey": false,
"fieldValue": "120656",
"fieldString": "120656"
},
{
"sourceFieldId": 2580,
"sortOrder": 0,
"isKey": false,
"fieldValue": "52fe850d-fc17-4dd9-a3a2-257485a9eb27",
"fieldString": "52fe850d-fc17-4dd9-a3a2-257485a9eb27"
}
],
"processName": "Vehicle Delivery",
"tenantId": 3,
"formDefinitionId": 729,
"id": 120655,
"assignee": "none",
"status": 4
},
"formTypeId": 3,
"tenantId": 3,
"status": 4
}
}
]
}
}
it's not returning documents where fieldValue": "120656" and referredInstKeyData element is in there. any idea what could be causing exists to not work? Elastic search version is 2.3.1
this is a known issue in Elastic search. exists isn't supported for nested properties. confirmed here. https://github.com/elastic/elasticsearch/issues/23325
I'm try do a query like this in Elastic Search:
Return me all the devices of an app that had some logs between two dates and for each device return me the total number of logs
For this I've a parent-child relationship. I've the parent device type that has the device information and then a child entity device_logs that has the number of logs for each day.
I tried to run the following query with a custom score function. I do get the right devices, but the score has the sum of all the device_logs entries instead of the entries in the dates range.
Any idea if it's possible to do this kind of query?
{
"query": {
"bool": {
"filter" :
[
{
"term": {"app": 347}
}
],
"must" :
[
{
"has_child": {
"type": "device_logs",
"inner_hits" : {},
"query": {
"bool": {
"filter": {
"range": {
"date": {
"from": "2017-01-15T00:00:00Z",
"include_lower": true,
"include_upper": true,
"to": "2017-01-17T23:59:59Z"
}
}
}
}
}
}
},
{
"has_child": {
"type": "device_logs",
"score_mode": "sum",
"query" : {
"function_score" : {
"script_score": {
"script": "_score * doc['logs'].value"
}
}
}
}
}
]
}
}
}
EDIT: Adding mappings and some docs
Here you have the mappings:
"mappings": {
"device": {
"properties": {
"app": {
"type": "long",
"include_in_all": false
},
"created_at": {
"type": "date",
"include_in_all": false
},
"id": {
"type": "long",
"include_in_all": false
},
"language": {
"type": "keyword",
"include_in_all": false,
"ignore_above": 256
},
"last_log_at": {
"type": "date",
"include_in_all": false
},
"last_ping_at": {
"type": "date",
"include_in_all": false
},
"last_seen_at": {
"type": "date"
},
"log_enabled": {
"type": "boolean"
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
},
"lowercase": {
"type": "text",
"analyzer": "case_insensitive_sort"
}
}
},
"os_version": {
"type": "keyword",
"include_in_all": false,
"ignore_above": 256
},
"timezone": {
"type": "keyword",
"include_in_all": false,
"ignore_above": 256
},
"type": {
"type": "keyword",
"ignore_above": 256
},
"udid": {
"type": "keyword",
"ignore_above": 256
},
"version": {
"properties": {
"build": {
"type": "keyword",
"include_in_all": false,
"ignore_above": 256
},
"id": {
"type": "long",
"include_in_all": false
},
"version": {
"type": "keyword",
"include_in_all": false,
"ignore_above": 256
}
}
}
}
},
"device_logs": {
"_parent": {
"type": "device"
},
"_routing": {
"required": true
},
"properties": {
"_": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"app": {
"type": "long",
"include_in_all": false
},
"date": {
"type": "date",
"include_in_all": false
},
"errors": {
"type": "long",
"include_in_all": false
},
"logs": {
"type": "long",
"include_in_all": false
},
"warnings": {
"type": "long",
"include_in_all": false
}
}
}
}
And some documents:
{
"_index": "devices",
"_type": "device_logs",
"_id": "22466_2017.01.17",
"_score": 1,
"_routing": "22466",
"_parent": "22466",
"_source": {
"_": "22466_2017.01.17",
"app": 200,
"date": "2017-01-17T00:00:00Z",
"logs": 660,
"warnings": 238,
"errors": 217
}
}
{
"_index": "devices",
"_type": "device",
"_id": "22466",
"_score": 1,
"_source": {
"id": 22466,
"udid": "770CA14ED7FE861EC452",
"name": "Edward's iPhone",
"type": "iPhone7,2",
"app": 200,
"log_enabled": false,
"created_at": "2016-12-21T10:55:02Z",
"last_seen_at": "2017-01-19T10:07:33Z",
"last_log_at": "2017-01-19T11:07:40.756275026+01:00",
"language": "en-US",
"os_version": "9.2",
"timezone": "GMT+1",
"version.id": 7305,
"version.version": "1",
"version.build": "100"
}
}
I have solved your query.
From the first look at the query, I was doubtful that you are not filtering the child documents in one of the must filters before applying the function score on the child document.
I have used the following set of documents for this query
parent doc
{
"id": 22466,
"udid": "770CA14ED7FE861EC452",
"name": "Edward's iPhone",
"type": "iPhone7,2",
"app": 347,
"log_enabled": false,
"created_at": "2016-12-21T10:55:02Z",
"last_seen_at": "2017-01-19T10:07:33Z",
"last_log_at": "2017-01-19T11:07:40.756275026+01:00",
"language": "en-US",
"os_version": "9.2",
"timezone": "GMT+1",
"version.id": 7305,
"version.version": "1",
"version.build": "100"
}
child docs
{
"_type": "device_logs",
"_id": "22466_2017.01.17",
"_score": 0,
"_routing": "22466",
"_parent": "22466",
"_source": {
"_": "22466_2017.01.17",
"app": 200,
"date": "2017-01-17T00:00:00Z",
"logs": 660,
"warnings": 238,
"errors": 217
}
},
{
"_type": "device_logs",
"_id": "22466_2017.02.17",
"_score": 0,
"_routing": "22466",
"_parent": "22466",
"_source": {
"_": "22466_2017.02.17",
"app": 200,
"date": "2017-01-17T00:00:00Z",
"logs": 200,
"warnings": 238,
"errors": 217
}
},
{
"_type": "device_logs",
"_id": "22466_2017.02.20",
"_score": 0,
"_routing": "22466",
"_parent": "22466",
"_source": {
"_": "22466_2017.02.20",
"app": 200,
"date": "2017-01-20T00:00:00Z",
"logs": 200,
"warnings": 238,
"errors": 217
}
}
Note - The first must filter only filter the documents for innerhits.
Please use the following query:
{
"query": {
"bool": {
"filter": [{
"term": {
"app": 347
}
}],
"must": [{
"has_child": {
"type": "device_logs",
"inner_hits": {},
"query": {
"bool": {
"filter": {
"range": {
"date": {
"from": "2017-01-15T00:00:00Z",
"include_lower": true,
"include_upper": true,
"to": "2017-01-17T23:59:59Z"
}
}
}
}
}
}
}, {
"has_child": {
"type": "device_logs",
"score_mode": "sum",
"query": {
"function_score": {
"query": {
"bool": {
"filter": {
"range": {
"date": {
"from": "2017-01-15T00:00:00Z",
"include_lower": true,
"include_upper": true,
"to": "2017-01-17T23:59:59Z"
}
}
}
}
},
"score_mode": "sum",
"boost_mode": "sum",
"script_score": {
"script": "_score + doc['logs'].value"
}
}
}
}
}]
}
}
}
Few references https://github.com/elastic/elasticsearch/issues/10051
Following is the response I get with explain bool set to true
{
"took": 7,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"failed": 0
},
"hits": {
"total": 1,
"max_score": 861,
"hits": [
{
"_shard": "[array_index1][0]",
"_node": "nnauJDrIS8-QCqicOMF23g",
"_index": "array_index1",
"_type": "device",
"_id": "22466",
"_score": 861,
"_source": {
"id": 22466,
"udid": "770CA14ED7FE861EC452",
"name": "Edward's iPhone",
"type": "iPhone7,2",
"app": 347,
"log_enabled": false,
"created_at": "2016-12-21T10:55:02Z",
"last_seen_at": "2017-01-19T10:07:33Z",
"last_log_at": "2017-01-19T11:07:40.756275026+01:00",
"language": "en-US",
"os_version": "9.2",
"timezone": "GMT+1",
"version.id": 7305,
"version.version": "1",
"version.build": "100"
},
"_explanation": {
"value": 861,
"description": "sum of:",
"details": [
{
"value": 1,
"description": "A match, join value 22466",
"details": []
},
{
"value": 860,
"description": "A match, join value 22466",
"details": []
},
{
"value": 0,
"description": "match on required clause, product of:",
"details": [
{
"value": 0,
"description": "# clause",
"details": []
},
{
"value": 1,
"description": "app:[347 TO 347], product of:",
"details": [
{
"value": 1,
"description": "boost",
"details": []
},
{
"value": 1,
"description": "queryNorm",
"details": []
}
]
}
]
}
]
},
"inner_hits": {
"device_logs": {
"hits": {
"total": 2,
"max_score": 0,
"hits": [
{
"_type": "device_logs",
"_id": "22466_2017.01.17",
"_score": 0,
"_routing": "22466",
"_parent": "22466",
"_source": {
"_": "22466_2017.01.17",
"app": 200,
"date": "2017-01-17T00:00:00Z",
"logs": 660,
"warnings": 238,
"errors": 217
}
},
{
"_type": "device_logs",
"_id": "22466_2017.02.17",
"_score": 0,
"_routing": "22466",
"_parent": "22466",
"_source": {
"_": "22466_2017.02.17",
"app": 200,
"date": "2017-01-17T00:00:00Z",
"logs": 200,
"warnings": 238,
"errors": 217
}
}
]
}
}
}
}
]
}
}
Please verify your results.