Elastic search not return all fields - elasticsearch

I have an index which name is news, here is the stored mapping in the elasticsearch:
{
"news": {
"mappings": {
"newsdetail": {
"properties": {
"active": {
"type": "long"
},
"authorId": {
"type": "long"
},
"creationDate": {
"type": "date",
"format": "dateOptionalTime"
},
"iDate": {
"type": "string"
},
"iTime": {
"type": "string"
},
"isExclusive": {
"type": "boolean"
},
"itemId": {
"type": "long"
},
"languageId": {
"type": "long"
},
"lead": {
"type": "string"
},
"main": {
"type": "long"
},
"mainTitr": {
"type": "long"
},
"mediaType": {
"type": "long"
},
"newsType": {
"type": "long"
},
"pictureId": {
"type": "long"
},
"ref": {
"type": "string"
},
"subMain": {
"type": "long"
},
"subMainTitr": {
"type": "long"
},
"subtitle": {
"type": "string"
},
"text": {
"type": "string"
},
"title": {
"type": "string"
},
"video": {
"type": "string"
},
"viewCount": {
"type": "long"
}
}
}
}
}
}
when I try to search in this index's data it returns this:
{
"took": 1,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"failed": 0
},
"hits": {
"total": 69612,
"max_score": 1,
"hits": [
{
"_index": "news",
"_type": "newsdetail",
"_id": "AU6Gt1fK36ANCl26_meY",
"_score": 1,
"_source": {
"itemId": 452,
"pictureId": 452,
"authorId": 1,
"languageId": 1,
"active": 1,
"title": "خبر تستی",
"main": 1,
"mainTitr": 1,
"subMain": 1,
"subMainTitr": 1,
"video": "ویدئوی تستی",
"newsType": 3,
"creationDate": "2014-06-09T18:38:47.203",
"viewCount": 201,
"isExclusive": false,
"mediaType": 0
}
}
]
}
}
some of fields has missed. what should I do to see whole fields. note that I have not select fields and my request is GET news/_search?pretty=true&size=1

Related

Searching through objects inside nested documents provides unexpected output

I am getting unexpected result from Elasticsearch while searching though object property which are inside nested property. I am using elasticsearch-dsl python library for creating document and querying. Is this bug or I am missing something in querying and mapping parts? Below are the elasticsearch json mapping, query, unexpected result and expected result
Mapping:
{
"deal_acls": {
"type": "nested",
"properties": {
"created_at": {
"type": "date"
},
"created_by": {
"properties": {
"id": {
"type": "keyword"
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"status": {
"type": "keyword",
"normalizer": "lowercase"
}
}
},
"permission": {
"properties": {
"CRM": {
"properties": {
"description": {
"properties": {
"created_by": {
"type": "keyword",
"normalizer": "lowercase"
},
"object": {
"type": "keyword",
"normalizer": "lowercase"
},
"object_id": {
"type": "keyword",
"normalizer": "lowercase"
},
"timestamp": {
"type": "date"
}
}
},
"permission": {
"properties": {
"delete": {
"type": "integer"
},
"edit": {
"type": "integer"
},
"manage": {
"type": "long"
},
"read": {
"type": "integer"
},
"write": {
"type": "integer"
}
}
}
}
},
"deal": {
"properties": {
"description": {
"properties": {
"created_by": {
"type": "keyword",
"normalizer": "lowercase"
},
"object": {
"type": "keyword",
"normalizer": "lowercase"
},
"object_id": {
"type": "keyword",
"normalizer": "lowercase"
},
"timestamp": {
"type": "date"
}
}
},
"permission": {
"properties": {
"delete": {
"type": "integer"
},
"edit": {
"type": "integer"
},
"manage": {
"type": "long"
},
"read": {
"type": "integer"
},
"write": {
"type": "integer"
}
}
}
}
},
"document": {
"properties": {
"description": {
"properties": {
"created_by": {
"type": "keyword",
"normalizer": "lowercase"
},
"object": {
"type": "keyword",
"normalizer": "lowercase"
},
"object_id": {
"type": "keyword",
"normalizer": "lowercase"
},
"timestamp": {
"type": "date"
}
}
},
"permission": {
"properties": {
"delete": {
"type": "integer"
},
"edit": {
"type": "integer"
},
"manage": {
"type": "long"
},
"read": {
"type": "integer"
},
"write": {
"type": "integer"
}
}
}
}
},
"external_deal_team": {
"properties": {
"description": {
"properties": {
"created_by": {
"type": "keyword",
"normalizer": "lowercase"
},
"object": {
"type": "keyword",
"normalizer": "lowercase"
},
"object_id": {
"type": "keyword",
"normalizer": "lowercase"
},
"timestamp": {
"type": "date"
}
}
},
"permission": {
"properties": {
"delete": {
"type": "integer"
},
"edit": {
"type": "integer"
},
"manage": {
"type": "long"
},
"read": {
"type": "integer"
},
"write": {
"type": "integer"
}
}
}
}
},
"internal_deal_team": {
"properties": {
"description": {
"properties": {
"created_by": {
"type": "keyword",
"normalizer": "lowercase"
},
"object": {
"type": "keyword",
"normalizer": "lowercase"
},
"object_id": {
"type": "keyword",
"normalizer": "lowercase"
},
"timestamp": {
"type": "date"
}
}
},
"permission": {
"properties": {
"delete": {
"type": "integer"
},
"edit": {
"type": "integer"
},
"manage": {
"type": "long"
},
"read": {
"type": "integer"
},
"write": {
"type": "integer"
}
}
}
}
}
}
},
"status": {
"type": "keyword",
"normalizer": "lowercase"
},
"updated_at": {
"type": "date"
},
"updated_by": {
"properties": {
"id": {
"type": "keyword"
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"status": {
"type": "keyword",
"normalizer": "lowercase"
}
}
},
"user": {
"properties": {
"id": {
"type": "keyword"
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"status": {
"type": "keyword",
"normalizer": "lowercase"
}
}
}
}
}
}
Query:
{
"query": {
"bool": {
"must": [
{
"nested": {
"path": "deal_acls",
"query": {
"term": {
"deal_acls.permission.deal.permission.read": 1
}
}
}
},
{
"nested": {
"path": "deal_acls",
"query": {
"terms": {
"deal_acls.user.id": [
"5f7cea05-6562-4bdd-8448-19cfbe11783a"
]
}
}
}
}
]
}
}
}
Unexpected result: Since the deal permission of user with id=5f7cea05-6562-4bdd-8448-19cfbe11783a is 0, it should be returning empty hits.
{
"took": 2,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 1,
"relation": "eq"
},
"max_score": 2,
"hits": [
{
"_index": "dev.crecentric.dealvault.deals",
"_type": "_doc",
"_id": "a928838d-3422-41db-b40e-28f5c793f806",
"_score": 2,
"_source": {
"id": "a928838d-3422-41db-b40e-28f5c793f806",
"deal_acls": [
{
"user": {
"id": "5f7cea05-6562-4bdd-8448-19cfbe11783a",
"name": "testerrrrs testesssss",
"status": "active"
},
"permission": {
"deal": {
"permission": {
"edit": 0,
"read": 0,
"write": 0,
"delete": 0,
"manage": 0
},
"description": {
"object": "workspace",
"object_id": "fbc840b1-8727-4945-a070-fa1c105f9550",
"timestamp": "2022-02-10T05:46:05.140867+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
},
"external_deal_team": {
"permission": {
"edit": 0,
"read": 1,
"write": 1,
"delete": 0,
"manage": 0
},
"description": {
"object": "workspace",
"object_id": "fbc840b1-8727-4945-a070-fa1c105f9550",
"timestamp": "2022-02-10T05:46:05.140902+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
},
"internal_deal_team": {
"permission": {
"edit": 0,
"read": 1,
"write": 1,
"delete": 0,
"manage": 0
},
"description": {
"object": "workspace",
"object_id": "fbc840b1-8727-4945-a070-fa1c105f9550",
"timestamp": "2022-02-10T05:46:05.140910+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
}
},
"status": "active",
"created_at": "2022-02-10T05:40:15.727598+05:45",
"updated_at": "2022-02-10T05:46:05.177076+05:45"
},
{
"user": {
"id": "d78411e5-6645-4b95-a98c-db6db8748580",
"name": "Ramesh Pradhan",
"status": "active"
},
"permission": {
"CRM": {
"permission": {
"edit": 1,
"read": 1,
"write": 1,
"delete": 1,
"manage": 1
},
"description": {
"object": "owner",
"object_id": "d78411e5-6645-4b95-a98c-db6db8748580",
"timestamp": "2022-02-10T05:35:41.453881+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
},
"deal": {
"permission": {
"edit": 1,
"read": 1,
"write": 1,
"delete": 1,
"manage": 1
},
"description": {
"object": "owner",
"object_id": "d78411e5-6645-4b95-a98c-db6db8748580",
"timestamp": "2022-02-10T05:35:41.453881+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
},
"document": {
"permission": {
"edit": 1,
"read": 1,
"write": 1,
"delete": 1,
"manage": 1
},
"description": {
"object": "owner",
"object_id": "d78411e5-6645-4b95-a98c-db6db8748580",
"timestamp": "2022-02-10T05:35:41.453881+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
},
"external_deal_team": {
"permission": {
"edit": 1,
"read": 1,
"write": 1,
"delete": 1,
"manage": 1
},
"description": {
"object": "owner",
"object_id": "d78411e5-6645-4b95-a98c-db6db8748580",
"timestamp": "2022-02-10T05:35:41.453881+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
},
"internal_deal_team": {
"permission": {
"edit": 1,
"read": 1,
"write": 1,
"delete": 1,
"manage": 1
},
"description": {
"object": "owner",
"object_id": "d78411e5-6645-4b95-a98c-db6db8748580",
"timestamp": "2022-02-10T05:35:41.453881+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
}
},
"status": "active",
"created_at": "2022-02-10T05:35:41.453913+05:45",
"updated_at": "2022-02-10T05:35:41.462956+05:45"
}
]
}
}
]
}
}
Expected result:
{
"took": 2,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 1,
"relation": "eq"
},
"max_score": 2,
"hits": []
}
}
You are using two nested queries: so the result is a document which contains "deal_acls.permission.deal.permission.read": 1 and "deal_acls.user.id": ["5f7cea05-6562-4bdd-8448-19cfbe11783a"], but not necessary in the same subobject of the nested field. Try using a terms query inside a single nested query:
{
"query": {
"nested": {
"path": "deal_acls",
"query": {
"bool": {
"must": [
{ "match": { "deal_acls.user.id": "5f7cea05-6562-4bdd-8448-19cfbe11783a" }},
{ "match": { "deal_acls.permission.deal.permission.read": 1 }}
]
}
}
}
}
}
In case if someone is searching for answer, I have fixed this using this query:
{
"query": {
"nested": {
"path": "deal_acls",
"query": {
"bool": {
"must": [
{
"term": {
"deal_acls.permission.deal.permission.read": 1
}
},
{
"terms": {
"deal_acls.user.id": [
"5f7cea05-6562-4bdd-8448-19cfbe11783a"
]
}
}
]
}
}
}
}
}

how do I get elasticsearch to always return the smallest field value ("url length")?

How do I always return the documents with the lowest value in the "url_length" field regardless of (from) that I sent to search?
in the query below, I request the results that have the word (netflix) and that the field (pgrk) is between 9 and 10 and that the field (url_length) is less than 4, but when I put it ("from": 1, "size ": 1) does not return the doc of (_id 15) that has the word (netflix) the field pgrk = 10 and the field (url_length) = 2. Returns the doc of (_id 14) that has the word (netflix) the field pgrk = 10 and the field (url_length) = 3
just return the doc of (_id 15) that has the field (url_length) = 2 if I put it in the query from ZERO ("from": 0, "size": 1)
because I had it searched ("from": 1, "size": 1,) and didn't bring the record of (_id 15) that has the ("url_length" = 2) brought the record of (_id 14) that has the ("url_length" = 3)
{
"from": 1,
"size": 1,
"sort": [
{
"pgrk": {
"order": "desc"
}
},
{
"url_length": {
"order": "asc"
}
}
],
"query": {
"bool": {
"must": {
"multi_match": {
"query": "netflix",
"type": "cross_fields",
"fields": [
"tittle",
"description",
"url"
],
"operator": "and"
}
},
"filter": [
{
"range": {
"pgrk": {
"gte": 9,
"lte" : 10
}
}
},
{
"range": {
"url_length": {
"lt" : 4
}
}
}
]
}
}
}
if I put ("from": 1, "size": 1,) it does not return the record (_id 15) that has "url_length = 2" returns the doc of _id 14 that has "url_length = 3" as shown below:
{
"took": 0,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 2,
"relation": "eq"
},
"max_score": null,
"hits": [
{
"_index": "teste",
"_type": "_doc",
"_id": "14",
"_score": null,
"_source": {
"url": "www.333.com",
"title": "netflix netflix netflix netflix netflix netflix netflix netflix netflix netflix",
"description": "tudo sobre netflix netflix netflix netflix netflix netflix",
"pgrk": "10",
"url_length": "3"
},
"sort": [
10,
3
]
}
]
}
}
if I put ("from": 0, "size": 1,) then it returns the record (_id 15) that has "url_length = 2"
{
"took": 0,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 2,
"relation": "eq"
},
"max_score": null,
"hits": [
{
"_index": "teste",
"_type": "_doc",
"_id": "15",
"_score": null,
"_source": {
"url": "www.netflix.yahoo.com",
"title": "melhor filme",
"description": "tudo sobre series",
"pgrk": "10",
"url_length": "2"
},
"sort": [
10,
2
]
}
]
}
}
how do I always return the documents with the lowest value in the "url_length" field regardless of (from) that I sent to search?
follows my mapping:
{
"settings": {
"index": {
"number_of_shards": "5",
"number_of_replicas": "0",
"analysis": {
"filter": {
"stemmer_plural_portugues": {
"name": "minimal_portuguese",
"stopwords" : ["http", "https", "ftp", "www"],
"type": "stemmer"
}
},
"analyzer": {
"analyzer_customizado": {
"filter": [
"lowercase",
"stemmer_plural_portugues",
"asciifolding"
],
"tokenizer": "lowercase"
}
}
}
}
},
"mappings": {
"properties": {
"q": {
"type": "text",
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
}
},
"id": {
"type": "long"
},
"#timestamp": {
"type": "date"
},
"data": {
"type": "date"
},
"#version": {
"type": "text",
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
}
},
"quebrado": {
"type": "byte"
},
"pgrk": {
"type": "integer"
},
"url_length": {
"type": "integer"
},
"term": {
"type": "text",
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
}
},
"titulo": {
"analyzer": "analyzer_customizado",
"type": "text",
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
}
},
"descricao": {
"analyzer": "analyzer_customizado",
"type": "text",
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
}
},
"url": {
"analyzer": "analyzer_customizado",
"type": "text",
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
}
}
}
}
}

Elasticsearch: Already re-mapping but It still wont show all fields

I'm just trying to sync my mongodb with ElasticSearch. I've done the sync with river, and river already worked. But the ES won't show all fields, it only show "_ts" field in the object of "_source":
Request:
GET localhost:9200/test/orders/_search
Response:
{
"took": 11,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"failed": 0
},
"hits": {
"total": 137,
"max_score": 1,
"hits": [
{
"_index": "test",
"_type": "orders",
"_id": "58a3251f761f35a107724add",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
},
{
"_index": "test",
"_type": "orders",
"_id": "58a340467f39c50f3a54c614",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
},
{
"_index": "test",
"_type": "orders",
"_id": "58b8ec806f34179d7c7b2431",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
},
{
"_index": "test",
"_type": "orders",
"_id": "58b8eff56f3417670f7b244a",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
},
{
"_index": "test",
"_type": "orders",
"_id": "58b8f0af6f3417fb207b244c",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
},
{
"_index": "test",
"_type": "orders",
"_id": "58b8f19a6f341761337b23da",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
},
{
"_index": "test",
"_type": "orders",
"_id": "58b9320c6f3417bc1c7b23c7",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
},
{
"_index": "test",
"_type": "orders",
"_id": "58b9339f6f341777237b23c6",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
},
{
"_index": "test",
"_type": "orders",
"_id": "58b934ab6f341778237b23c7",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
},
{
"_index": "test",
"_type": "orders",
"_id": "58b96ef76f34174a4b7b23c8",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
}
]
}
}
I already did the remapping, and It shows the updated mapping. See this:
request:
GET localhost:9200/test/orders/_mapping
response:
{
"test": {
"mappings": {
"orders": {
"properties": {
"_ts": {
"type": "long"
},
"activeDate": {
"type": "text"
},
"awbNumber": {
"type": "text"
},
"batchID": {
"type": "text"
},
"consignee": {
"properties": {
"id": {
"type": "text"
},
"name": {
"type": "text"
},
"phoneNumber": {
"type": "text"
}
}
},
"consigner": {
"properties": {
"id": {
"type": "text"
},
"name": {
"type": "text"
},
"phoneNumber": {
"type": "text"
}
}
},
"courier": {
"properties": {
"actualRate": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"id": {
"type": "integer"
},
"max_day": {
"type": "integer"
},
"min_day": {
"type": "integer"
},
"name": {
"type": "text"
},
"rate": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"rate_id": {
"type": "integer"
},
"rate_name": {
"type": "text"
},
"shipmentType": {
"type": "integer"
}
}
},
"creationDate": {
"type": "text"
},
"destination": {
"properties": {
"address": {
"type": "text"
},
"cityID": {
"type": "integer"
},
"cityName": {
"type": "text"
},
"id": {
"type": "integer"
},
"provinceID": {
"type": "integer"
},
"provinceName": {
"type": "text"
}
}
},
"driver": {
"properties": {
"feedback": {
"properties": {
"comment": {
"type": "text"
},
"score": {
"type": "long"
}
}
},
"id": {
"type": "long"
},
"isPaymentCollected": {
"type": "integer"
},
"name": {
"type": "text"
},
"phoneNumber": {
"type": "text"
},
"vehicleNumber": {
"type": "text"
},
"vehicleType": {
"type": "text"
}
}
},
"externalID": {
"type": "text"
},
"groupID": {
"type": "integer"
},
"id": {
"type": "text"
},
"isActive": {
"type": "long"
},
"isAutoTrack": {
"type": "integer"
},
"isCustomAWB": {
"type": "integer"
},
"isEscrow": {
"type": "integer"
},
"isLabelPrinted": {
"type": "integer"
},
"lastUpdatedDate": {
"type": "text"
},
"origin": {
"properties": {
"address": {
"type": "text"
},
"cityID": {
"type": "integer"
},
"cityName": {
"type": "text"
},
"id": {
"type": "integer"
},
"provinceID": {
"type": "integer"
},
"provinceName": {
"type": "text"
}
}
},
"package": {
"properties": {
"content": {
"type": "text"
},
"contents": {
"type": "integer"
},
"cubicalWeight": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"dimension": {
"properties": {
"height": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"length": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"width": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
}
}
},
"fragile": {
"type": "integer"
},
"isConfirmed": {
"type": "integer"
},
"itemName": {
"type": "text"
},
"itemSubtype": {
"type": "integer"
},
"itemType": {
"type": "integer"
},
"pictureURL": {
"type": "text"
},
"price": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"type": {
"type": "integer"
},
"weight": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
}
}
},
"paymentType": {
"type": "text"
},
"pickUpTime": {
"type": "text"
},
"rates": {
"properties": {
"actualInsurance": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "long"
}
}
},
"actualShipment": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"escrowCost": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"fulfillmentCost": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"insurance": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "long"
}
}
},
"itemPrice": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"liability": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"shipment": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
}
}
},
"readyTime": {
"type": "text"
},
"shipmentStatus": {
"properties": {
"description": {
"type": "text"
},
"name": {
"type": "text"
},
"statusCode": {
"type": "integer"
},
"updateDate": {
"type": "text"
},
"updatedBy": {
"type": "text"
}
}
},
"source": {
"type": "text"
},
"specialID": {
"type": "text"
},
"stickerNumber": {
"type": "text"
},
"useInsurance": {
"type": "integer"
}
}
}
}
}
}
I'm expecting I can get the whole fields (not just a single "_ts" field) in the "_source" field which it should be the same with what I've mapped.
I've tried to delete the index and recreate one, still not worked. Any clue of this kind of issue. I really really need help, thank you so much.
ElasticSearch has a behavior of creating fields on its own. So if you do
PUT document/index/1
{
"id" : "1",
"name" : "kashish",
"phoneNumber" : "9740683281"
}
This will automatically create fields for your index. What I am illustrating is if explicitly defining of fields is not working for you for some reason , you can empty the index(if contains dummy data) and then just put your json which ES will pickup automatically.

exists isn't working on a two level nested structure in Elastic Search

I have mapping as follow
{
"dsi2": {
"mappings": {
"dsi2": {
"_all": {
"enabled": true
},
"properties": {
"instance": {
"properties": {
"assignee": {
"type": "string"
},
"createdTs": {
"type": "long"
},
"dataSourceId": {
"type": "integer"
},
"dsTypeId": {
"type": "integer"
},
"entitlements": {
"properties": {
"create": {
"type": "boolean"
},
"delete": {
"type": "boolean"
},
"edit": {
"type": "boolean"
},
"read": {
"type": "boolean"
},
"roleId": {
"type": "integer"
}
},
"type": "nested"
},
"formDefinitionId": {
"type": "long"
},
"formTypeId": {
"type": "long"
},
"id": {
"fields": {
"raw": {
"type": "integer"
}
},
"type": "integer"
},
"instFields": {
"properties": {
"fieldBoolean": {
"type": "boolean"
},
"fieldDate": {
"format": "strict_date_optional_time||epoch_millis",
"type": "date"
},
"fieldDouble": {
"fields": {
"raw": {
"type": "double"
}
},
"type": "double"
},
"fieldLong": {
"fields": {
"raw": {
"type": "long"
}
},
"type": "long"
},
"fieldString": {
"fields": {
"raw": {
"index": "not_analyzed",
"type": "string"
}
},
"type": "string"
},
"fieldValue": {
"fields": {
"raw": {
"index": "not_analyzed",
"type": "string"
}
},
"type": "string"
},
"isKey": {
"type": "boolean"
},
"referredInstKeyData": {
"properties": {
"fieldValue": {
"fields": {
"raw": {
"index": "not_analyzed",
"type": "string"
}
},
"type": "string"
},
"sortOrder": {
"type": "integer"
},
"sourceFieldId": {
"type": "integer"
}
},
"type": "nested"
},
"sortOrder": {
"type": "long"
},
"sourceFieldId": {
"type": "integer"
}
},
"type": "nested"
},
"instanceId": {
"type": "integer"
},
"modifiedTs": {
"type": "long"
},
"parentEntityId": {
"type": "integer"
},
"processId": {
"type": "string"
},
"processName": {
"type": "string"
},
"relatedInstances": {
"properties": {
"createdTs": {
"type": "long"
},
"dataSourceId": {
"type": "integer"
},
"dsTypeId": {
"type": "integer"
},
"entitlements": {
"properties": {
"create": {
"type": "boolean"
},
"delete": {
"type": "boolean"
},
"edit": {
"type": "boolean"
},
"read": {
"type": "boolean"
},
"roleId": {
"type": "long"
}
}
},
"formDefinitionId": {
"type": "long"
},
"formTypeId": {
"type": "long"
},
"id": {
"type": "integer"
},
"instFields": {
"properties": {
"fieldBoolean": {
"type": "boolean"
},
"fieldDate": {
"format": "strict_date_optional_time||epoch_millis",
"type": "date"
},
"fieldDouble": {
"fields": {
"raw": {
"type": "double"
}
},
"type": "double"
},
"fieldLong": {
"fields": {
"raw": {
"type": "long"
}
},
"type": "long"
},
"fieldString": {
"fields": {
"raw": {
"index": "not_analyzed",
"type": "string"
}
},
"type": "string"
},
"fieldValue": {
"fields": {
"raw": {
"index": "not_analyzed",
"type": "string"
}
},
"type": "string"
},
"isKey": {
"type": "boolean"
},
"referredInstKeyData": {
"properties": {
"fieldString": {
"fields": {
"raw": {
"index": "not_analyzed",
"type": "string"
}
},
"type": "string"
},
"fieldValue": {
"fields": {
"raw": {
"index": "not_analyzed",
"type": "string"
}
},
"type": "string"
},
"sortOrder": {
"type": "integer"
},
"sourceFieldId": {
"type": "integer"
}
},
"type": "nested"
},
"sortOrder": {
"type": "long"
},
"sourceFieldId": {
"type": "integer"
}
},
"type": "nested"
},
"instanceId": {
"type": "integer"
},
"modifiedTs": {
"type": "long"
},
"parentEntityId": {
"type": "integer"
},
"status": {
"type": "long"
},
"tenantId": {
"type": "long"
}
},
"type": "nested"
},
"status": {
"type": "long"
},
"taskAssignees": {
"properties": {
"role": {
"type": "integer"
},
"user": {
"type": "integer"
}
},
"type": "nested"
},
"taskCompleted": {
"format": "strict_date_optional_time||epoch_millis",
"type": "date"
},
"taskCompletedBy": {
"type": "string"
},
"taskCreated": {
"format": "strict_date_optional_time||epoch_millis",
"type": "date"
},
"taskName": {
"type": "string"
},
"tenantId": {
"type": "long"
}
}
},
"status": {
"type": "integer"
},
"tenantId": {
"type": "integer"
}
}
}
}
}
}
when using following query
{
"query": {
"nested": {
"path": "instance.instFields",
"query": {
"bool": {
"must": [
{
"term": {
"instance.instFields.fieldValue": "120656"
}
},
{
"exists": {
"field": "instance.instFields.referredInstKeyData"
}
}
]
}
}
}
}
}
one of the example documents in the index is as follow
{
"took": 1,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"failed": 0
},
"hits": {
"total": 1,
"max_score": 10.130359,
"hits": [
{
"_shard": 0,
"_node": "_FohF7-YRyCSh82NmYVslQ",
"_index": "dsi2",
"_type": "dsi2",
"_id": "120655",
"_score": 10.130359,
"_source": {
"instance": {
"entitlements": [
{
"read": true,
"edit": false,
"roleId": 169,
"create": false,
"delete": false
}
],
"formTypeId": 3,
"parentEntityId": 23589,
"createdTs": 1487701846526,
"dataSourceId": 22,
"dsTypeId": 5,
"modifiedTs": 1487783663308,
"instanceId": 17790,
"instFields": [
{
"fieldDouble": 120658,
"sourceFieldId": 1594,
"sortOrder": 0,
"referredInstKeyData": [
{
"sourceFieldId": 1233,
"sortOrder": 0,
"fieldValue": "Depot Transport"
},
{
"sourceFieldId": 1232,
"sortOrder": 1,
"fieldValue": "LONDON"
},
{
"sourceFieldId": 1236,
"sortOrder": 4,
"fieldValue": "513069"
},
{
"sourceFieldId": 1658,
"sortOrder": 5,
"fieldValue": "7378"
}
],
"isKey": false,
"fieldValue": "120658",
"fieldString": "120658"
},
{
"fieldDouble": 120656,
"sourceFieldId": 1593,
"sortOrder": 0,
"referredInstKeyData": [
{
"sourceFieldId": 1233,
"sortOrder": 0,
"fieldValue": "Joe Cadillac Ii Ltd"
},
{
"sourceFieldId": 1236,
"sortOrder": 4,
"fieldValue": "518508"
},
{
"sourceFieldId": 1658,
"sortOrder": 5,
"fieldValue": "7376"
}
],
"isKey": false,
"fieldValue": "120656",
"fieldString": "120656"
},
{
"sourceFieldId": 2580,
"sortOrder": 0,
"isKey": false,
"fieldValue": "52fe850d-fc17-4dd9-a3a2-257485a9eb27",
"fieldString": "52fe850d-fc17-4dd9-a3a2-257485a9eb27"
}
],
"processName": "Vehicle Delivery",
"tenantId": 3,
"formDefinitionId": 729,
"id": 120655,
"assignee": "none",
"status": 4
},
"formTypeId": 3,
"tenantId": 3,
"status": 4
}
}
]
}
}
it's not returning documents where fieldValue": "120656" and referredInstKeyData element is in there. any idea what could be causing exists to not work? Elastic search version is 2.3.1
this is a known issue in Elastic search. exists isn't supported for nested properties. confirmed here. https://github.com/elastic/elasticsearch/issues/23325

Elastic Search sum of child items

I'm try do a query like this in Elastic Search:
Return me all the devices of an app that had some logs between two dates and for each device return me the total number of logs
For this I've a parent-child relationship. I've the parent device type that has the device information and then a child entity device_logs that has the number of logs for each day.
I tried to run the following query with a custom score function. I do get the right devices, but the score has the sum of all the device_logs entries instead of the entries in the dates range.
Any idea if it's possible to do this kind of query?
{
"query": {
"bool": {
"filter" :
[
{
"term": {"app": 347}
}
],
"must" :
[
{
"has_child": {
"type": "device_logs",
"inner_hits" : {},
"query": {
"bool": {
"filter": {
"range": {
"date": {
"from": "2017-01-15T00:00:00Z",
"include_lower": true,
"include_upper": true,
"to": "2017-01-17T23:59:59Z"
}
}
}
}
}
}
},
{
"has_child": {
"type": "device_logs",
"score_mode": "sum",
"query" : {
"function_score" : {
"script_score": {
"script": "_score * doc['logs'].value"
}
}
}
}
}
]
}
}
}
EDIT: Adding mappings and some docs
Here you have the mappings:
"mappings": {
"device": {
"properties": {
"app": {
"type": "long",
"include_in_all": false
},
"created_at": {
"type": "date",
"include_in_all": false
},
"id": {
"type": "long",
"include_in_all": false
},
"language": {
"type": "keyword",
"include_in_all": false,
"ignore_above": 256
},
"last_log_at": {
"type": "date",
"include_in_all": false
},
"last_ping_at": {
"type": "date",
"include_in_all": false
},
"last_seen_at": {
"type": "date"
},
"log_enabled": {
"type": "boolean"
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
},
"lowercase": {
"type": "text",
"analyzer": "case_insensitive_sort"
}
}
},
"os_version": {
"type": "keyword",
"include_in_all": false,
"ignore_above": 256
},
"timezone": {
"type": "keyword",
"include_in_all": false,
"ignore_above": 256
},
"type": {
"type": "keyword",
"ignore_above": 256
},
"udid": {
"type": "keyword",
"ignore_above": 256
},
"version": {
"properties": {
"build": {
"type": "keyword",
"include_in_all": false,
"ignore_above": 256
},
"id": {
"type": "long",
"include_in_all": false
},
"version": {
"type": "keyword",
"include_in_all": false,
"ignore_above": 256
}
}
}
}
},
"device_logs": {
"_parent": {
"type": "device"
},
"_routing": {
"required": true
},
"properties": {
"_": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"app": {
"type": "long",
"include_in_all": false
},
"date": {
"type": "date",
"include_in_all": false
},
"errors": {
"type": "long",
"include_in_all": false
},
"logs": {
"type": "long",
"include_in_all": false
},
"warnings": {
"type": "long",
"include_in_all": false
}
}
}
}
And some documents:
{
"_index": "devices",
"_type": "device_logs",
"_id": "22466_2017.01.17",
"_score": 1,
"_routing": "22466",
"_parent": "22466",
"_source": {
"_": "22466_2017.01.17",
"app": 200,
"date": "2017-01-17T00:00:00Z",
"logs": 660,
"warnings": 238,
"errors": 217
}
}
{
"_index": "devices",
"_type": "device",
"_id": "22466",
"_score": 1,
"_source": {
"id": 22466,
"udid": "770CA14ED7FE861EC452",
"name": "Edward's iPhone",
"type": "iPhone7,2",
"app": 200,
"log_enabled": false,
"created_at": "2016-12-21T10:55:02Z",
"last_seen_at": "2017-01-19T10:07:33Z",
"last_log_at": "2017-01-19T11:07:40.756275026+01:00",
"language": "en-US",
"os_version": "9.2",
"timezone": "GMT+1",
"version.id": 7305,
"version.version": "1",
"version.build": "100"
}
}
I have solved your query.
From the first look at the query, I was doubtful that you are not filtering the child documents in one of the must filters before applying the function score on the child document.
I have used the following set of documents for this query
parent doc
{
"id": 22466,
"udid": "770CA14ED7FE861EC452",
"name": "Edward's iPhone",
"type": "iPhone7,2",
"app": 347,
"log_enabled": false,
"created_at": "2016-12-21T10:55:02Z",
"last_seen_at": "2017-01-19T10:07:33Z",
"last_log_at": "2017-01-19T11:07:40.756275026+01:00",
"language": "en-US",
"os_version": "9.2",
"timezone": "GMT+1",
"version.id": 7305,
"version.version": "1",
"version.build": "100"
}
child docs
{
"_type": "device_logs",
"_id": "22466_2017.01.17",
"_score": 0,
"_routing": "22466",
"_parent": "22466",
"_source": {
"_": "22466_2017.01.17",
"app": 200,
"date": "2017-01-17T00:00:00Z",
"logs": 660,
"warnings": 238,
"errors": 217
}
},
{
"_type": "device_logs",
"_id": "22466_2017.02.17",
"_score": 0,
"_routing": "22466",
"_parent": "22466",
"_source": {
"_": "22466_2017.02.17",
"app": 200,
"date": "2017-01-17T00:00:00Z",
"logs": 200,
"warnings": 238,
"errors": 217
}
},
{
"_type": "device_logs",
"_id": "22466_2017.02.20",
"_score": 0,
"_routing": "22466",
"_parent": "22466",
"_source": {
"_": "22466_2017.02.20",
"app": 200,
"date": "2017-01-20T00:00:00Z",
"logs": 200,
"warnings": 238,
"errors": 217
}
}
Note - The first must filter only filter the documents for innerhits.
Please use the following query:
{
"query": {
"bool": {
"filter": [{
"term": {
"app": 347
}
}],
"must": [{
"has_child": {
"type": "device_logs",
"inner_hits": {},
"query": {
"bool": {
"filter": {
"range": {
"date": {
"from": "2017-01-15T00:00:00Z",
"include_lower": true,
"include_upper": true,
"to": "2017-01-17T23:59:59Z"
}
}
}
}
}
}
}, {
"has_child": {
"type": "device_logs",
"score_mode": "sum",
"query": {
"function_score": {
"query": {
"bool": {
"filter": {
"range": {
"date": {
"from": "2017-01-15T00:00:00Z",
"include_lower": true,
"include_upper": true,
"to": "2017-01-17T23:59:59Z"
}
}
}
}
},
"score_mode": "sum",
"boost_mode": "sum",
"script_score": {
"script": "_score + doc['logs'].value"
}
}
}
}
}]
}
}
}
Few references https://github.com/elastic/elasticsearch/issues/10051
Following is the response I get with explain bool set to true
{
"took": 7,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"failed": 0
},
"hits": {
"total": 1,
"max_score": 861,
"hits": [
{
"_shard": "[array_index1][0]",
"_node": "nnauJDrIS8-QCqicOMF23g",
"_index": "array_index1",
"_type": "device",
"_id": "22466",
"_score": 861,
"_source": {
"id": 22466,
"udid": "770CA14ED7FE861EC452",
"name": "Edward's iPhone",
"type": "iPhone7,2",
"app": 347,
"log_enabled": false,
"created_at": "2016-12-21T10:55:02Z",
"last_seen_at": "2017-01-19T10:07:33Z",
"last_log_at": "2017-01-19T11:07:40.756275026+01:00",
"language": "en-US",
"os_version": "9.2",
"timezone": "GMT+1",
"version.id": 7305,
"version.version": "1",
"version.build": "100"
},
"_explanation": {
"value": 861,
"description": "sum of:",
"details": [
{
"value": 1,
"description": "A match, join value 22466",
"details": []
},
{
"value": 860,
"description": "A match, join value 22466",
"details": []
},
{
"value": 0,
"description": "match on required clause, product of:",
"details": [
{
"value": 0,
"description": "# clause",
"details": []
},
{
"value": 1,
"description": "app:[347 TO 347], product of:",
"details": [
{
"value": 1,
"description": "boost",
"details": []
},
{
"value": 1,
"description": "queryNorm",
"details": []
}
]
}
]
}
]
},
"inner_hits": {
"device_logs": {
"hits": {
"total": 2,
"max_score": 0,
"hits": [
{
"_type": "device_logs",
"_id": "22466_2017.01.17",
"_score": 0,
"_routing": "22466",
"_parent": "22466",
"_source": {
"_": "22466_2017.01.17",
"app": 200,
"date": "2017-01-17T00:00:00Z",
"logs": 660,
"warnings": 238,
"errors": 217
}
},
{
"_type": "device_logs",
"_id": "22466_2017.02.17",
"_score": 0,
"_routing": "22466",
"_parent": "22466",
"_source": {
"_": "22466_2017.02.17",
"app": 200,
"date": "2017-01-17T00:00:00Z",
"logs": 200,
"warnings": 238,
"errors": 217
}
}
]
}
}
}
}
]
}
}
Please verify your results.

Resources