Elasticsearch: Already re-mapping but It still wont show all fields - elasticsearch

I'm just trying to sync my mongodb with ElasticSearch. I've done the sync with river, and river already worked. But the ES won't show all fields, it only show "_ts" field in the object of "_source":
Request:
GET localhost:9200/test/orders/_search
Response:
{
"took": 11,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"failed": 0
},
"hits": {
"total": 137,
"max_score": 1,
"hits": [
{
"_index": "test",
"_type": "orders",
"_id": "58a3251f761f35a107724add",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
},
{
"_index": "test",
"_type": "orders",
"_id": "58a340467f39c50f3a54c614",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
},
{
"_index": "test",
"_type": "orders",
"_id": "58b8ec806f34179d7c7b2431",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
},
{
"_index": "test",
"_type": "orders",
"_id": "58b8eff56f3417670f7b244a",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
},
{
"_index": "test",
"_type": "orders",
"_id": "58b8f0af6f3417fb207b244c",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
},
{
"_index": "test",
"_type": "orders",
"_id": "58b8f19a6f341761337b23da",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
},
{
"_index": "test",
"_type": "orders",
"_id": "58b9320c6f3417bc1c7b23c7",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
},
{
"_index": "test",
"_type": "orders",
"_id": "58b9339f6f341777237b23c6",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
},
{
"_index": "test",
"_type": "orders",
"_id": "58b934ab6f341778237b23c7",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
},
{
"_index": "test",
"_type": "orders",
"_id": "58b96ef76f34174a4b7b23c8",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
}
]
}
}
I already did the remapping, and It shows the updated mapping. See this:
request:
GET localhost:9200/test/orders/_mapping
response:
{
"test": {
"mappings": {
"orders": {
"properties": {
"_ts": {
"type": "long"
},
"activeDate": {
"type": "text"
},
"awbNumber": {
"type": "text"
},
"batchID": {
"type": "text"
},
"consignee": {
"properties": {
"id": {
"type": "text"
},
"name": {
"type": "text"
},
"phoneNumber": {
"type": "text"
}
}
},
"consigner": {
"properties": {
"id": {
"type": "text"
},
"name": {
"type": "text"
},
"phoneNumber": {
"type": "text"
}
}
},
"courier": {
"properties": {
"actualRate": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"id": {
"type": "integer"
},
"max_day": {
"type": "integer"
},
"min_day": {
"type": "integer"
},
"name": {
"type": "text"
},
"rate": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"rate_id": {
"type": "integer"
},
"rate_name": {
"type": "text"
},
"shipmentType": {
"type": "integer"
}
}
},
"creationDate": {
"type": "text"
},
"destination": {
"properties": {
"address": {
"type": "text"
},
"cityID": {
"type": "integer"
},
"cityName": {
"type": "text"
},
"id": {
"type": "integer"
},
"provinceID": {
"type": "integer"
},
"provinceName": {
"type": "text"
}
}
},
"driver": {
"properties": {
"feedback": {
"properties": {
"comment": {
"type": "text"
},
"score": {
"type": "long"
}
}
},
"id": {
"type": "long"
},
"isPaymentCollected": {
"type": "integer"
},
"name": {
"type": "text"
},
"phoneNumber": {
"type": "text"
},
"vehicleNumber": {
"type": "text"
},
"vehicleType": {
"type": "text"
}
}
},
"externalID": {
"type": "text"
},
"groupID": {
"type": "integer"
},
"id": {
"type": "text"
},
"isActive": {
"type": "long"
},
"isAutoTrack": {
"type": "integer"
},
"isCustomAWB": {
"type": "integer"
},
"isEscrow": {
"type": "integer"
},
"isLabelPrinted": {
"type": "integer"
},
"lastUpdatedDate": {
"type": "text"
},
"origin": {
"properties": {
"address": {
"type": "text"
},
"cityID": {
"type": "integer"
},
"cityName": {
"type": "text"
},
"id": {
"type": "integer"
},
"provinceID": {
"type": "integer"
},
"provinceName": {
"type": "text"
}
}
},
"package": {
"properties": {
"content": {
"type": "text"
},
"contents": {
"type": "integer"
},
"cubicalWeight": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"dimension": {
"properties": {
"height": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"length": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"width": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
}
}
},
"fragile": {
"type": "integer"
},
"isConfirmed": {
"type": "integer"
},
"itemName": {
"type": "text"
},
"itemSubtype": {
"type": "integer"
},
"itemType": {
"type": "integer"
},
"pictureURL": {
"type": "text"
},
"price": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"type": {
"type": "integer"
},
"weight": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
}
}
},
"paymentType": {
"type": "text"
},
"pickUpTime": {
"type": "text"
},
"rates": {
"properties": {
"actualInsurance": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "long"
}
}
},
"actualShipment": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"escrowCost": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"fulfillmentCost": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"insurance": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "long"
}
}
},
"itemPrice": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"liability": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"shipment": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
}
}
},
"readyTime": {
"type": "text"
},
"shipmentStatus": {
"properties": {
"description": {
"type": "text"
},
"name": {
"type": "text"
},
"statusCode": {
"type": "integer"
},
"updateDate": {
"type": "text"
},
"updatedBy": {
"type": "text"
}
}
},
"source": {
"type": "text"
},
"specialID": {
"type": "text"
},
"stickerNumber": {
"type": "text"
},
"useInsurance": {
"type": "integer"
}
}
}
}
}
}
I'm expecting I can get the whole fields (not just a single "_ts" field) in the "_source" field which it should be the same with what I've mapped.
I've tried to delete the index and recreate one, still not worked. Any clue of this kind of issue. I really really need help, thank you so much.

ElasticSearch has a behavior of creating fields on its own. So if you do
PUT document/index/1
{
"id" : "1",
"name" : "kashish",
"phoneNumber" : "9740683281"
}
This will automatically create fields for your index. What I am illustrating is if explicitly defining of fields is not working for you for some reason , you can empty the index(if contains dummy data) and then just put your json which ES will pickup automatically.

Related

Return document based on nested array matched field count in Elastic search

Using Elastic version 7.15.1
{
"mappings": {
"properties": {
"Activity": {
"type": "nested",
"properties": {
"Data": {
"type": "text"
},
"Type": {
"type": "keyword"
},
"created_at": {
"type": "date"
},
"updated_at": {
"type": "date"
}
}
},
"FirstName": {
"type": "text",
"analyzer": "standard_autocomplete",
"search_analyzer": "standard_autocomplete_search"
}
}
}
}
Example Data
{
"Activity": [
{
"Type": "type1",
"Data": "data",
"created_at": "2022-08-08T15:23:58.000000Z"
},
{
"Type": "type1",
"Data": "data",
"created_at": "2022-08-08T15:25:45.000000Z"
},
{
"Type": "type2",
"Data": "data",
"created_at": "2022-08-08T15:26:03.000000Z"
}
],
"FirstName": "Testtt"
}
Want this document to return only if "Activity.Type" is "type1" and the count of the "type1" is greater than 1.
Also how can we use created_at in nested array with above constraint

Get data with different value but same field name in array of object with must?

I want to get data with different value but same field name in array of object
I have this data with stringFacets array of object contains criteria in elastict search 7.9
{
"took": 2238,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"skipped": 0,
"failed": 0
},
"hits": [{
"stringFacets": [{
"name": "criterion",
"value": "Accès Wifi"
},
{
"name": "criterion",
"value": "Piscine"
}
]
}]
}
I want in my search get documents where stringFacets.name = "criterion" and stringFacets.value = "Piscine" and stringFacets.value = "Accès Wifi"
I tried this but no result
{
"query": {
"bool": {
"must": [
{
"nested": {
"path": "stringFacets",
"query": {
"bool": {
"must": [
{
"term": {
"stringFacets.name": "criterion"
}
},
{
"term": {
"stringFacets.value": "Piscine"
}
},
{
"term": {
"stringFacets.value": "Accès Wifi"
}
}
]
}
}
}
}
]
}
}
My mapping
{
"settings": {
"number_of_shards": "1"
},
"mappings": {
"dynamic": false,
"dynamic_templates": [{
"results": {
"mapping": {
"type": "text",
"index": false
},
"path_match": "results.*"
}
}
],
"properties": {
"#version" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"#timestamp" : {
"type" : "date"
},
"booleanFacets": {
"type": "nested",
"properties": {
"name": {
"type": "keyword"
},
"value": {
"type": "boolean"
}
}
},
"stringFacets": {
"type": "nested",
"properties": {
"name": {
"type": "keyword"
},
"value": {
"type": "keyword"
}
}
},
"locationFacets": {
"type": "geo_point"
},
"integerFacets": {
"type": "nested",
"properties": {
"name": {
"type": "keyword"
},
"value": {
"type": "long"
}
}
},
"decimalFacets": {
"type": "nested",
"properties": {
"name": {
"type": "keyword"
},
"value": {
"type": "double"
}
}
},
"datetimeFacets": {
"type": "nested",
"properties": {
"name": {
"type": "keyword"
},
"value": {
"type": "date"
}
}
},
"availabilities": {
"type": "nested",
"properties": {
"start": {
"type": "date",
"format": "yyyy-MM-dd HH:mm:ss.SSSSSS"
},
"end": {
"type": "date",
"format": "yyyy-MM-dd HH:mm:ss.SSSSSS"
},
"price": {
"type": "double"
},
"duration": {
"type": "long"
}
}
}
}
}
}
Thanks
The nested type is a specialized version of the object data type that
allows arrays of objects to be indexed in a way that they can be
queried independently of each other.
You are getting 0 results because there is no single object in your sample data that match all three conditions.
You can use inner_hits resulting in an inner nested query to automatically match the relevant nesting level, rather than the root
Modify your query as
{
"query": {
"nested": {
"path": "stringFacets",
"query": {
"bool": {
"should": [
{
"term": {
"stringFacets.name": "criterion"
}
},
{
"term": {
"stringFacets.value": "Piscine"
}
},
{
"term": {
"stringFacets.value": "Accès Wifi"
}
}
],
"minimum_should_match":2
}
},
"inner_hits": {}
}
}
}
Search Result will be
"hits": {
"total": {
"value": 1,
"relation": "eq"
},
"max_score": 0.8754687,
"hits": [
{
"_index": "66170374",
"_type": "_doc",
"_id": "1",
"_score": 0.8754687,
"_source": {
"stringFacets": [
{
"name": "criterion",
"value": "Accès Wifi"
},
{
"name": "criterion",
"value": "Piscine"
}
]
},
"inner_hits": {
"stringFacets": {
"hits": {
"total": {
"value": 2,
"relation": "eq"
},
"max_score": 0.8754687,
"hits": [
{
"_index": "66170374",
"_type": "_doc",
"_id": "1",
"_nested": {
"field": "stringFacets",
"offset": 0
},
"_score": 0.8754687,
"_source": {
"name": "criterion", // note this
"value": "Accès Wifi"
}
},
{
"_index": "66170374",
"_type": "_doc",
"_id": "1",
"_nested": {
"field": "stringFacets",
"offset": 1
},
"_score": 0.8754687,
"_source": {
"name": "criterion", // note this
"value": "Piscine"
}
}
]
}
}
}
}
]
}

exists isn't working on a two level nested structure in Elastic Search

I have mapping as follow
{
"dsi2": {
"mappings": {
"dsi2": {
"_all": {
"enabled": true
},
"properties": {
"instance": {
"properties": {
"assignee": {
"type": "string"
},
"createdTs": {
"type": "long"
},
"dataSourceId": {
"type": "integer"
},
"dsTypeId": {
"type": "integer"
},
"entitlements": {
"properties": {
"create": {
"type": "boolean"
},
"delete": {
"type": "boolean"
},
"edit": {
"type": "boolean"
},
"read": {
"type": "boolean"
},
"roleId": {
"type": "integer"
}
},
"type": "nested"
},
"formDefinitionId": {
"type": "long"
},
"formTypeId": {
"type": "long"
},
"id": {
"fields": {
"raw": {
"type": "integer"
}
},
"type": "integer"
},
"instFields": {
"properties": {
"fieldBoolean": {
"type": "boolean"
},
"fieldDate": {
"format": "strict_date_optional_time||epoch_millis",
"type": "date"
},
"fieldDouble": {
"fields": {
"raw": {
"type": "double"
}
},
"type": "double"
},
"fieldLong": {
"fields": {
"raw": {
"type": "long"
}
},
"type": "long"
},
"fieldString": {
"fields": {
"raw": {
"index": "not_analyzed",
"type": "string"
}
},
"type": "string"
},
"fieldValue": {
"fields": {
"raw": {
"index": "not_analyzed",
"type": "string"
}
},
"type": "string"
},
"isKey": {
"type": "boolean"
},
"referredInstKeyData": {
"properties": {
"fieldValue": {
"fields": {
"raw": {
"index": "not_analyzed",
"type": "string"
}
},
"type": "string"
},
"sortOrder": {
"type": "integer"
},
"sourceFieldId": {
"type": "integer"
}
},
"type": "nested"
},
"sortOrder": {
"type": "long"
},
"sourceFieldId": {
"type": "integer"
}
},
"type": "nested"
},
"instanceId": {
"type": "integer"
},
"modifiedTs": {
"type": "long"
},
"parentEntityId": {
"type": "integer"
},
"processId": {
"type": "string"
},
"processName": {
"type": "string"
},
"relatedInstances": {
"properties": {
"createdTs": {
"type": "long"
},
"dataSourceId": {
"type": "integer"
},
"dsTypeId": {
"type": "integer"
},
"entitlements": {
"properties": {
"create": {
"type": "boolean"
},
"delete": {
"type": "boolean"
},
"edit": {
"type": "boolean"
},
"read": {
"type": "boolean"
},
"roleId": {
"type": "long"
}
}
},
"formDefinitionId": {
"type": "long"
},
"formTypeId": {
"type": "long"
},
"id": {
"type": "integer"
},
"instFields": {
"properties": {
"fieldBoolean": {
"type": "boolean"
},
"fieldDate": {
"format": "strict_date_optional_time||epoch_millis",
"type": "date"
},
"fieldDouble": {
"fields": {
"raw": {
"type": "double"
}
},
"type": "double"
},
"fieldLong": {
"fields": {
"raw": {
"type": "long"
}
},
"type": "long"
},
"fieldString": {
"fields": {
"raw": {
"index": "not_analyzed",
"type": "string"
}
},
"type": "string"
},
"fieldValue": {
"fields": {
"raw": {
"index": "not_analyzed",
"type": "string"
}
},
"type": "string"
},
"isKey": {
"type": "boolean"
},
"referredInstKeyData": {
"properties": {
"fieldString": {
"fields": {
"raw": {
"index": "not_analyzed",
"type": "string"
}
},
"type": "string"
},
"fieldValue": {
"fields": {
"raw": {
"index": "not_analyzed",
"type": "string"
}
},
"type": "string"
},
"sortOrder": {
"type": "integer"
},
"sourceFieldId": {
"type": "integer"
}
},
"type": "nested"
},
"sortOrder": {
"type": "long"
},
"sourceFieldId": {
"type": "integer"
}
},
"type": "nested"
},
"instanceId": {
"type": "integer"
},
"modifiedTs": {
"type": "long"
},
"parentEntityId": {
"type": "integer"
},
"status": {
"type": "long"
},
"tenantId": {
"type": "long"
}
},
"type": "nested"
},
"status": {
"type": "long"
},
"taskAssignees": {
"properties": {
"role": {
"type": "integer"
},
"user": {
"type": "integer"
}
},
"type": "nested"
},
"taskCompleted": {
"format": "strict_date_optional_time||epoch_millis",
"type": "date"
},
"taskCompletedBy": {
"type": "string"
},
"taskCreated": {
"format": "strict_date_optional_time||epoch_millis",
"type": "date"
},
"taskName": {
"type": "string"
},
"tenantId": {
"type": "long"
}
}
},
"status": {
"type": "integer"
},
"tenantId": {
"type": "integer"
}
}
}
}
}
}
when using following query
{
"query": {
"nested": {
"path": "instance.instFields",
"query": {
"bool": {
"must": [
{
"term": {
"instance.instFields.fieldValue": "120656"
}
},
{
"exists": {
"field": "instance.instFields.referredInstKeyData"
}
}
]
}
}
}
}
}
one of the example documents in the index is as follow
{
"took": 1,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"failed": 0
},
"hits": {
"total": 1,
"max_score": 10.130359,
"hits": [
{
"_shard": 0,
"_node": "_FohF7-YRyCSh82NmYVslQ",
"_index": "dsi2",
"_type": "dsi2",
"_id": "120655",
"_score": 10.130359,
"_source": {
"instance": {
"entitlements": [
{
"read": true,
"edit": false,
"roleId": 169,
"create": false,
"delete": false
}
],
"formTypeId": 3,
"parentEntityId": 23589,
"createdTs": 1487701846526,
"dataSourceId": 22,
"dsTypeId": 5,
"modifiedTs": 1487783663308,
"instanceId": 17790,
"instFields": [
{
"fieldDouble": 120658,
"sourceFieldId": 1594,
"sortOrder": 0,
"referredInstKeyData": [
{
"sourceFieldId": 1233,
"sortOrder": 0,
"fieldValue": "Depot Transport"
},
{
"sourceFieldId": 1232,
"sortOrder": 1,
"fieldValue": "LONDON"
},
{
"sourceFieldId": 1236,
"sortOrder": 4,
"fieldValue": "513069"
},
{
"sourceFieldId": 1658,
"sortOrder": 5,
"fieldValue": "7378"
}
],
"isKey": false,
"fieldValue": "120658",
"fieldString": "120658"
},
{
"fieldDouble": 120656,
"sourceFieldId": 1593,
"sortOrder": 0,
"referredInstKeyData": [
{
"sourceFieldId": 1233,
"sortOrder": 0,
"fieldValue": "Joe Cadillac Ii Ltd"
},
{
"sourceFieldId": 1236,
"sortOrder": 4,
"fieldValue": "518508"
},
{
"sourceFieldId": 1658,
"sortOrder": 5,
"fieldValue": "7376"
}
],
"isKey": false,
"fieldValue": "120656",
"fieldString": "120656"
},
{
"sourceFieldId": 2580,
"sortOrder": 0,
"isKey": false,
"fieldValue": "52fe850d-fc17-4dd9-a3a2-257485a9eb27",
"fieldString": "52fe850d-fc17-4dd9-a3a2-257485a9eb27"
}
],
"processName": "Vehicle Delivery",
"tenantId": 3,
"formDefinitionId": 729,
"id": 120655,
"assignee": "none",
"status": 4
},
"formTypeId": 3,
"tenantId": 3,
"status": 4
}
}
]
}
}
it's not returning documents where fieldValue": "120656" and referredInstKeyData element is in there. any idea what could be causing exists to not work? Elastic search version is 2.3.1
this is a known issue in Elastic search. exists isn't supported for nested properties. confirmed here. https://github.com/elastic/elasticsearch/issues/23325

Elastic search not return all fields

I have an index which name is news, here is the stored mapping in the elasticsearch:
{
"news": {
"mappings": {
"newsdetail": {
"properties": {
"active": {
"type": "long"
},
"authorId": {
"type": "long"
},
"creationDate": {
"type": "date",
"format": "dateOptionalTime"
},
"iDate": {
"type": "string"
},
"iTime": {
"type": "string"
},
"isExclusive": {
"type": "boolean"
},
"itemId": {
"type": "long"
},
"languageId": {
"type": "long"
},
"lead": {
"type": "string"
},
"main": {
"type": "long"
},
"mainTitr": {
"type": "long"
},
"mediaType": {
"type": "long"
},
"newsType": {
"type": "long"
},
"pictureId": {
"type": "long"
},
"ref": {
"type": "string"
},
"subMain": {
"type": "long"
},
"subMainTitr": {
"type": "long"
},
"subtitle": {
"type": "string"
},
"text": {
"type": "string"
},
"title": {
"type": "string"
},
"video": {
"type": "string"
},
"viewCount": {
"type": "long"
}
}
}
}
}
}
when I try to search in this index's data it returns this:
{
"took": 1,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"failed": 0
},
"hits": {
"total": 69612,
"max_score": 1,
"hits": [
{
"_index": "news",
"_type": "newsdetail",
"_id": "AU6Gt1fK36ANCl26_meY",
"_score": 1,
"_source": {
"itemId": 452,
"pictureId": 452,
"authorId": 1,
"languageId": 1,
"active": 1,
"title": "خبر تستی",
"main": 1,
"mainTitr": 1,
"subMain": 1,
"subMainTitr": 1,
"video": "ویدئوی تستی",
"newsType": 3,
"creationDate": "2014-06-09T18:38:47.203",
"viewCount": 201,
"isExclusive": false,
"mediaType": 0
}
}
]
}
}
some of fields has missed. what should I do to see whole fields. note that I have not select fields and my request is GET news/_search?pretty=true&size=1

ElasticSearch _suggest not returning results

I have an index that I'd like to get suggestions on:
{
"book": {
"_index": {
"enabled": true
},
"_id": {
"index": "not_analyzed",
"store": "yes"
},
"properties": {
"author": {
"type": "completion"
},
"characters": {
"type": "string"
},
"copies": {
"type": "long",
"ignore_malformed": false
},
"otitle": {
"type": "string"
},
"tags": {
"type": "string"
},
"title": {
"type": "string"
},
"year": {
"type": "long",
"ignore_malformed": false,
"index": "analyzed"
},
"available": {
"type": "boolean"
}
}
}
}
I've loaded some data into it using _bulk endpoint.
When I try to query against the suggest endpoint:
POST library/book/_suggest
{
"my_suggestion_1":{
"text": "He",
"term":{
"analyzer":"standard",
"field":"author"
}
}
}
I get back this:
{
"_index": "library",
"_type": "book",
"_id": "_suggest",
"_version": 8,
"created": false
}
What am I missing here?
I think you just need to use completion instead of term in your request body when you ask for suggestions:
POST library/book/_suggest
{
"my_suggestion_1":{
"text": "He",
"completion":{
"analyzer":"standard",
"field":"author"
}
}
}

Resources