I'm just trying to sync my mongodb with ElasticSearch. I've done the sync with river, and river already worked. But the ES won't show all fields, it only show "_ts" field in the object of "_source":
Request:
GET localhost:9200/test/orders/_search
Response:
{
"took": 11,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"failed": 0
},
"hits": {
"total": 137,
"max_score": 1,
"hits": [
{
"_index": "test",
"_type": "orders",
"_id": "58a3251f761f35a107724add",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
},
{
"_index": "test",
"_type": "orders",
"_id": "58a340467f39c50f3a54c614",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
},
{
"_index": "test",
"_type": "orders",
"_id": "58b8ec806f34179d7c7b2431",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
},
{
"_index": "test",
"_type": "orders",
"_id": "58b8eff56f3417670f7b244a",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
},
{
"_index": "test",
"_type": "orders",
"_id": "58b8f0af6f3417fb207b244c",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
},
{
"_index": "test",
"_type": "orders",
"_id": "58b8f19a6f341761337b23da",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
},
{
"_index": "test",
"_type": "orders",
"_id": "58b9320c6f3417bc1c7b23c7",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
},
{
"_index": "test",
"_type": "orders",
"_id": "58b9339f6f341777237b23c6",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
},
{
"_index": "test",
"_type": "orders",
"_id": "58b934ab6f341778237b23c7",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
},
{
"_index": "test",
"_type": "orders",
"_id": "58b96ef76f34174a4b7b23c8",
"_score": 1,
"_source": {
"_ts": 6438761296509796000
}
}
]
}
}
I already did the remapping, and It shows the updated mapping. See this:
request:
GET localhost:9200/test/orders/_mapping
response:
{
"test": {
"mappings": {
"orders": {
"properties": {
"_ts": {
"type": "long"
},
"activeDate": {
"type": "text"
},
"awbNumber": {
"type": "text"
},
"batchID": {
"type": "text"
},
"consignee": {
"properties": {
"id": {
"type": "text"
},
"name": {
"type": "text"
},
"phoneNumber": {
"type": "text"
}
}
},
"consigner": {
"properties": {
"id": {
"type": "text"
},
"name": {
"type": "text"
},
"phoneNumber": {
"type": "text"
}
}
},
"courier": {
"properties": {
"actualRate": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"id": {
"type": "integer"
},
"max_day": {
"type": "integer"
},
"min_day": {
"type": "integer"
},
"name": {
"type": "text"
},
"rate": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"rate_id": {
"type": "integer"
},
"rate_name": {
"type": "text"
},
"shipmentType": {
"type": "integer"
}
}
},
"creationDate": {
"type": "text"
},
"destination": {
"properties": {
"address": {
"type": "text"
},
"cityID": {
"type": "integer"
},
"cityName": {
"type": "text"
},
"id": {
"type": "integer"
},
"provinceID": {
"type": "integer"
},
"provinceName": {
"type": "text"
}
}
},
"driver": {
"properties": {
"feedback": {
"properties": {
"comment": {
"type": "text"
},
"score": {
"type": "long"
}
}
},
"id": {
"type": "long"
},
"isPaymentCollected": {
"type": "integer"
},
"name": {
"type": "text"
},
"phoneNumber": {
"type": "text"
},
"vehicleNumber": {
"type": "text"
},
"vehicleType": {
"type": "text"
}
}
},
"externalID": {
"type": "text"
},
"groupID": {
"type": "integer"
},
"id": {
"type": "text"
},
"isActive": {
"type": "long"
},
"isAutoTrack": {
"type": "integer"
},
"isCustomAWB": {
"type": "integer"
},
"isEscrow": {
"type": "integer"
},
"isLabelPrinted": {
"type": "integer"
},
"lastUpdatedDate": {
"type": "text"
},
"origin": {
"properties": {
"address": {
"type": "text"
},
"cityID": {
"type": "integer"
},
"cityName": {
"type": "text"
},
"id": {
"type": "integer"
},
"provinceID": {
"type": "integer"
},
"provinceName": {
"type": "text"
}
}
},
"package": {
"properties": {
"content": {
"type": "text"
},
"contents": {
"type": "integer"
},
"cubicalWeight": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"dimension": {
"properties": {
"height": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"length": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"width": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
}
}
},
"fragile": {
"type": "integer"
},
"isConfirmed": {
"type": "integer"
},
"itemName": {
"type": "text"
},
"itemSubtype": {
"type": "integer"
},
"itemType": {
"type": "integer"
},
"pictureURL": {
"type": "text"
},
"price": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"type": {
"type": "integer"
},
"weight": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
}
}
},
"paymentType": {
"type": "text"
},
"pickUpTime": {
"type": "text"
},
"rates": {
"properties": {
"actualInsurance": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "long"
}
}
},
"actualShipment": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"escrowCost": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"fulfillmentCost": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"insurance": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "long"
}
}
},
"itemPrice": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"liability": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
},
"shipment": {
"properties": {
"UoM": {
"type": "text"
},
"value": {
"type": "integer"
}
}
}
}
},
"readyTime": {
"type": "text"
},
"shipmentStatus": {
"properties": {
"description": {
"type": "text"
},
"name": {
"type": "text"
},
"statusCode": {
"type": "integer"
},
"updateDate": {
"type": "text"
},
"updatedBy": {
"type": "text"
}
}
},
"source": {
"type": "text"
},
"specialID": {
"type": "text"
},
"stickerNumber": {
"type": "text"
},
"useInsurance": {
"type": "integer"
}
}
}
}
}
}
I'm expecting I can get the whole fields (not just a single "_ts" field) in the "_source" field which it should be the same with what I've mapped.
I've tried to delete the index and recreate one, still not worked. Any clue of this kind of issue. I really really need help, thank you so much.
ElasticSearch has a behavior of creating fields on its own. So if you do
PUT document/index/1
{
"id" : "1",
"name" : "kashish",
"phoneNumber" : "9740683281"
}
This will automatically create fields for your index. What I am illustrating is if explicitly defining of fields is not working for you for some reason , you can empty the index(if contains dummy data) and then just put your json which ES will pickup automatically.
Related
Using Elastic version 7.15.1
{
"mappings": {
"properties": {
"Activity": {
"type": "nested",
"properties": {
"Data": {
"type": "text"
},
"Type": {
"type": "keyword"
},
"created_at": {
"type": "date"
},
"updated_at": {
"type": "date"
}
}
},
"FirstName": {
"type": "text",
"analyzer": "standard_autocomplete",
"search_analyzer": "standard_autocomplete_search"
}
}
}
}
Example Data
{
"Activity": [
{
"Type": "type1",
"Data": "data",
"created_at": "2022-08-08T15:23:58.000000Z"
},
{
"Type": "type1",
"Data": "data",
"created_at": "2022-08-08T15:25:45.000000Z"
},
{
"Type": "type2",
"Data": "data",
"created_at": "2022-08-08T15:26:03.000000Z"
}
],
"FirstName": "Testtt"
}
Want this document to return only if "Activity.Type" is "type1" and the count of the "type1" is greater than 1.
Also how can we use created_at in nested array with above constraint
I want to get data with different value but same field name in array of object
I have this data with stringFacets array of object contains criteria in elastict search 7.9
{
"took": 2238,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"skipped": 0,
"failed": 0
},
"hits": [{
"stringFacets": [{
"name": "criterion",
"value": "Accès Wifi"
},
{
"name": "criterion",
"value": "Piscine"
}
]
}]
}
I want in my search get documents where stringFacets.name = "criterion" and stringFacets.value = "Piscine" and stringFacets.value = "Accès Wifi"
I tried this but no result
{
"query": {
"bool": {
"must": [
{
"nested": {
"path": "stringFacets",
"query": {
"bool": {
"must": [
{
"term": {
"stringFacets.name": "criterion"
}
},
{
"term": {
"stringFacets.value": "Piscine"
}
},
{
"term": {
"stringFacets.value": "Accès Wifi"
}
}
]
}
}
}
}
]
}
}
My mapping
{
"settings": {
"number_of_shards": "1"
},
"mappings": {
"dynamic": false,
"dynamic_templates": [{
"results": {
"mapping": {
"type": "text",
"index": false
},
"path_match": "results.*"
}
}
],
"properties": {
"#version" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"#timestamp" : {
"type" : "date"
},
"booleanFacets": {
"type": "nested",
"properties": {
"name": {
"type": "keyword"
},
"value": {
"type": "boolean"
}
}
},
"stringFacets": {
"type": "nested",
"properties": {
"name": {
"type": "keyword"
},
"value": {
"type": "keyword"
}
}
},
"locationFacets": {
"type": "geo_point"
},
"integerFacets": {
"type": "nested",
"properties": {
"name": {
"type": "keyword"
},
"value": {
"type": "long"
}
}
},
"decimalFacets": {
"type": "nested",
"properties": {
"name": {
"type": "keyword"
},
"value": {
"type": "double"
}
}
},
"datetimeFacets": {
"type": "nested",
"properties": {
"name": {
"type": "keyword"
},
"value": {
"type": "date"
}
}
},
"availabilities": {
"type": "nested",
"properties": {
"start": {
"type": "date",
"format": "yyyy-MM-dd HH:mm:ss.SSSSSS"
},
"end": {
"type": "date",
"format": "yyyy-MM-dd HH:mm:ss.SSSSSS"
},
"price": {
"type": "double"
},
"duration": {
"type": "long"
}
}
}
}
}
}
Thanks
The nested type is a specialized version of the object data type that
allows arrays of objects to be indexed in a way that they can be
queried independently of each other.
You are getting 0 results because there is no single object in your sample data that match all three conditions.
You can use inner_hits resulting in an inner nested query to automatically match the relevant nesting level, rather than the root
Modify your query as
{
"query": {
"nested": {
"path": "stringFacets",
"query": {
"bool": {
"should": [
{
"term": {
"stringFacets.name": "criterion"
}
},
{
"term": {
"stringFacets.value": "Piscine"
}
},
{
"term": {
"stringFacets.value": "Accès Wifi"
}
}
],
"minimum_should_match":2
}
},
"inner_hits": {}
}
}
}
Search Result will be
"hits": {
"total": {
"value": 1,
"relation": "eq"
},
"max_score": 0.8754687,
"hits": [
{
"_index": "66170374",
"_type": "_doc",
"_id": "1",
"_score": 0.8754687,
"_source": {
"stringFacets": [
{
"name": "criterion",
"value": "Accès Wifi"
},
{
"name": "criterion",
"value": "Piscine"
}
]
},
"inner_hits": {
"stringFacets": {
"hits": {
"total": {
"value": 2,
"relation": "eq"
},
"max_score": 0.8754687,
"hits": [
{
"_index": "66170374",
"_type": "_doc",
"_id": "1",
"_nested": {
"field": "stringFacets",
"offset": 0
},
"_score": 0.8754687,
"_source": {
"name": "criterion", // note this
"value": "Accès Wifi"
}
},
{
"_index": "66170374",
"_type": "_doc",
"_id": "1",
"_nested": {
"field": "stringFacets",
"offset": 1
},
"_score": 0.8754687,
"_source": {
"name": "criterion", // note this
"value": "Piscine"
}
}
]
}
}
}
}
]
}
I have mapping as follow
{
"dsi2": {
"mappings": {
"dsi2": {
"_all": {
"enabled": true
},
"properties": {
"instance": {
"properties": {
"assignee": {
"type": "string"
},
"createdTs": {
"type": "long"
},
"dataSourceId": {
"type": "integer"
},
"dsTypeId": {
"type": "integer"
},
"entitlements": {
"properties": {
"create": {
"type": "boolean"
},
"delete": {
"type": "boolean"
},
"edit": {
"type": "boolean"
},
"read": {
"type": "boolean"
},
"roleId": {
"type": "integer"
}
},
"type": "nested"
},
"formDefinitionId": {
"type": "long"
},
"formTypeId": {
"type": "long"
},
"id": {
"fields": {
"raw": {
"type": "integer"
}
},
"type": "integer"
},
"instFields": {
"properties": {
"fieldBoolean": {
"type": "boolean"
},
"fieldDate": {
"format": "strict_date_optional_time||epoch_millis",
"type": "date"
},
"fieldDouble": {
"fields": {
"raw": {
"type": "double"
}
},
"type": "double"
},
"fieldLong": {
"fields": {
"raw": {
"type": "long"
}
},
"type": "long"
},
"fieldString": {
"fields": {
"raw": {
"index": "not_analyzed",
"type": "string"
}
},
"type": "string"
},
"fieldValue": {
"fields": {
"raw": {
"index": "not_analyzed",
"type": "string"
}
},
"type": "string"
},
"isKey": {
"type": "boolean"
},
"referredInstKeyData": {
"properties": {
"fieldValue": {
"fields": {
"raw": {
"index": "not_analyzed",
"type": "string"
}
},
"type": "string"
},
"sortOrder": {
"type": "integer"
},
"sourceFieldId": {
"type": "integer"
}
},
"type": "nested"
},
"sortOrder": {
"type": "long"
},
"sourceFieldId": {
"type": "integer"
}
},
"type": "nested"
},
"instanceId": {
"type": "integer"
},
"modifiedTs": {
"type": "long"
},
"parentEntityId": {
"type": "integer"
},
"processId": {
"type": "string"
},
"processName": {
"type": "string"
},
"relatedInstances": {
"properties": {
"createdTs": {
"type": "long"
},
"dataSourceId": {
"type": "integer"
},
"dsTypeId": {
"type": "integer"
},
"entitlements": {
"properties": {
"create": {
"type": "boolean"
},
"delete": {
"type": "boolean"
},
"edit": {
"type": "boolean"
},
"read": {
"type": "boolean"
},
"roleId": {
"type": "long"
}
}
},
"formDefinitionId": {
"type": "long"
},
"formTypeId": {
"type": "long"
},
"id": {
"type": "integer"
},
"instFields": {
"properties": {
"fieldBoolean": {
"type": "boolean"
},
"fieldDate": {
"format": "strict_date_optional_time||epoch_millis",
"type": "date"
},
"fieldDouble": {
"fields": {
"raw": {
"type": "double"
}
},
"type": "double"
},
"fieldLong": {
"fields": {
"raw": {
"type": "long"
}
},
"type": "long"
},
"fieldString": {
"fields": {
"raw": {
"index": "not_analyzed",
"type": "string"
}
},
"type": "string"
},
"fieldValue": {
"fields": {
"raw": {
"index": "not_analyzed",
"type": "string"
}
},
"type": "string"
},
"isKey": {
"type": "boolean"
},
"referredInstKeyData": {
"properties": {
"fieldString": {
"fields": {
"raw": {
"index": "not_analyzed",
"type": "string"
}
},
"type": "string"
},
"fieldValue": {
"fields": {
"raw": {
"index": "not_analyzed",
"type": "string"
}
},
"type": "string"
},
"sortOrder": {
"type": "integer"
},
"sourceFieldId": {
"type": "integer"
}
},
"type": "nested"
},
"sortOrder": {
"type": "long"
},
"sourceFieldId": {
"type": "integer"
}
},
"type": "nested"
},
"instanceId": {
"type": "integer"
},
"modifiedTs": {
"type": "long"
},
"parentEntityId": {
"type": "integer"
},
"status": {
"type": "long"
},
"tenantId": {
"type": "long"
}
},
"type": "nested"
},
"status": {
"type": "long"
},
"taskAssignees": {
"properties": {
"role": {
"type": "integer"
},
"user": {
"type": "integer"
}
},
"type": "nested"
},
"taskCompleted": {
"format": "strict_date_optional_time||epoch_millis",
"type": "date"
},
"taskCompletedBy": {
"type": "string"
},
"taskCreated": {
"format": "strict_date_optional_time||epoch_millis",
"type": "date"
},
"taskName": {
"type": "string"
},
"tenantId": {
"type": "long"
}
}
},
"status": {
"type": "integer"
},
"tenantId": {
"type": "integer"
}
}
}
}
}
}
when using following query
{
"query": {
"nested": {
"path": "instance.instFields",
"query": {
"bool": {
"must": [
{
"term": {
"instance.instFields.fieldValue": "120656"
}
},
{
"exists": {
"field": "instance.instFields.referredInstKeyData"
}
}
]
}
}
}
}
}
one of the example documents in the index is as follow
{
"took": 1,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"failed": 0
},
"hits": {
"total": 1,
"max_score": 10.130359,
"hits": [
{
"_shard": 0,
"_node": "_FohF7-YRyCSh82NmYVslQ",
"_index": "dsi2",
"_type": "dsi2",
"_id": "120655",
"_score": 10.130359,
"_source": {
"instance": {
"entitlements": [
{
"read": true,
"edit": false,
"roleId": 169,
"create": false,
"delete": false
}
],
"formTypeId": 3,
"parentEntityId": 23589,
"createdTs": 1487701846526,
"dataSourceId": 22,
"dsTypeId": 5,
"modifiedTs": 1487783663308,
"instanceId": 17790,
"instFields": [
{
"fieldDouble": 120658,
"sourceFieldId": 1594,
"sortOrder": 0,
"referredInstKeyData": [
{
"sourceFieldId": 1233,
"sortOrder": 0,
"fieldValue": "Depot Transport"
},
{
"sourceFieldId": 1232,
"sortOrder": 1,
"fieldValue": "LONDON"
},
{
"sourceFieldId": 1236,
"sortOrder": 4,
"fieldValue": "513069"
},
{
"sourceFieldId": 1658,
"sortOrder": 5,
"fieldValue": "7378"
}
],
"isKey": false,
"fieldValue": "120658",
"fieldString": "120658"
},
{
"fieldDouble": 120656,
"sourceFieldId": 1593,
"sortOrder": 0,
"referredInstKeyData": [
{
"sourceFieldId": 1233,
"sortOrder": 0,
"fieldValue": "Joe Cadillac Ii Ltd"
},
{
"sourceFieldId": 1236,
"sortOrder": 4,
"fieldValue": "518508"
},
{
"sourceFieldId": 1658,
"sortOrder": 5,
"fieldValue": "7376"
}
],
"isKey": false,
"fieldValue": "120656",
"fieldString": "120656"
},
{
"sourceFieldId": 2580,
"sortOrder": 0,
"isKey": false,
"fieldValue": "52fe850d-fc17-4dd9-a3a2-257485a9eb27",
"fieldString": "52fe850d-fc17-4dd9-a3a2-257485a9eb27"
}
],
"processName": "Vehicle Delivery",
"tenantId": 3,
"formDefinitionId": 729,
"id": 120655,
"assignee": "none",
"status": 4
},
"formTypeId": 3,
"tenantId": 3,
"status": 4
}
}
]
}
}
it's not returning documents where fieldValue": "120656" and referredInstKeyData element is in there. any idea what could be causing exists to not work? Elastic search version is 2.3.1
this is a known issue in Elastic search. exists isn't supported for nested properties. confirmed here. https://github.com/elastic/elasticsearch/issues/23325
I have an index which name is news, here is the stored mapping in the elasticsearch:
{
"news": {
"mappings": {
"newsdetail": {
"properties": {
"active": {
"type": "long"
},
"authorId": {
"type": "long"
},
"creationDate": {
"type": "date",
"format": "dateOptionalTime"
},
"iDate": {
"type": "string"
},
"iTime": {
"type": "string"
},
"isExclusive": {
"type": "boolean"
},
"itemId": {
"type": "long"
},
"languageId": {
"type": "long"
},
"lead": {
"type": "string"
},
"main": {
"type": "long"
},
"mainTitr": {
"type": "long"
},
"mediaType": {
"type": "long"
},
"newsType": {
"type": "long"
},
"pictureId": {
"type": "long"
},
"ref": {
"type": "string"
},
"subMain": {
"type": "long"
},
"subMainTitr": {
"type": "long"
},
"subtitle": {
"type": "string"
},
"text": {
"type": "string"
},
"title": {
"type": "string"
},
"video": {
"type": "string"
},
"viewCount": {
"type": "long"
}
}
}
}
}
}
when I try to search in this index's data it returns this:
{
"took": 1,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"failed": 0
},
"hits": {
"total": 69612,
"max_score": 1,
"hits": [
{
"_index": "news",
"_type": "newsdetail",
"_id": "AU6Gt1fK36ANCl26_meY",
"_score": 1,
"_source": {
"itemId": 452,
"pictureId": 452,
"authorId": 1,
"languageId": 1,
"active": 1,
"title": "خبر تستی",
"main": 1,
"mainTitr": 1,
"subMain": 1,
"subMainTitr": 1,
"video": "ویدئوی تستی",
"newsType": 3,
"creationDate": "2014-06-09T18:38:47.203",
"viewCount": 201,
"isExclusive": false,
"mediaType": 0
}
}
]
}
}
some of fields has missed. what should I do to see whole fields. note that I have not select fields and my request is GET news/_search?pretty=true&size=1
I have an index that I'd like to get suggestions on:
{
"book": {
"_index": {
"enabled": true
},
"_id": {
"index": "not_analyzed",
"store": "yes"
},
"properties": {
"author": {
"type": "completion"
},
"characters": {
"type": "string"
},
"copies": {
"type": "long",
"ignore_malformed": false
},
"otitle": {
"type": "string"
},
"tags": {
"type": "string"
},
"title": {
"type": "string"
},
"year": {
"type": "long",
"ignore_malformed": false,
"index": "analyzed"
},
"available": {
"type": "boolean"
}
}
}
}
I've loaded some data into it using _bulk endpoint.
When I try to query against the suggest endpoint:
POST library/book/_suggest
{
"my_suggestion_1":{
"text": "He",
"term":{
"analyzer":"standard",
"field":"author"
}
}
}
I get back this:
{
"_index": "library",
"_type": "book",
"_id": "_suggest",
"_version": 8,
"created": false
}
What am I missing here?
I think you just need to use completion instead of term in your request body when you ask for suggestions:
POST library/book/_suggest
{
"my_suggestion_1":{
"text": "He",
"completion":{
"analyzer":"standard",
"field":"author"
}
}
}