I created index using:
curl -XPUT localhost:9200/mobapp -d '{
"mappings": {
"publish_messages": {
"properties": {
"title": {
"type": "string"
},
"location": {
"type": "nested",
"position": {
"type": "geo_point"
},
"name": {
"type": "string"
},
"state": {
"type": "string"
},
"country": {
"type": "string"
},
"city": {
"type": "integer"
}
},
"time": {
"type": "date",
"format": "dd-MM-YYYY"
}
}
}
}
}'
I have this index
"hits": [
{
"_index": "mobapp",
"_type": "publish_messages",
"_id": "184123e0-6123-11e5-83d5-7bdc2a9aa3c7",
"_score": 1,
"_source": {
"title": "Kolkata rocka",
"tags": [
"Tag5",
"Tag4"
],
"date": "2015-09-22T12:11:46.335Z",
"location": {
"position": {
"lat": 11.81776,
"lon": 10.9376
},
"country": "India",
"locality": "Bengaluru",
"sublocality_level_1": "Koramangala"
}
}
}
]
I am trying to do this query:
FilterBuilder filter = geoDistanceFilter("location")
.point(lat, lon)
.distance(distanceRangeInkm, DistanceUnit.KILOMETERS)
.optimizeBbox("memory")
.geoDistance(GeoDistance.ARC);
FilterBuilder boolFilter = boolFilter()
.must(termFilter("tags", tag))
.must(filter);
GeoDistanceSortBuilder geoSort = SortBuilders.geoDistanceSort("location").point(lat, lon).order(SortOrder.ASC);
SearchResponse searchResponse
= client.prepareSearch(AppConstants.ES_INDEX)
.setTypes("publish_messages")
.addSort("time", SortOrder.DESC)
.addSort(geoSort)
.setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
.setPostFilter(boolFilter)
.setFrom(startPage).setSize(AppConstants.DEFAULT_PAGINATION_SIZE)
.execute()
.actionGet();
I am getting QueryParsingException[[mobapp] failed to find geo_point field [location.position]]; }
If you only want to keep your location data together, you don't need to use the nested type, simply use a normal object type (i.e. the default), like this:
curl -XPUT localhost:9200/mobapp -d '{
"mappings": {
"publish_messages": {
"properties": {
"title": {
"type": "string"
},
"location": {
"type": "object", <--- use object here
"properties": { <--- and don't forget properties here
"position": {
"type": "geo_point"
},
"name": {
"type": "string"
},
"state": {
"type": "string"
},
"country": {
"type": "string"
},
"city": {
"type": "integer"
}
}
},
"time": {
"type": "date",
"format": "dd-MM-YYYY"
}
}
}
}
}'
Note that you first need to wipe out your current index using curl -XDELETE localhost:9200/mobapp and then recreate it with the above command and reindex your data. Your query should work afterwards.
Related
Using Elastic version 7.15.1
{
"mappings": {
"properties": {
"Activity": {
"type": "nested",
"properties": {
"Data": {
"type": "text"
},
"Type": {
"type": "keyword"
},
"created_at": {
"type": "date"
},
"updated_at": {
"type": "date"
}
}
},
"FirstName": {
"type": "text",
"analyzer": "standard_autocomplete",
"search_analyzer": "standard_autocomplete_search"
}
}
}
}
Example Data
{
"Activity": [
{
"Type": "type1",
"Data": "data",
"created_at": "2022-08-08T15:23:58.000000Z"
},
{
"Type": "type1",
"Data": "data",
"created_at": "2022-08-08T15:25:45.000000Z"
},
{
"Type": "type2",
"Data": "data",
"created_at": "2022-08-08T15:26:03.000000Z"
}
],
"FirstName": "Testtt"
}
Want this document to return only if "Activity.Type" is "type1" and the count of the "type1" is greater than 1.
Also how can we use created_at in nested array with above constraint
I am reindexing my index data from ES 5.0(parent-child) to ES 6.2(Join type)
Data in index ES 5.0 is stored as parent-child documents in separate types and for reindex i have created new index/mapping based on 6.2 in my new cluster.
The parent documents flawlessly reindex to new index but the child documents throwing error as below
{
"index": "index_two",
"type": "_doc",
"id": "AVpisCkMuwDYFnQZiFXl",
"cause": {
"type": "mapper_parsing_exception",
"reason": "failed to parse",
"caused_by": {
"type": "illegal_argument_exception",
"reason": "[routing] is missing for join field [field_relationship]"
}
},
"status": 400
}
scripts i am using to reindex the data
{
"source": {
"remote": {
"host": "http://myescluster.com:9200",
"socket_timeout": "1m",
"connect_timeout": "20s"
},
"index": "index_two",
"type": ["actions"],
"size": 5000,
"query":{
"bool":{
"must":[
{"term": {"client_id.raw": "cl14ous0ydao"}}
]
}
}
},
"dest": {
"index": "index_two",
"type": "_doc"
},
"script": {
"params": {
"jdata": {
"name": "actions"
}
},
"source": "ctx._routing=ctx._routing;ctx.remove('_parent');params.jdata.parent=ctx._source.user_id;ctx._source.field_relationship=params.jdata"
}
}
I have passed the routing field in painless script as the documents are dynamic from source index.
Mapping of the destination index
{
"index_two": {
"mappings": {
"_doc": {
"dynamic_templates": [
{
"template_actions": {
"match_mapping_type": "string",
"mapping": {
"fields": {
"raw": {
"index": true,
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
}
}
}
],
"date_detection": false,
"properties": {
"attributes": {
"type": "nested"
}
},
"cl_other_params": {
"type": "nested"
},
"cl_triggered_ts": {
"type": "date"
},
"cl_utm_params": {
"type": "nested"
},
"end_ts": {
"type": "date"
},
"field_relationship": {
"type": "join",
"eager_global_ordinals": true,
"relations": {
"users": [
"actions",
"segments"
]
}
},
"ip_address": {
"type": "ip"
},
"location": {
"type": "geo_point"
},
"processed_ts": {
"type": "date"
},
"processing_time": {
"type": "date"
},
"products": {
"type": "nested",
"properties": {
"traits": {
"type": "nested"
}
}
},
"segment_id": {
"type": "integer"
},
"start_ts": {
"type": "date"
}
}
}
}
}
My sample source document
{
"_index": "index_two",
"_type": "actions",
"_id": "AVvKUYcceQCc2OyLKWZ9",
"_score": 7.4023576,
"_routing": "cl14ous0ydaob71ab2a1-837c-4904-a755-11e13410fb94",
"_parent": "cl14ous0ydaob71ab2a1-837c-4904-a755-11e13410fb94",
"_source": {
"user_id": "cl14ous0ydaob71ab2a1-837c-4904-a755-11e13410fb94",
"client_id": "cl14ous0ydao",
"session_id": "CL-e0ec3941-6dad-4d2d-bc9b",
"source": "betalist",
"action": "pageview",
"action_type": "pageview",
"device": "Desktop",
"ip_address": "49.35.14.224",
"location": "20.7333 , 77",
"attributes": [
{
"key": "url",
"value": "https://www.google.com/",
"type": "string"
}
],
"products": []
}
}
I had the same issue and searching in elasticsearch discussions I found this that works:
POST _reindex
{
"source": {
"index": "old_index",
"type": "actions"
},
"dest": {
"index": "index_two"
},
"script": {
"source": """
ctx._type = "_doc";
String routingCode = ctx._source.user_id;
Map join = new HashMap();
join.put('name', 'actions');
join.put('parent', routingCode);
ctx._source.put('field_relationship', join);
ctx._parent = null;
ctx._routing = new StringBuffer(routingCode)"""
}
}
Hope this helps :) .
I'd like to point out that routing is generally not required for a join field, however if you're creating the child before the parent is created, then you're going to face this problem.
It's advisable to re-index all the parents first then the children.
I'd like to show my dataset on tile map. I'm using kibana 4.1.1.
My data is set like this:
{
"_index": "business-data",
"_type": "users",
"_id": "AVkRMFztZOUsFUpKvZ-0",
"_score": 1,
"_source": {
"first_name": "Nessa",
"gender": "female",
"location": {
"lat": 48.8668481401949,
"lon": 2.19256871957155
}
}
}
The mapping:
{
"mappings": {
"user": {
"properties": {
"first_name": {
"type": "string"
},
"gender": {
"type": "string"
},
"location": {
"type": "geo_point"
}
}
}
}
}
Location is a valid geo_point.
The tile map is shown when the visualisation is being created, but "No result" when the basic geohash aggregation bu location field is requested.
giving:
I managed to do it with you advice, thanks. Anyways, the kibana index is not event based in my case.
New mapping:
{
"mappings": {
"user": {
"properties": {
"#timestamp" : {
"type" : "date",
"format" : "dateOptionalTime"
},
"user_id": {
"type": "integer"
},
"first_name": {
"type": "string"
},
"gender": {
"type": "string"
},
"age": {
"type": "integer"
},
"location": {
"type": "geo_point"
}
}
}
}
}
I have a log of HTTP requests, one of the fields is a URI field. I want to get the average duration in ms for each URI. I set the y-axis in Kibana to
"Aggregation: Average , Field: durationInMs".
For the x-axis I have
"Aggregation: terms, Field uri, Order by: metric average durationInMs, Order: descending: 5"
Image to clarify:
This gives me a result but it doesn't use the entire URI. It instead splits up the URI and matches parts of it. After a quick google I found "Multi-fields" and I have added a URI.raw field on my index. The analyzed field warning disappeared but I get no result at all.
Any hints or tips?
lsc-logs2 mapping:
{
"lsc-logs2": {
"mappings": {
"httplogentry": {
"properties": {
"context": {
"type": "string"
},
"durationInMs": {
"type": "double"
},
"id": {
"type": "long"
},
"method": {
"type": "string"
},
"source": {
"type": "string"
},
"startTime": {
"type": "date",
"format": "strict_date_optional_time||epoch_millis"
},
"status": {
"type": "long"
},
"uri": {
"type": "string",
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed"
}
}
},
"username": {
"type": "string"
},
"version": {
"type": "long"
}
}
}
}
}
}
An example document:
{
"_index": "lsc-logs2",
"_type": "httplogentry",
"_id": "1148440",
"_score": 1,
"_source": {
"startTime": "2016-08-22T10:30:57.2298086+02:00",
"context": "contexturi",
"method": "GET",
"uri": "http://uri/plannings/unassigned?date=2016-08-22T03:58:57.168Z&page=1&pageSize=9999",
"username": "user",
"source": "192.168.1.82",
"durationInMs": 171.83710000000002,
"status": 200,
"id": 1148440,
"version": 1
}
}
When reindexing data, the httplogentry mapping doesn't get ported from lsc-logs to lsc-logs2, you need to create the destination index+mapping first and only then reindex.
First delete the current destination index
curl -XDELETE localhost:9200/lsc-logs2
Then create it anew by specifying the proper mapping
curl -XPUT localhost:9200/lsc-logs2 -d '{
"mappings": {
"httplogentry": {
"properties": {
"context": {
"type": "string"
},
"durationInMs": {
"type": "double"
},
"id": {
"type": "long"
},
"method": {
"type": "string"
},
"source": {
"type": "string"
},
"startTime": {
"type": "date",
"format": "strict_date_optional_time||epoch_millis"
},
"status": {
"type": "long"
},
"uri": {
"type": "string",
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed"
}
}
},
"username": {
"type": "string"
},
"version": {
"type": "long"
}
}
}
}
}'
Then you can reindex your data:
curl -XPOST localhost:9200/_reindex -d '{
"source": {
"index": "lsc-logs"
},
"dest": {
"index": "lsc-logs2"
}
}'
Then refresh your the fields in your index pattern in Kibana and it should work.
I'm using ES2.3
create new index mapping with below command works.
curl -XPUT 'http://localhost:9200/megacorp' -d '
{
"settings": {
"number_of_shards": 3,
"number_of_replicas": 1
},
"mappings": {
"employee": {
"properties": {
"first_name": {
"type": "string"
},
"last_name": {
"type": "string"
},
"age": {
"type": "integer"
},
"about": {
"type": "string"
},
"interests": {
"type": "string"
},
"join_time": {
"type": "date",
"format": "dateOptionalTime",
"index": "not_analyzed"
}
}
}
}
}
'
now i hope can use a json file to create same index. tmap.json file like below
{
"settings": {
"number_of_shards": 3,
"number_of_replicas": 1
},
"mappings": {
"employee": {
"properties": {
"first_name": {
"type": "string"
},
"last_name": {
"type": "string"
},
"age": {
"type": "integer"
},
"about": {
"type": "string"
},
"interests": {
"type": "string"
},
"join_time": {
"type": "date",
"format": "dateOptionalTime",
"index": "not_analyzed"
}
}
}
},
"aliases": [ "source" ]
}
then i usr curl to create it.
curl -s -XPOST 'localhost:9200/megacorp' --data-binary #tmap.json
and
curl -XPUT 'http://localhost:9200/megacorp' -d #tmap.json
both above commands not working, get error like below.
{"error":{"root_cause":[{"type":"class_cast_exception","reason":"java.util.ArrayList cannot be cast to java.util.Map"}],"type":"class_cast_exception","reason":"java.util.ArrayList cannot be cast to java.util.Map"},"status":500}%
how to create index with curl and my json file? this is really confused me for long time.
can any body help me? thanks.
The way you define alias is wrong. It should be a map instead of an array.
{
"settings": {
"number_of_shards": 3,
"number_of_replicas": 1
},
"mappings": {
"employee": {
"properties": {
"first_name": {
"type": "string"
},
"last_name": {
"type": "string"
},
"age": {
"type": "integer"
},
"about": {
"type": "string"
},
"interests": {
"type": "string"
},
"join_time": {
"type": "date",
"format": "dateOptionalTime",
"index": "not_analyzed"
}
}
}
},
"aliases": { "source": {} }
}
More info about aliases in index creation:
https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html#create-index-aliases