how to find all the unique records in elastic search based on an attribute in it..
Is there a way?
GET questiondetails/question/_search
{
"size": 0,
"aggs" : {
"Name" : {
"terms" : {
"field" : "owner.user_id.keyword"
}
}
}
}
I tried this way.. But I want it to give all the docs wherever user id is present in the whole record.
Whether it is in inner loop also....
Is there a way to get this in a single query instead of making in multiple queries..
Here is my sample record.
{
"_index": "questiondetails",
"_type": "question",
"_id": "BOktPnABW1evGoA0CqDZ",
"_score": 1,
"_source": {
"tags": [
"android",
"kotlin"
],
"owner": {
"user_id": "10633771"
},
"reply_count": "1",
"reply": [
{
"owner": {
"user_id": "12632101",
"user_type": "registered",
}
}
]
}
}
Index
{
"questiondetails": {
"aliases": {},
"mappings": {
"question": {
"properties": {
"accepted_answer_id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"answer_count": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"answers": {
"properties": {
"answer_id": {
"type": "long"
},
"body": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"body_markdown": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"comment_count": {
"type": "long"
},
"comments": {
"properties": {
"comment_id": {
"type": "long"
},
"creation_date": {
"type": "long"
},
"edited": {
"type": "boolean"
},
"owner": {
"properties": {
"accept_rate": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"display_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"link": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"profile_image": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"reputation": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"post_id": {
"type": "long"
},
"reply_to_user": {
"properties": {
"display_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"link": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"profile_image": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"reputation": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"score": {
"type": "long"
}
}
},
"creation_date": {
"type": "long"
},
"down_vote_count": {
"type": "long"
},
"is_accepted": {
"type": "boolean"
},
"last_activity_date": {
"type": "long"
},
"last_edit_date": {
"type": "long"
},
"link": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"owner": {
"properties": {
"accept_rate": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"display_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"link": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"profile_image": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"reputation": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"question_id": {
"type": "long"
},
"score": {
"type": "long"
},
"share_link": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"title": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"up_vote_count": {
"type": "long"
}
}
},
"body": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"body_markdown": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"close_vote_count": {
"type": "long"
},
"comment_count": {
"type": "long"
},
"comments": {
"properties": {
"body": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"body_markdown": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"comment_id": {
"type": "long"
},
"creation_date": {
"type": "long"
},
"edited": {
"type": "boolean"
},
"owner": {
"properties": {
"accept_rate": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"display_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"link": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"profile_image": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"reputation": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"post_id": {
"type": "long"
},
"reply_to_user": {
"properties": {
"accept_rate": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"display_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"link": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"profile_image": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"reputation": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"score": {
"type": "long"
}
}
},
"creation_date": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"delete_vote_count": {
"type": "long"
},
"down_vote_count": {
"type": "long"
},
"is_answered": {
"type": "boolean"
},
"last_activity_date": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"last_edit_date": {
"type": "long"
},
"last_editor": {
"properties": {
"accept_rate": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"display_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"link": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"profile_image": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"reputation": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"link": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"owner": {
"properties": {
"accept_rate": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"display_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"link": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"profile_image": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"reputation": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"question_id": {
"type": "long"
},
"reopen_vote_count": {
"type": "long"
},
"score": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"share_link": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"tags": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"title": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"up_vote_count": {
"type": "long"
},
"view_count": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
},
"settings": {
"index": {
"creation_date": "1581585452123",
"number_of_shards": "5",
"number_of_replicas": "1",
"uuid": "_nvo_G7ZRvqMIwLmddUXPw",
"version": {
"created": "6020399"
},
"provided_name": "questiondetails"
}
}
}
}
You can achieve what you want with a scripted terms aggregation, like this:
POST questions/_search
{
"size": 0,
"aggs": {
"ids": {
"terms": {
"script": "[doc['owner.user_id.keyword'].value, doc['reply.owner.user_id.keyword'].value]",
"size": 10
}
}
}
}
Related
Both the below questions are related to same index
1) I am using Elastic search. My text contains word "companies"
For example: John has worked with companies like Bharat Electronics
and Gillette in senior management positions, wherein he has set up
new units, managed talent acquisition, systems, processes and laying
out SOP for Human Resources
When I search using following query, I don't find any result however it should have returned data:
{
"query": {
"bool": {
"should": [
{ "match":
{
"fields.bio": "company"
}
}
]
}
}
}
I get zero records however it should have found above record.
2) While creating an index I used english stop words but it still returns records bases on words like an.
The code I used to create an index:
{
"settings": {
"analysis": {
"filter": {
"my_stop": {
"type": "stop",
"stopwords": "_english_"
}
}
}
}
}
So, it should not return any record even though the text contain an word But in reality it is returning records if it finds an word in records.
The index mapping is as follows:
{
"experts": {
"mappings": {
"properties": {
"categories": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"expertise": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"fields": {
"properties": {
"annual_income": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"avg_rating": {
"type": "long"
},
"bio": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"calls": {
"type": "long"
},
"current_company": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"current_country_id": {
"type": "long"
},
"current_position": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"email": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"id": {
"type": "long"
},
"linkedin_url": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"mobileno": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"profile_photo_url": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"review_count": {
"type": "long"
},
"total_experience": {
"type": "long"
},
"username": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"users": {
"type": "long"
}
}
},
"functions": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"industries": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"qualifications": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}
Kindly help.
I'm using Kibana 6 to display aggregations from terms indexed in ElasticSearch 6.
I'm encoutering an issue while displaying a bucket aggregating terms (bidRequest.site.domain.keyword) by keywords whereas the records are effectively present in the Index and date range is okay:
I don't understand why there is no result despite having those in the Index:
Here's my mapping:
{
"mapping": {
"BidRequestRecord": {
"properties": {
"bidRequest": {
"type": "nested",
"properties": {
"allimps": {
"type": "integer"
},
"app": {
"type": "nested",
"properties": {
"bundle": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"cat": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"domain": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ext": {
"type": "object"
},
"id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"keywords": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"pagecat": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"publisher": {
"type": "nested",
"properties": {
"ext": {
"type": "object"
},
"id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"storeurl": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ver": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"at": {
"type": "integer"
},
"badv": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"bcat": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"cur": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"device": {
"type": "nested",
"properties": {
"carrier": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"connectiontype": {
"type": "integer"
},
"devicetype": {
"type": "integer"
},
"didmd5": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"didsha1": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"dnt": {
"type": "integer"
},
"dpidmd5": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"dpidsha1": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ext": {
"type": "nested",
"properties": {
"is_app": {
"type": "integer"
}
}
},
"geo": {
"type": "nested",
"properties": {
"city": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"country": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"lat": {
"type": "float"
},
"lon": {
"type": "float"
},
"region": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"type": {
"type": "integer"
},
"zip": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"ifa": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ip": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"js": {
"type": "integer"
},
"language": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"make": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"model": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"os": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"osv": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ua": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"ext": {
"type": "object"
},
"id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"imp": {
"type": "nested",
"properties": {
"banner": {
"type": "nested",
"properties": {
"api": {
"type": "integer"
},
"battr": {
"type": "integer"
},
"btype": {
"type": "integer"
},
"expdir": {
"type": "integer"
},
"ext": {
"type": "object"
},
"format": {
"type": "nested",
"properties": {
"ext": {
"type": "object"
},
"h": {
"type": "integer"
},
"w": {
"type": "integer"
}
}
},
"h": {
"type": "integer"
},
"hmax": {
"type": "integer"
},
"hmin": {
"type": "integer"
},
"id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"mimes": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"pos": {
"type": "integer"
},
"topframe": {
"type": "integer"
},
"w": {
"type": "integer"
},
"wmax": {
"type": "integer"
},
"wmin": {
"type": "integer"
}
}
},
"bidFloorFloat": {
"type": "float"
},
"bidfloor": {
"type": "double"
},
"bidfloorInEur": {
"type": "double"
},
"bidfloorcur": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"displaymanager": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"displaymanagerver": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ext": {
"type": "object"
},
"id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"iframebuster": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"instl": {
"type": "integer"
},
"native": {
"type": "object"
},
"pmp": {
"type": "nested",
"properties": {
"deals": {
"type": "nested",
"properties": {
"at": {
"type": "integer"
},
"bidFloorFloat": {
"type": "float"
},
"bidFloorInEuro": {
"type": "double"
},
"bidFloorInEuroFloat": {
"type": "float"
},
"bidfloor": {
"type": "double"
},
"bidfloorcur": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ext": {
"type": "object"
},
"id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"wadomain": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"wseat": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"ext": {
"type": "object"
},
"private_auction": {
"type": "integer"
}
}
},
"secure": {
"type": "integer"
},
"tagid": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"video": {
"type": "nested",
"properties": {
"api": {
"type": "integer"
},
"battr": {
"type": "integer"
},
"boxingallowed": {
"type": "integer"
},
"companionad": {
"type": "nested",
"properties": {
"api": {
"type": "integer"
},
"battr": {
"type": "integer"
},
"btype": {
"type": "integer"
},
"expdir": {
"type": "integer"
},
"ext": {
"type": "object"
},
"format": {
"type": "nested",
"properties": {
"ext": {
"type": "object"
},
"h": {
"type": "integer"
},
"w": {
"type": "integer"
}
}
},
"h": {
"type": "integer"
},
"hmax": {
"type": "integer"
},
"hmin": {
"type": "integer"
},
"id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"mimes": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"pos": {
"type": "integer"
},
"topframe": {
"type": "integer"
},
"w": {
"type": "integer"
},
"wmax": {
"type": "integer"
},
"wmin": {
"type": "integer"
}
}
},
"companiontype": {
"type": "integer"
},
"ext": {
"type": "object"
},
"h": {
"type": "integer"
},
"linearity": {
"type": "integer"
},
"maxbitrate": {
"type": "integer"
},
"maxduration": {
"type": "integer"
},
"mimes": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"minbitrate": {
"type": "integer"
},
"minduration": {
"type": "integer"
},
"playbackmethod": {
"type": "integer"
},
"pos": {
"type": "integer"
},
"protocols": {
"type": "integer"
},
"startdelay": {
"type": "integer"
},
"w": {
"type": "integer"
}
}
}
}
},
"regs": {
"type": "nested",
"properties": {
"coppa": {
"type": "integer"
},
"ext": {
"type": "object"
}
}
},
"site": {
"type": "nested",
"properties": {
"cat": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"domain": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ext": {
"type": "object"
},
"id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"keywords": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"page": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"pagecat": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"source": {
"type": "nested",
"properties": {
"ext": {
"type": "object"
},
"fd": {
"type": "integer"
},
"pchain": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"tid": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
},
"bidRequestDate": {
"type": "date"
}
}
}
}
}
Well apologies for this one, I was using a not up-to-date index and at that time mapping was wrong. Works perfect with the correctly set mapping.
I am trying to write a query that returns all docs which don't have a particular field which is a nested property but it doesn't work.
I am using ES 5.4
{
"query": {
"bool": {
"must_not": [
{
"exists": {
"field": "shares"
}
}
]
}
}
}
What am I doing wrong?
This is my mapping
{
"test": {
"aliases": {},
"mappings": {
"vendor": {
"properties": {
"address": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"categories": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"city": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"displayTypes": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"emailId": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"facebookAppId": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"foursquareType": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"googleTypes": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"hours": {
"properties": {
"day": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"from": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"to": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"isOnBoarded": {
"type": "boolean"
},
"kiaskCategories": {
"type": "keyword"
},
"latitude": {
"type": "float"
},
"location": {
"type": "geo_point"
},
"longitude": {
"type": "float"
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"phoneNumber": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"pictures": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"placeId": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"price": {
"type": "float"
},
"rating": {
"type": "float"
},
"shares": {
"type": "nested",
"properties": {
"_id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"content": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"createdDate": {
"type": "date"
},
"endDate": {
"type": "date"
},
"facebookLikes": {
"type": "long"
},
"images": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"price": {
"type": "long"
},
"quantity": {
"type": "long"
},
"shareType": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"startDate": {
"type": "date"
},
"tags": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"source": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"state": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"timezone": {
"type": "float"
},
"url": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"website": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"yelpTypes": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
},
"settings": {
"index": {
"creation_date": "1496319860700",
"number_of_shards": "5",
"number_of_replicas": "1",
"uuid": "0S0A9YN-S-SrW4eeOHX06w",
"version": {
"created": "5040099"
},
"provided_name": "test"
}
}
}
}
this should work
You have enforce nested data type mappings and use nested query.
{
"query" : {
"nested": {
"path": "shares",
"query": {
"bool": {
"must_not": [
{
"exists" : {
"field" : "shares._id"
}
}
]
}
}
}
}
}
Note: The following setup works for me.
PUT test_index
{
"mappings": {
"document_type" : {
"properties": {
"name" : {
"type": "text"
},
"shares" : {
"type": "nested"
}
}
}
}
}
POST test_index/document_type
{
"name" : "vicky"
}
POST test_index/_search
{
"query": {
"nested": {
"path": "shares",
"query": {
"bool": {
"must_not": [
{
"exists" : {
"field" : "shares.city"
}
}
]
}
}
}
}
}
I have a log that returns a field called 'ip' from a parsed json field called 'message'.
I've set up my logstash.conf file as so:
filter {
json {
source => "message"
}
geoip {
source => "ip"
target => "geoip"
#add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
#add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
}
#mutate {
# convert => [ "[geoip][coordinates]", "float"]
#}
}
However, I'm not seeing anything I can use in my index pattern to create a map. Kibana gives me the error:
The "myindex*" index pattern does not contain any of the following field types: geo_point
What am I doing wrong here?
EDIT: results of curl on /_mapping
{
"<my index>": {
"mappings": {
"%{[#metadata][type]}": {
"properties": {
"#timestamp": {
"type": "date"
},
"#version": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"host": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"message": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"tracking log": {
"properties": {
"#timestamp": {
"type": "date"
},
"#version": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"accept_language": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"agent": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"beat": {
"properties": {
"hostname": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"version": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"context": {
"properties": {
"course_id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"course_user_tags": {
"type": "object"
},
"org_id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"path": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user_id": {
"type": "long"
}
}
},
"event": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"event_source": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"event_type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"geoip": {
"properties": {
"city_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"continent_code": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"coordinates": {
"type": "float"
},
"country_code2": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"country_code3": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"country_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"dma_code": {
"type": "long"
},
"ip": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"latitude": {
"type": "float"
},
"location": {
"type": "float"
},
"longitude": {
"type": "float"
},
"postal_code": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"region_code": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"region_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"timezone": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"host": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"input_type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"ip": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"message": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"offset": {
"type": "long"
},
"page": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"referer": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"session": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"source": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"tags": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"time": {
"type": "date"
},
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"username": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}
}
I used this command to execute the map:
curl -XPOST -u elastic 'localhost:9200/_template/<my index>' -H 'Content-Type: application/json' -d'
{"order": 0, "template": "<my index>*","mappings": {"_default_": {"dynamic_templates": [{"location_fields": {"mapping": {"type": "geo_point"},"match": "geoip"}}]}}}
'
You don't need the add_field lines or the mutate/convert. What you do need is to properly map the geoip field as a geo_point in elasticsearch.
Generally you'd do this with an index template -- this mapps any field named geoip to a geo_point -- you can be more specific with your templates though.
POST /_template/myindex
{
"order": 0,
"template": "myindex*",
"mappings": {
"_default_": {
"dynamic_templates": [
{
"location_fields": {
"mapping": {
"type": "geo_point"
},
"match": "location"
}
}
]
}
}
}
In any event, you'll need to re-index your data to get the mapping right and then tell kibana to reload your index mappings (under settings).
Ok - I solved this with the help of #Alcanzar. Apparently, as he said, it is impossible to change a the type of a field once data has been sent to it -- essentially, once you've started logstash and started using it.
So -- to map geoip.location to type geo_point -- here it what I did:
Steps:
1) I stopped logstash.
2) Deleted my index using 'DELETE /"
3) Created a new index with the name I wanted using 'PUT /' in the Kibana dev console.
4) Applied the mapping #Alcanzar contributed above using 'geoip.location' instead of 'location'
5) Restarted logstash.
Now geoip.location is of type geo_point.
It might be obvious that these are the steps to take for some of you experienced ES users, but it wasn't to me. I hope this helps someone else.
I'm very new to ES. I'm using https://github.com/dariusk/corpora/blob/master/data/humans/us_presidents.json as a learning set.
At first I entered this in Dev Tools tab in Kibana:
POST /presidents/president/1
{ "bo" :
{
"website":"",
"startdate":"2009-01-20",
"role_type_label":"President",
"enddate":"2013-01-20",
"description":"President",
"district":null,
"phone":null,
"title":"President",
"congress_numbers":[
111,
112,
113
],
"title_long":"President",
"current":false,
"person":{
"name":"President Barack Obama [D]",
"firstname":"Barack",
"twitterid":null,
"middlename":"",
"gender":"male",
"bioguideid":"O000167",
"namemod":"",
"birthday":"1961-08-04",
"link":"https://www.govtrack.us/congress/members/barack_obama/400629",
"youtubeid":null,
"sortname":"Obama, Barack (President) [D]",
"lastname":"Obama",
"gender_label":"Male",
"osid":"N00009638",
"pvsid":"9490",
"nickname":"",
"id":400629,
"cspanid":null
}
...
}
}
Then I realized that if I want add more data about individual presidents, I should rather do this:
POST /presidents/president/1
{
"website":"",
"startdate":"2009-01-20",
"role_type_label":"President",
"enddate":"2013-01-20",
"description":"President",
"district":null,
"phone":null,
"title":"President",
"congress_numbers":[
111,
112,
113
],
"title_long":"President",
"current":false,
"person":{
"name":"President Barack Obama [D]",
"firstname":"Barack",
"twitterid":null,
"middlename":"",
"gender":"male",
"bioguideid":"O000167",
"namemod":"",
"birthday":"1961-08-04",
"link":"https://www.govtrack.us/congress/members/barack_obama/400629",
"youtubeid":null,
"sortname":"Obama, Barack (President) [D]",
"lastname":"Obama",
"gender_label":"Male",
"osid":"N00009638",
"pvsid":"9490",
"nickname":"",
"id":400629,
"cspanid":null
}
}
OK, so ES accepted the update fine.
But now, when I go to Management / Index Patterns in Kibana, I see both person.lastname and bo.person.lastname as fields.
Why the previous field remained? Is this normal for ES to retain fields that are no longer in the updated document?
And obviously, apart from exceptionally funny ones, please no quips about today's election results.
This is normal, expected, behaviour for Elasticsearch.
ES, by default, will dynamically map the data you are inserting. When you index multiple objects under the same type in an index, all of these objects share the same mapping. The intention is to allow objects to be inserted that do not necessarily carry all of the potential fields of its type, and be inserted into the index anyways.
You can define a mapping yourself, either upon index creation, or by defining a new type for an index. Mappings can also be updated, with some caveats.
To see the mapping of the type in your index, execute the following:
GET /tk_file.2016/TK_FILE/_mapping
Your response will look like this:
{
"presidents": {
"mappings": {
"president": {
"properties": {
"bo": {
"properties": {
"congress_numbers": {
"type": "long"
},
"current": {
"type": "boolean"
},
"description": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"enddate": {
"type": "date"
},
"person": {
"properties": {
"bioguideid": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"birthday": {
"type": "date"
},
"firstname": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"gender": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"gender_label": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"id": {
"type": "long"
},
"lastname": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"link": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"middlename": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"namemod": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"nickname": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"osid": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"pvsid": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"sortname": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"role_type_label": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"startdate": {
"type": "date"
},
"title": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"title_long": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"website": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"congress_numbers": {
"type": "long"
},
"current": {
"type": "boolean"
},
"description": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"enddate": {
"type": "date"
},
"person": {
"properties": {
"bioguideid": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"birthday": {
"type": "date"
},
"firstname": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"gender": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"gender_label": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"id": {
"type": "long"
},
"lastname": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"link": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"middlename": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"namemod": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"nickname": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"osid": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"pvsid": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"sortname": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"role_type_label": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"startdate": {
"type": "date"
},
"title": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"title_long": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"website": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}
}
Notice here that you have a set of mappings for the bo object and its related sub-fields, as well as having a mapping for each field of your subsequent document. This is the effect of the dynamic mapping.
To disable this mapping flexibility, you can disable dynamic mappings explicitly, and defining the structure of the document yourself, including its datatypes. Perhaps you want the bioguideid to be an integer instead of text, as it is defined in the current mapping? I direct you to the Mappings API.