ElasticSearch/Kibana fields on update of document - elasticsearch

Question

I'm very new to ES. I'm using https://github.com/dariusk/corpora/blob/master/data/humans/us_presidents.json as a learning set.
At first I entered this in Dev Tools tab in Kibana:
POST /presidents/president/1
{ "bo" :
{
"website":"",
"startdate":"2009-01-20",
"role_type_label":"President",
"enddate":"2013-01-20",
"description":"President",
"district":null,
"phone":null,
"title":"President",
"congress_numbers":[
111,
112,
113
],
"title_long":"President",
"current":false,
"person":{
"name":"President Barack Obama [D]",
"firstname":"Barack",
"twitterid":null,
"middlename":"",
"gender":"male",
"bioguideid":"O000167",
"namemod":"",
"birthday":"1961-08-04",
"link":"https://www.govtrack.us/congress/members/barack_obama/400629",
"youtubeid":null,
"sortname":"Obama, Barack (President) [D]",
"lastname":"Obama",
"gender_label":"Male",
"osid":"N00009638",
"pvsid":"9490",
"nickname":"",
"id":400629,
"cspanid":null
}
...
}
}
Then I realized that if I want add more data about individual presidents, I should rather do this:
POST /presidents/president/1
{
"website":"",
"startdate":"2009-01-20",
"role_type_label":"President",
"enddate":"2013-01-20",
"description":"President",
"district":null,
"phone":null,
"title":"President",
"congress_numbers":[
111,
112,
113
],
"title_long":"President",
"current":false,
"person":{
"name":"President Barack Obama [D]",
"firstname":"Barack",
"twitterid":null,
"middlename":"",
"gender":"male",
"bioguideid":"O000167",
"namemod":"",
"birthday":"1961-08-04",
"link":"https://www.govtrack.us/congress/members/barack_obama/400629",
"youtubeid":null,
"sortname":"Obama, Barack (President) [D]",
"lastname":"Obama",
"gender_label":"Male",
"osid":"N00009638",
"pvsid":"9490",
"nickname":"",
"id":400629,
"cspanid":null
}
}
OK, so ES accepted the update fine.
But now, when I go to Management / Index Patterns in Kibana, I see both person.lastname and bo.person.lastname as fields.
Why the previous field remained? Is this normal for ES to retain fields that are no longer in the updated document?
And obviously, apart from exceptionally funny ones, please no quips about today's election results.

Answer 1

This is normal, expected, behaviour for Elasticsearch.
ES, by default, will dynamically map the data you are inserting. When you index multiple objects under the same type in an index, all of these objects share the same mapping. The intention is to allow objects to be inserted that do not necessarily carry all of the potential fields of its type, and be inserted into the index anyways.
You can define a mapping yourself, either upon index creation, or by defining a new type for an index. Mappings can also be updated, with some caveats.
To see the mapping of the type in your index, execute the following:
GET /tk_file.2016/TK_FILE/_mapping
Your response will look like this:
{
"presidents": {
"mappings": {
"president": {
"properties": {
"bo": {
"properties": {
"congress_numbers": {
"type": "long"
},
"current": {
"type": "boolean"
},
"description": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"enddate": {
"type": "date"
},
"person": {
"properties": {
"bioguideid": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"birthday": {
"type": "date"
},
"firstname": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"gender": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"gender_label": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"id": {
"type": "long"
},
"lastname": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"link": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"middlename": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"namemod": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"nickname": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"osid": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"pvsid": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"sortname": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"role_type_label": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"startdate": {
"type": "date"
},
"title": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"title_long": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"website": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"congress_numbers": {
"type": "long"
},
"current": {
"type": "boolean"
},
"description": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"enddate": {
"type": "date"
},
"person": {
"properties": {
"bioguideid": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"birthday": {
"type": "date"
},
"firstname": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"gender": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"gender_label": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"id": {
"type": "long"
},
"lastname": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"link": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"middlename": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"namemod": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"nickname": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"osid": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"pvsid": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"sortname": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"role_type_label": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"startdate": {
"type": "date"
},
"title": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"title_long": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"website": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}
}
Notice here that you have a set of mappings for the bo object and its related sub-fields, as well as having a mapping for each field of your subsequent document. This is the effect of the dynamic mapping.
To disable this mapping flexibility, you can disable dynamic mappings explicitly, and defining the structure of the document yourself, including its datatypes. Perhaps you want the bioguideid to be an integer instead of text, as it is defined in the current mapping? I direct you to the Mappings API.