Elasticsearch mapping document for epoch in seconds - elasticsearch

Question

As far as I understand, es only supports epoch in ms. My data source is couchbase and the json documents in there have an insertEpoch that is stored in seconds. I have been struggling to make my mapping document do the seconds to ms conversion for me.
Here is my mapping doc:
{
"template" : "cb*",
"order" : 10,
"mappings" : {
"couchbaseCheckpoint" : {
"_source" : {
"includes" : ["doc.*"]
},
"dynamic_templates": [
{
"store_no_index": {
"match": "*",
"mapping": {
"store" : "no",
"index" : "no",
"include_in_all" : false
}
}
}
]
},
"couchbaseDoc" : {
"_source" : {
"includes" : ["meta.*","doc.*"]
},
"properties" : {
"meta" : {
"type" : "object",
"include_in_all" : false
},
"doc" : {
"type" : "nested",
"include_in_all" : false,
"transform": {
"script": "ctx._source['insertEpoch'] = ctx._source['insertEpoch'] * 1000",
"params": {},
"lang": "groovy"
}
}
}
}
}
}
The transform isn't happening.
New mapping document:
{
"template" : "wheepl",
"order" : 10,
"mappings" : {
"couchbaseCheckpoint" : {
"_source" : {
"includes" : ["doc.*"]
},
"dynamic_templates": [
{
"store_no_index": {
"match": "*",
"mapping": {
"store" : "no",
"index" : "no",
"include_in_all" : false
}
}
}
]
},
"couchbaseDoc" : {
"_timestamp" : {
"enabled" : true,
"store" : true
},
"properties" : {
"meta" : {
"type" : "object",
"include_in_all" : false
},
"doc" : {
"type" : "object",
"include_in_all" : false,
"updateEpoch" : {
"type" : "date",
"format" : "date_time",
"numeric_resolution" : "seconds"
}
}
}
}
}
}
I don't even see the _timestamp field that I should be seeing!
Here's a Kibana screenie:
Thanks

Answer 1

This works in 1.6, as per https://github.com/elastic/elasticsearch/pull/10420.
Even if, internally, the date itself will be kept in milliseconds, you can index it as seconds, retrieve it as seconds, meaning just like you indexed it.
I've tried out a simple test, to see this in action:
PUT /test_dates
{
"mappings": {
"test": {
"properties": {
"time_stamp": {
"type": "date",
"format": "date_time",
"numeric_resolution": "seconds"
}
}
}
}
}
Test data:
POST /test_dates/test/1
{
"time_stamp": "9231200"
}
Retrieving it:
"hits": {
"total": 1,
"max_score": 1,
"hits": [
{
"_index": "test_dates",
"_type": "test",
"_id": "1",
"_score": 1,
"_source": {
"time_stamp": "9231200"
}
}
]
}
To prove it works, running this aggregation:
GET /test_dates/test/_search?search_type=count
{
"aggs": {
"NAME": {
"date_histogram": {
"field": "time_stamp",
"interval": "second",
"format": "yyyy-MM-dd"
}
}
}
}
returns
"aggregations": {
"NAME": {
"buckets": [
{
"key_as_string": "1970-04-17",
"key": 9231200000,
"doc_count": 1
}
]
}
}
Also, your template is a bit wrong. It should be:
"couchbaseDocument": {
"_timestamp": {
"enabled": true,
"store": true
},
"properties": {
"meta": {
"type": "object",
"include_in_all": false
},
"doc": {
"type": "object",
"include_in_all": false,
"properties": {
"updateEpoch": {
"type": "date",
"format": "date_time",
"numeric_resolution": "seconds"
}
}
}
}
}