Creating custom elasticsearch index with logstash - elasticsearch

I have to create custom index in elasticsearch using logstash. I have created new template in elasticsearch, and in logstash configuration i have specify template path,template_name and template_overwrite value,but still whenever I run logstash, new index is generated with logstash-dd-mm-yy regex,not with template_name specified in properties,
logstash -config file is
input {
file {
path => "/temp/file.txt"
type => "words"
start_position => "beginning"
}
}
filter {
mutate {
add_field => {"words" => "%{message}"}
}
}
output {
elasticsearch {
hosts => ["elasticserver:9200"]
template => "pathtotemplate.json"
template_name => "newIndexName-*"
template_overwrite => true
}
stdout{}
}
Index template file is
{
"template": "dictinary-*",
"settings" : {
"number_of_shards" : 1,
"number_of_replicas" : 0,
"index" : {
"query" : { "default_field" : "#words" },
"store" : { "compress" : { "stored" : true, "tv": true } }
}
},
"mappings": {
"_default_": {
"_all": { "enabled": false },
"_source": { "compress": true },
"dynamic_templates": [
{
"string_template" : {
"match" : "*",
"mapping": { "type": "string", "index": "not_analyzed" },
"match_mapping_type" : "string"
}
}
],
"properties" : {
"#fields": { "type": "object", "dynamic": true, "path": "full" },
"#words" : { "type" : "string", "index" : "analyzed" },
"#source" : { "type" : "string", "index" : "not_analyzed" },
"#source_host" : { "type" : "string", "index" : "not_analyzed" },
"#source_path" : { "type" : "string", "index" : "not_analyzed" },
"#tags": { "type": "string", "index" : "not_analyzed" },
"#timestamp" : { "type" : "date", "index" : "not_analyzed" },
"#type" : { "type" : "string", "index" : "not_analyzed" }
}
}
}
}
Please help

To do what you want, you have to set the index parameter in the Elasticsearch output block. Your output block will look like this:
output {
elasticsearch {
hosts => ["elasticserver:9200"]
index => "newIndexName-%{+YYYY.MM.dd}"
template => "pathtotemplate.json"
template_name => "newIndexName-*"
template_overwrite => true
}
stdout{}
}

Related

Not Able to generate geo_point field with geo_point data type Logstash

I'm trying to load data from mysql table ES index using logstash. I'm able to load the data into ES but the location field mapping is not not coming as geo_point type. Its showing as keyword type. So that I'm not able to query on geo_point field.
Any help what is the issue? ES version: 6.7.0
This is my template.json file:
{
"settings" :
{
"number_of_shards" : 1,
"codec": "best_compression",
"number_of_replicas" : 0,
"index.translog.flush_threshold_size": "2g",
"bootstrap.mlockall": true,
"indices.fielddata.cache.size": 25%
},
"mappings":
{
"_doc" :
'| "dynamic_templates":
[
{
"message_field": {
"path_match": "message",
"match_mapping_type": "string",
"mapping": {
"type": "text",
"norms": false
}
}
},
{
"string_fields": {
"match": "*",
"match_mapping_type": "string",
"mapping": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
],
"properties" : {
"#timestamp": {
"type": "date"
},
"#version": {
"type": "keyword"
},
"location": {
"type": "geo_point"
},
"lat" : { "type" : "keyword", "index" : "not_analyzed","index_options" : "docs" },
"lon" : { "type" : "keyword", "index" : "not_analyzed","index_options" : "docs" }
}
}
}
}
logstash.conf fileinput
{
jdbc {
..........
}
}
filter
{
mutate {
convert => { "lon" => "float" }
convert => { "lat" => "float" }
rename => {
"lon" => "[location][lon]"
"lat" => "[location][lat]"
}
}
}
output {
elasticsearch {
hosts => "host:80"
index => "my_index"
manage_template => "false"
document_type => "_doc"
template_name=>"template.json"
document_id => "%{id}"
}
}
I think you're just missing manage_template => true and you can also add template_overwrite => true just to make sure the template is overridden:
elasticsearch {
hosts => "host:80"
index => "my_index"
manage_template => "true" <---- change this
template_overwrite => true <---- also add this
document_type => "_doc"
template_name=>"template.json"
document_id => "%{id}"
}

Ingesting data from Spark to Elasticsearch with index template

In our existing design we are using logstash to fetch data from Kafka (JSON) and put it in ElasticSearch.
We are also using index template mapping while inserting data from logstash to ES and this could be done by setting 'template' property of ES output plugin of logstash, e.g.,
output {
elasticsearch {
template => "elasticsearch-template.json", //template file path
hosts => "localhost:9200"
template_overwrite => true
manage_template => true
codec=>plain
}
}
elasticsearch-template.json looks like below,
{
"template" : "logstash-*",
"settings" : {
"index.refresh_interval" : "3s"
},
"mappings" : {
"_default_" : {
"_all" : {"enabled" : true},
"dynamic_templates" : [ {
"string_fields" : {
"match" : "*",
"match_mapping_type" : "string",
"mapping" : {
"type" : "string", "index" : "analyzed", "omit_norms" : true,
"fields" : {
"raw" : {"type": "string", "index" : "not_analyzed", "ignore_above" : 256, "doc_values":true}
}
}
}
} ],
"properties" : {
"#version": { "type": "string", "index": "not_analyzed" },
"geoip" : {
"type" : "object",
"dynamic": true,
"properties" : {
"location" : { "type" : "geo_point" }
}
}
}
}
}
}
Now we are going to replace logstash with Apache Spark and I want to use similar kind of usage of index template in Spark while inserting data to ES.
I am using elasticsearch-spark_2.11 library for this implementation.
Thanks.

Logstash issues in creating index remove .raw field in kibana

I have written a logstash conf filefor reading logs. If I use the default index, that is logstash-*, I could see .raw field in kibana. However, if I create a new index in conf file in logstash like
output{
elasticsearch {
hosts => "localhost"
index => "batchjob-*"}
}
Then the new index cant configure .raw field. Is there any resolve ways to solve it? Great Thanks.
The raw fields are created by a specific index template that the Logstash elasticsearch output creates in Elasticsearch.
What you can do is simply copy that template to a file named batchjob.json and change the template name to batchjob-* (see below)
{
"template" : "batchjob-*",
"settings" : {
"index.refresh_interval" : "5s"
},
"mappings" : {
"_default_" : {
"_all" : {"enabled" : true, "omit_norms" : true},
"dynamic_templates" : [ {
"message_field" : {
"match" : "message",
"match_mapping_type" : "string",
"mapping" : {
"type" : "string", "index" : "analyzed", "omit_norms" : true,
"fielddata" : { "format" : "disabled" }
}
}
}, {
"string_fields" : {
"match" : "*",
"match_mapping_type" : "string",
"mapping" : {
"type" : "string", "index" : "analyzed", "omit_norms" : true,
"fielddata" : { "format" : "disabled" },
"fields" : {
"raw" : {"type": "string", "index" : "not_analyzed", "ignore_above" : 256}
}
}
}
} ],
"properties" : {
"#timestamp": { "type": "date" },
"#version": { "type": "string", "index": "not_analyzed" },
"geoip" : {
"dynamic": true,
"properties" : {
"ip": { "type": "ip" },
"location" : { "type" : "geo_point" },
"latitude" : { "type" : "float" },
"longitude" : { "type" : "float" }
}
}
}
}
}
}
Then you can modify your elasticsearch output like this:
output {
elasticsearch {
hosts => "localhost"
index => "batchjob-*"
template_name => "batchjob"
template => "/path/to/batchjob.json"
}
}

How to map geoip field in logstash with elasticsearch in order to display it in tile map of Kibana4

I'd like to display geoip fields in tile map of Kibana4.
Using the standard / automatic logstash geoip mapping to elasticsearch it all works fine.
However when creating a non-standard geoip field, I am not quite sure how to customize the elasticsearch-template.json in logstash in order to represent this field correctly in elasticsearch so that it can be chosen in Kibana4 for tile map creation.
Sure, customizing the standard template is not the best way - better create a custom template and point to it in elasticsearch output of logstash.conf. I just quickly wanted to check how the mapping has to be defined, so I modified the standard template.
My logstash.conf:
input {
tcp {
port => 514
type => syslog
}
udp {
port => 514
type => syslog
}
}
filter {
# Standard geoip field is automatically mapped by logstash to
# elastic search by using the elasticsearch-template.json file
geoip { source => "host" }
grok {
match => [
"message", "<%{POSINT:syslog_pri}>%{YEAR} %{SYSLOGTIMESTAMP:syslog_timestamp} %{DATA:device} <%{POSINT:status}> %{WORD:activity} %{DATA:inout} \(%{DATA:msg}\) Src:%{IPV4:src} SPort:%{INT:sport} Dst:%{IPV4:dst} DPort:%{INT:dport} IPP:%{INT:ipp} Rule:%{INT:rule} Interface:%{WORD:iface}",
"message", "<%{POSINT:syslog_pri}>%{YEAR} %{SYSLOGTIMESTAMP:syslog_timestamp} %{DATA:device} <%{POSINT:status}> %{WORD:activity} %{DATA:inout} \(%{DATA:msg}\) Src:%{IPV4:src} Dst:%{IPV4:dst} IPP:%{INT:ipp} Rule:%{INT:rule} Interface:%{WORD:iface}",
"message", "<%{POSINT:syslog_pri}>%{YEAR} %{SYSLOGTIMESTAMP:syslog_timestamp} %{DATA:device} <%{POSINT:status}> %{WORD:activity} %{DATA:inout} \(%{DATA:msg}\) Src:%{IPV4:src} Dst:%{IPV4:dst} Type:%{POSINT:type} Code:%{INT:code} IPP:%{INT:ipp} Rule:%{INT:rule} Interface:%{WORD:iface}"
]
}
# Is not mapped automatically by logstash in that it can be
# chosen in Kibana4 for tile map creation
geoip {
source => "src"
target => "src_geoip"
}
}
output {
elasticsearch {
host => "localhost"
protocol => "http"
}
}
My ...logstash-1.4.2\lib\logstash\outputs\elasticsearch\elasticsearch-template.json:
{
"template" : "logstash-*",
"settings" : {
"index.refresh_interval" : "5s"
},
"mappings" : {
"_default_" : {
"_all" : {"enabled" : true},
"dynamic_templates" : [ {
"string_fields" : {
"match" : "*",
"match_mapping_type" : "string",
"mapping" : {
"type" : "string", "index" : "analyzed", "omit_norms" : true,
"fields" : {
"raw" : {"type": "string", "index" : "not_analyzed", "ignore_above" : 256}
}
}
}
} ],
"properties" : {
"#version": { "type": "string", "index": "not_analyzed" },
"geoip" : {
"type" : "object",
"dynamic": true,
"path": "full",
"properties" : {
"location" : { "type" : "geo_point" }
}
},
"src_geoip" : {
"type" : "object",
"dynamic": true,
"path": "full",
"properties" : {
"location" : { "type" : "geo_point" }
}
}
}
}
}
}
UPDATE: I havent figured out yet when this json file gets applied in elasticsearch. I followed the hints outlined in this question and copied the json file to a config/templates folder in elasticsearch directory. After deleting the indizes and restart of elasticsearch, the template was applied successfully.
Anyway, the field "src_geoip.location" still does not show up in the tile map creation form of Kibana4 (only the standard geoip.location field does).
Try overwrite template after editing template. Re-create indexes in Kibana after config change.
output {
elasticsearch {
template_overwrite => "true"
...
}
}
You also need to add objects for the src_geoip object in the index template on your elasticsearch instance. To set the default template for all indexes that match "logstash-netflow-*", execute the following on your elasticsearch instance:
curl -XPUT localhost:9200/_template/logstash-netflow -d '{
"template" : "logstash-netflow-*",
"mappings" : {
"_default_" : {
"_all" : {
"enabled" : false
},
"properties" : {
"#timestamp" : { "index" : "analyzed", "type" : "date" },
"#version" : { "index" : "analyzed", "type" : "integer" },
"src_geoip" : {
"dynamic" : true,
"type" : "object",
"properties" : {
"area_code" : { "type" : "long" },
"city_name" : { "type" : "string" },
"continent_code" : { "type" : "string" },
"country_code2" : { "type" : "string" },
"country_code3" : { "type" : "string" },
"country_name" : { "type" : "string" },
"dma_code" : { "type" : "long" },
"ip" : { "type" : "string" },
"latitude" : { "type" : "double" },
"location" : { "type" : "double" },
"longitude" : { "type" : "double" },
"postal_code" : { "type" : "string" },
"real_region_name" : { "type" : "string" },
"region_name" : { "type" : "string" },
"timezone" : { "type" : "string" }
}
},
"netflow" : { ....snipped......
}
}
}
}}'

can i set logstash default elasticsearch mapping through elasticsearch-template.json

I use logstash + elasticsearch to collect syslog and want to set ttl for log ageing
I find a file named elasticsearch-template.json in the logstash,the path is logstash/logstash-1.4.2/lib/logstash/outputs/elasticsearch/elasticsearch-template.json
I add ttl info in the file like this:
{
"template" : "logstash-*",
"settings" : {
"index.refresh_interval" : "5s"
},
"mappings" : {
"_default_" : {
"_all" : {"enabled" : true},
"dynamic_templates" : [ {
"string_fields" : {
"match" : "*",
"match_mapping_type" : "string",
"mapping" : {
"type" : "string", "index" : "analyzed", "omit_norms" : true,
"fields" : {
"raw" : {"type": "string", "index" : "not_analyzed", "ignore_above" : 256}
}
}
}
} ],
"_ttl": {
"enabled": true,
"default": "1d"
},
"properties" : {
"#version": { "type": "string", "index": "not_analyzed" },
"geoip" : {
"type" : "object",
"dynamic": true,
"path": "full",
"properties" : {
"location" : { "type" : "geo_point" }
}
}
}
}
}
}
then restart logstash, delete all elasticsearch index.
I check the new index's mapping in the elasticsearch, but it didn't work in this way.
How can I config the index template?
you need to change your logstash configuration.
if you have followed the default settings, logstash has already created a template inside elasticsearch named logstash, logstash will keep on using that template stored in elasticsearch unless you tell it not to explicitly.
modify that template file you found but in addition to that, in your logstash configuration, set the following:
output {
elasticsearch {
...
template_overwrite => true
...
}
}
Doesn't look like that JSON file is in the correct folder. Here is the documentation on how to use the templates:
http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/indices-templates.html
About the folder:
Config
Index templates can also be placed within the config location (path.conf) under the templates directory (note, make sure to place them on all master eligible nodes). For example, a file called template_1.json can be placed under config/templates and it will be added if it matches an index. Here is a sample of the mentioned file:
I've created new template.json file and defined path to it into elasticsearch output block of logstash.yml config file:
stdout { codec => json_lines }
elasticsearch {
"hosts" => ["ip:port"]
"index" => "name-of-index-%{+dd.MM.YYYY}"
template => "/{path-to-logstash-folder}/templates/your-template.json"
template_overwrite => true
manage_template => false
}
document_type for Elastic I defined into input block of logstash.yml config file:
input {
file {
path => "/your-path-to-directory/*.log"
type => "name-of-type"
}
}
There is my template.json file
{
"name-of-index": {
"order": 0,
"version": 50001,
"template": "name-of-index-*",
"settings": {
"index": {
"refresh_interval": "5s"
}
},
"mappings": {
"_default_": {
"dynamic_templates": [
{
"message_field": {
"path_match": "message",
"mapping": {
"norms": false,
"type": "text"
},
"match_mapping_type": "string"
}
},
{
"string_fields": {
"mapping": {
"norms": false,
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"match_mapping_type": "string",
"match": "*"
}
}
],
"_all": {
"norms": false,
"enabled": true
},
"properties": {
"#timestamp": {
"include_in_all": false,
"type": "date"
},
"geoip": {
"dynamic": true,
"properties": {
"ip": {
"type": "ip"
},
"latitude": {
"type": "half_float"
},
"location": {
"type": "geo_point"
},
"longitude": {
"type": "half_float"
}
}
},
"#version": {
"include_in_all": false,
"type": "keyword"
}
}
}
},
"aliases": {}
}
}

Resources