Kibana don't store the right data into the right fiels - elasticsearch

Question

Ok so now I've my mapping into kibana.
Here's my mapping:
PUT logstash-2019.05.09
{
"mappings": {
"doc": {
"properties": {
"index": {
"_index": {
"type": "keyword"
},
"_type": {
"type": "text"
}
},
"#timestamp": {
"type": "date"
},
"ip": {
"type": "ip"
},
"extension": {
"type": "text"
},
"response": {
"type": "text"
},
"geo": {
"coordinates": {
"type": "geo_point"
},
"src": {
"type": "text"
},
"dest": {
"type": "text"
},
"srcdest": {
"type": "text"
}
},
"tags": {
"type": "text"
},
"utc_time": {
"type": "date"
},
"referer": {
"type": "text"
},
"agent": {
"type": "text"
},
"clientip": {
"type": "ip"
},
"bytes": {
"type": "integer"
},
"host": {
"type": "text"
},
"request": {
"type": "text"
},
"url": {
"type": "text"
},
"#message": {
"type": "text"
},
"spaces": {
"type": "text"
},
"xss": {
"type": "text"
},
"links": {
"type": "text"
},
"relatedContent": {
"url": {
"type": "text"
},
"og:type": {
"type": "text"
},
"og:title": {
"type": "text"
},
"og:description": {
"type": ""
},
"og:url": {
"type": ""
},
"article:published_time": {
"type": "date"
},
"article:modified_time": {
"type": "date"
},
"article:section": {
"type": "keyword"
},
"article:tag": {
"type": "text"
},
"og:image": {
"type": "text"
},
"og:image:height": {
"type": "integer"
},
"og:image:width": {
"type": "integer"
},
"og:site_name": {
"type": "text"
},
"twitter:title": {
"type": "text"
},
"twitter:description": {
"type": "text"
},
"twitter:card": {
"type": "keyword"
},
"twitter:image": {
"type": "text"
},
"twitter:site": {
"type": "keyword"
}
},
"machine": {
"os": {
"type": "text"
},
"ram": {
"type": "integer"
}
},
"#version": {
"type": "integer"
}
}
}
}
}
But I don't know why, Kibana don't store the right information into the right field. He just put all the information into a message field. I think it's because I've a dynamic mapping by default, I'm not really sure. Here's the result :
Result (table)
result (json):
{
"_index": "logstash-2019.05.09",
"_type": "doc",
"_id": "9zfam2oBWngGU4Wy3Id5",
"_version": 1,
"_score": null,
"_source": {
"#version": "1",
"#timestamp": "2019-05-09T09:09:32.167Z",
"path": "/home/secunix/logs/TestLogPourMapping_09_05.json",
"message": "{\"#timestamp\":\"2019-05-07T09:56:33.996Z\",\"ip\":\"181.144.250.19\",\"extension\":\"jpg\",\"response\":\"200\",\"geo\":{\"coordinates\":{\"lat\":44.12326,\"lon\":-123.2186856},\"src\":\"IN\",\"dest\":\"CN\",\"srcdest\":\"IN:CN\"},\"#tags\":[\"success\",\"info\"],\"utc_time\":\"2019-05-07T09:56:33.996Z\",\"referer\":\"http://www.slate.com/success/thomas-marshburn\",\"agent\":\"Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.50 Safari/534.24\",\"clientip\":\"181.144.250.19\",\"bytes\":2553,\"host\":\"media-for-the-masses.theacademyofperformingartsandscience.org\",\"request\":\"/uploads/fyodor-yurchikhin.jpg\",\"url\":\"https://media-for-the-masses.theacademyofperformingartsandscience.org/uploads/fyodor-yurchikhin.jpg\",\"#message\":\"181.144.250.19 - - [2019-05-07T09:56:33.996Z] \\\"GET /uploads/fyodor-yurchikhin.jpg HTTP/1.1\\\" 200 2553 \\\"-\\\" \\\"Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.50 Safari/534.24\\\"\",\"spaces\":\"this is a thing with lots of spaces wwwwoooooo\",\"xss\":\"<script>console.log(\\\"xss\\\")</script>\",\"headings\":[\"<h3>ulrich-walter</h5>\",\"http://www.slate.com/success/susan-still-kilrain\"],\"links\":[\"viktor-m-afanasyev#twitter.com\",\"http://twitter.com/security/stephen-oswald\",\"www.twitter.com\"],\"relatedContent\":[],\"machine\":{\"os\":\"win xp\",\"ram\":6442450944},\"#version\":\"1\"}\r",
"host": "qvisbcld0051"
},
"fields": {
"#timestamp": [
"2019-05-09T09:09:32.167Z"
]
},
"sort": [
1557392972167
]
}
And that's what I have when I check my mapping:
{
"mapping": {
"doc": {
"dynamic_templates": [
{
"message_field": {
"path_match": "message",
"match_mapping_type": "string",
"mapping": {
"norms": false,
"type": "text"
}
}
},
{
"string_fields": {
"match": "*",
"match_mapping_type": "string",
"mapping": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"norms": false,
"type": "text"
}
}
}
],
"properties": {
"#message": {
"type": "text"
},
"#timestamp": {
"type": "date"
},
"#version": {
"type": "integer"
},
"agent": {
"type": "text"
},
"bytes": {
"type": "integer"
},
"clientip": {
"type": "ip"
},
"extension": {
"type": "text"
},
"geo": {
"properties": {
"coordinates": {
"type": "geo_point"
},
"dest": {
"type": "text"
},
"src": {
"type": "text"
},
"srcdest": {
"type": "text"
}
}
},
"geoip": {
"dynamic": "true",
"properties": {
"ip": {
"type": "ip"
},
"latitude": {
"type": "half_float"
},
"location": {
"type": "geo_point"
},
"longitude": {
"type": "half_float"
}
}
},
"host": {
"type": "text"
},
"ip": {
"type": "ip"
},
"links": {
"type": "text"
},
"machine": {
"properties": {
"os": {
"type": "text"
},
"ram": {
"type": "integer"
}
}
},
"message": {
"type": "text",
"norms": false
},
"path": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"referer": {
"type": "text"
},
"relatedContent": {
"properties": {
"article:modified_time": {
"type": "date"
},
"article:published_time": {
"type": "date"
},
"article:section": {
"type": "keyword"
},
"article:tag": {
"type": "text"
},
"og:description": {
"type": "text"
},
"og:image": {
"type": "text"
},
"og:image:height": {
"type": "integer"
},
"og:image:width": {
"type": "integer"
},
"og:site_name": {
"type": "text"
},
"og:title": {
"type": "text"
},
"og:type": {
"type": "text"
},
"og:url": {
"type": "text"
},
"twitter:card": {
"type": "keyword"
},
"twitter:description": {
"type": "text"
},
"twitter:image": {
"type": "text"
},
"twitter:site": {
"type": "keyword"
},
"twitter:title": {
"type": "text"
},
"url": {
"type": "text"
}
}
},
"request": {
"type": "text"
},
"response": {
"type": "text"
},
"spaces": {
"type": "text"
},
"tags": {
"type": "text"
},
"url": {
"type": "text"
},
"utc_time": {
"type": "date"
},
"xss": {
"type": "text"
}
}
},
"_default_": {
"dynamic_templates": [
{
"message_field": {
"path_match": "message",
"match_mapping_type": "string",
"mapping": {
"norms": false,
"type": "text"
}
}
},
{
"string_fields": {
"match": "*",
"match_mapping_type": "string",
"mapping": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"norms": false,
"type": "text"
}
}
}
],
"properties": {
"#timestamp": {
"type": "date"
},
"#version": {
"type": "keyword"
},
"geoip": {
"dynamic": "true",
"properties": {
"ip": {
"type": "ip"
},
"latitude": {
"type": "half_float"
},
"location": {
"type": "geo_point"
},
"longitude": {
"type": "half_float"
}
}
}
}
}
}
}
I send my data thanks to logstash, so here's the conf of the input:
input {
beats {
port => 5044
tags => "fromBeats"
}
file {
path => [
"/home/secunix/logs/*",
"/tech/*"
]
start_position => "beginning"
sincedb_path => "/dev/null"
}
tcp {
port => 5514
type => "syslog"
tags => "from Syslog-ng"
}
}
filter {
if [type] == "syslog"{
grok {
match => ["message", "<(?<sys_priority>\d+?)>(?<syslog_timestamp>%{CISCOTIMESTAMP})\s(?<logsource>%{URIHOST})(\s(?:(?<application>.*?)(%(?<project>.*?))?))?:(?:\s)?(?<logmessage>.*$)"]
}
if [logmessage] {
mutate {
replace => [ "message", "%{logmessage}" ]
remove_field => [ "logmessage" ]
}
}
if [project] {
mutate {
replace => [ "type", "%{project}" ]
remove_field => [ "project" ]
}
}else if [application] {
mutate {
lowercase => [ "application" ]
}
mutate {
gsub => [ "application", " ", "_" ]
}
mutate {
replace => [ "type", "%{application}" ]
}
}else {
mutate {
replace => [ "type", "uknapp" ]
add_field => { "application" => "uknapp" }
}
}
}
}
and the output:
output {
elasticsearch {
hosts => ["localhost:9200"]
index => "logstash-%{+YYYY.MM.dd}"
#+++ sa Added by scr-sop-af-config-elksandbox. Do not remove this line.
user => "logstash"
#--- sa Added by scr-sop-af-config-elksandbox. Do not remove this line.
#+++ sa Added by scr-sop-af-config-elksandbox. Do not remove this line.
password => "logstash"
#--- sa Added by scr-sop-af-config-elksandbox. Do not remove this line.
}
}
Can someone tell me how I can fix this please ?