How to create field using Logstash and grok plugin - elasticsearch

Question

I have a tomcat log of below format
10.0.6.35 - - [21/Oct/2019:00:00:04 +0000] "GET /rest/V1/productlist/category/4259/ar/final_price/asc/4/20 HTTP/1.1" 200 14970 12
I want to create the field of last two column which is bytes and duration and want to analyze it using Kibana. I had used Filebeat and Logstash for transferring data to the Elasticsearch.
My Logstash configuration file is below:
I had tried with below configuration but not able to see the field on kibana.
input {
beats {
port => 5044
}
}
filter {
grok {
match => ["message" => "%{IP:client} %{WORD:method} %{URIPATHPARAM:request} %{NUMBER:bytes}(?m) %{NUMBER:duration}" ]
#match=>{"duration"=> "%{NUMBER:duration}"}
# match => { "message" => "%{COMBINEDAPACHELOG}" }
}
# mutate {
# remove_field => ["#version", "#timestamp"]
# }
date {
match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ]
}
}
output {
if [fields][log_type] == "access-log"
{
elasticsearch {
hosts => ["172.31.30.73:9200"]
index => "%{[fields][service]}-%{+YYYY.MM.dd}"
}
}
if [fields][log_type] == "application-log"
{
elasticsearch {
hosts => ["172.31.30.73:9200"]
index => "%{[fields][service]}-%{+YYYY.MM.dd}"
}
}
else
{
elasticsearch {
hosts => ["172.31.30.73:9200"]
index => "logstashhh-%{+YYYY.MM.dd}"
}
I want that duration and bytes becomes my field on Kibana for visualization.

Answer 1

Try this as your logstash configuration:
input {
beats {
port => 5044
}
}
filter {
grok {
match => ["message" => "%{NUMBER:bytes}(?m) %{NUMBER:duration}$" ]
#match=>{"duration"=> "%{NUMBER:duration}"}
# match => { "message" => "%{COMBINEDAPACHELOG}" }
}
# mutate {
# remove_field => ["#version", "#timestamp"]
# }
date {
match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ]
}
}
output {
if [fields][log_type] == "access-log"
{
elasticsearch {
hosts => ["172.31.30.73:9200"]
index => "%{[fields][service]}-%{+YYYY.MM.dd}"
}
}
if [fields][log_type] == "application-log"
{
elasticsearch {
hosts => ["172.31.30.73:9200"]
index => "%{[fields][service]}-%{+YYYY.MM.dd}"
}
}
else
{
elasticsearch {
hosts => ["172.31.30.73:9200"]
index => "logstashhh-%{+YYYY.MM.dd}"
}