I am printing below log messages
{"timestamp":"15-06-2020 22:12:35","level":"INFO","thread":"http-nio-8080-exec-2","mdc":{"Z-Request-Id":"20200615101234-2c078173-66c2-49ce-93ec-40dfab2a7312","destination":"backendorg"},"logger":"com.AbcHandler","message":"host: localhost, port: 9200, index: zindex and protocol: http","context":"ZPlatform"}
{"timestamp":"15-06-2020 22:12:35","level":"INFO","thread":"http-nio-8080-exec-2","mdc":{"Z-Request-Id":"20200615101234-2c078173-66c2-49ce-93ec-40dfab2a7312","destination":"backendorg"},"logger":"com.AbcHandler","message":"batchNumber: 1 and batchSize: 50","context":"ZPlatform"}
Parsing above messages using Multiline codec, Below is my logstash config file
input {
file {
start_position => "end"
sincedb_path => "/tmp/sincedb_file"
codec => multiline {
pattern => "^Spalanzani"
negate => true
what => previous
}
}
}
filter {
if [type] == "app" {
grok {
match => [ "message","%{GREEDYDATA:jsonstring}"]
}
json {
source => "jsonstring"
target => "parsedJson"
remove_field=>["jsonstring"]
}
mutate {
add_field => {
"frontendDateTime" => "%{[parsedJson][timestamp]}"
"logMessage" => "%{[parsedJson][message]}"
}
}
mutate {
remove_field => [ "parsedJson" ]
}
}
}
But what i am seeing all above messages were clubbed together. Don't know why it is happening. It should show me different log message
{
"tags" => [
[0] "multiline"
],
"message" => "{\"timestamp\":\"15-06-2020 22:12:35\",\"level\":\"INFO\",\"thread\":\"http-nio-8080-exec-2\",\"mdc\":{\"Z-Request-Id\":\"20200615101234-2c078173-66c2-49ce-93ec-40dfab2a7312\",\"destination\":\"backendorg\"},\"logger\":\"com.AbcHandler\",\"message\":\"host: localhost, port: 9200, index: zindex and protocol: http\",\"context\":\"ZPlatform\"}\n{\"timestamp\":\"15-06-2020 22:12:35\",\"level\":\"INFO\",\"thread\":\"http-nio-8080-exec-2\",\"mdc\":{\"Z-Request-Id\":\"20200615101234-2c078173-66c2-49ce-93ec-40dfab2a7312\",\"destination\":\"backendorg\"},\"logger\":\"com.AbcHandler\",\"message\":\"batchNumber: 1 and batchSize: 50\",\"context\":\"ZPlatform\"}",
"logMessage" => "search string: ",
"#timestamp" => 2020-06-15T16:42:38.256Z
}
could someone help me.
Related
I want to change a value inside a field of logstash config file.
For my case my logstash config file is like this..
# Read input from filebeat by listening to port 5044 on which filebeat will send the data
input {
beats {
port => "5044"
}
}
filter {
######################################### For Solr ##############################################
if "solr" in [log][file][path] {
grok {
match => {"message" => "%{DATA:timestamp}%{SPACE}%{LOGLEVEL:log-level}%{SPACE}%{GREEDYDATA:log-message}"}
#remove_field => ["message"]
#add_field => {"message" => "%{log-message}"}
}
}
############################################## For Server ##############################################
if "server.log" in [log][file][path] {
grok {
match => {"message" => "%{DATA:timestamp}%{SPACE}%{LOGLEVEL:log-level}%{SPACE}\[%{DATA}\]%{SPACE}\(%{DATA:thread}\)%{SPACE}%{GREEDYDATA:log-message}"}
#match => { "[log][file][path]" => "%{GREEDYDATA}/%{GREEDYDATA:jboss-log}.log"}
#remove_field => ["message"]
#add_field => {"message" => "%{log-message}"}
}
}
############################################## For Mongo ##############################################
else if "mongos" in [log][file][path] or "config" in [log][file][path] or "shard" in [log][file][path] or "metrics_" in [log][file][path]{
grok {
match => {"message" => "%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{GREEDYDATA:log-message}"}
#remove_field => ["message"]
#add_field => {"message" => "%{log-message}"}
}
}
############################################## For mongo.log #####################################################
else if "mongo" in [log][file][path] {
grok {
match => {"message" => "\[%{DATA:timestamp}\]%{SPACE}%{LOGLEVEL:log-level}%{SPACE}\[%{DATA:class}\]%{SPACE}%{GREEDYDATA:log-message}"}
#remove_field => ["message"]
#add_field => {"message" => "%{log-message}"}
}
}
############################################## For Kafka ##############################################
else if "kafka" in [log][file][path] {
grok {
match => {"message" => "\[%{DATA:timestamp}\]%{SPACE}%{LOGLEVEL:log-level}%{SPACE}\[%{DATA:class}\]%{SPACE}%{GREEDYDATA:log-message}"}
#remove_field => ["message"]
#add_field => {"message" => "%{log-message}"}
}
}
############################################## For mongodb_output & mongodb_exception ##############################################
else if "mongodb_exception" in [log][file][path] or "mongodb_output" in [log][file][path]{
grok {
match => {"message" => "\[%{DATA:timestamp}\]%{SPACE}%{LOGLEVEL:log-level}%{SPACE}\[%{DATA:class}\]%{SPACE}%{GREEDYDATA:log-message}"}
#remove_field => ["message"]
#add_field => {"message" => "%{log-message}"}
}
}
############################################## Other Logs ##############################################
else {
grok {
#match => {"message" => "\[%{MONTHDAY:day}%{SPACE}%{MONTH:month}%{SPACE}%{YEAR:year},%{SPACE}%{TIME:time}\]%{SPACE}%{LOGLEVEL:log-level}%{SPACE}\[%{DATA:class}\]\[%{DATA:thread}\]%{SPACE}%{GREEDYDATA:log-message}"}
match => {"message" => "\[%{DATA:timestamp}\]%{SPACE}%{LOGLEVEL:log-level}%{SPACE}\[%{DATA:class}\]\[%{DATA:thread}\]%{SPACE}%{GREEDYDATA:log-message}"}
#remove_field => ["message"]
#add_field => {"message" => "%{log-message}"}
}
}
################################################################
grok {
match => { "[log][file][path]" => ["%{GREEDYDATA}/%{GREEDYDATA:component}.log" , "%{PATH}\\%{GREEDYDATA:component}\_%{GREEDYDATA}.log" ]}
}
if [component] =~ "^server" {
mutate {
rename => { "%{server}" => "renamed_server" }
}
}
}
output {
# sending properly parsed log events to elasticsearch
elasticsearch {
hosts => ["localhost:9200"]
}
}
I am getting a value of component field as server but I want to change the value of component field server to renamed_server.
I have tried the above but I am not getting any output.
Please help me to find out the required solution.
I guess the problem is with this block:
if [component] =~ "^server" {
mutate {
rename => { "%{server}" => "renamed_server" }
}
}
.. since it doesn't do what you desire, i.e.
I want to change the value of component field server to renamed_server.
rename mutate configuration doesn't change values, it renames fields.
If you want to change value, you can use gsub. And since you want to change the exact value, maybe you can get by without the conditional altogether. E.g.:
mutate {
gsub => [
# replace `server` value with `renamed_server` in component field
"component", "^server$", "renamed_server"
]
}
I have modified this field with gsub it also worked as well.
mutate {
gsub => [
"component", "^server$", "renamed_server",
"component", "^[0-9]{3}.*[0-9]{3}.*[0-9]{2}.*[0-9]{2}.*[0-9]{5}.*output$" , "client_output"
]
}
i could see some events are missing while reporting logs to elastic search. Take an example i am sending 5 logs event only 4 or 3 are reporting.
Basically i am using logstash 7.4 to read my log messages and store the information on elastic search 7.4. below is my logstash configuration
input {
file {
type => "web"
path => ["/Users/a0053/Downloads/logs/**/*-web.log"]
start_position => "beginning"
sincedb_path => "/tmp/sincedb_file"
codec => multiline {
pattern => "^(%{MONTHDAY}-%{MONTHNUM}-%{YEAR} %{TIME}) "
negate => true
what => previous
}
}
}
filter {
if [type] == "web" {
grok {
match => [ "message","(?<frontendDateTime>%{MONTHDAY}-%{MONTHNUM}-%{YEAR} %{TIME})%{SPACE}(\[%{DATA:thread}\])?( )?%{LOGLEVEL:level}%{SPACE}%{USERNAME:zhost}%{SPACE}%{JAVAFILE:javaClass} %{USERNAME:orgId} (?<loginId>[\w.+=:-]+#[0-9A-Za-z][0-9A-Za-z-]{0,62}(?:[.](?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*) %{GREEDYDATA:jsonstring}"]
}
json {
source => "jsonstring"
target => "parsedJson"
remove_field=>["jsonstring"]
}
mutate {
add_field => {
"actionType" => "%{[parsedJson][actionType]}"
"errorMessage" => "%{[parsedJson][errorMessage]}"
"actionName" => "%{[parsedJson][actionName]}"
"Payload" => "%{[parsedJson][Payload]}"
"pageInfo" => "%{[parsedJson][pageInfo]}"
"browserInfo" => "%{[parsedJson][browserInfo]}"
"dateTime" => "%{[parsedJson][dateTime]}"
}
}
}
}
output{
if "_grokparsefailure" in [tags]
{
elasticsearch
{
hosts => "localhost:9200"
index => "grokparsefailure-%{+YYYY.MM.dd}"
}
}
else {
elasticsearch
{
hosts => "localhost:9200"
index => "zindex"
}
}
stdout{codec => rubydebug}
}
As keep on new logs are writing to log files, i could see a difference of log counts.
Any suggestions would be appreciated.
In the logstash config file I am trying to just get the XML data to parse.
This is my config file:
input {
file {
path => "/home/elastic-stack/logstash-7.3.2/event-data/telmetry.log"
start_position => "beginning"
type => "sandbox-out"
codec => multiline {
pattern => "^</datastore-contents-xml>"
negate => "true"
what => "next"
}
}
http {
host => "127.0.0.1"
port => 8080
type => "sandbox-out"
}
}
filter {
grok {
match => { "message" => "\[%{USER:host_name} %{IP:ip_address} %{USER:session-id} %{NUMBER:session-id-num}\]"}
}
grok {
match => { "message" => "\Subscription Id \: %{BASE16NUM:subcription-id:int}"}
}
grok {
match => { "message" => "\Event time \: %{TIMESTAMP_ISO8601:event-time}"}
}
grok {
match => {"message" => "\<%{USERNAME:Statistic}\>"}
}
mutate {
remove_field => ["headers", "host_name", "session-id","message"]
}
date {
match => ["timestamp","dd/MMM/yyyy:HH:mm:ss Z"]
}
ruby { code => 'event.set("justXml", event.get("message").match(/.+(<datastore-contents-xml.*)/m)[1])' }
xml {
#remove_namespaces => "true"
#not even the namspace option is working to access the http link
source => "justXml"
target => "xml-content"
#force_array => "false"
xpath => [
"//*[name()='datastore-contents-xml']/*[name()='memory-statistics']/*[name()='memory-statistic'][1]/*[name()='name']/text()" , "name" ,
"//*[name()='datastore-contents-xml']/*[name()='memory-statistics']/*[name()='memory-statistic'][1]/*[name()='total-memory']/text()" , "total-memory",
"//*[name()='datastore-contents-xml']/*[name()='memory-statistics']/*[name()='memory-statistic'][1]/*[name()='used-memory']/text()" , "used-memory",
"//*[name()='datastore-contents-xml']/*[name()='memory-statistics']/*[name()='memory-statistic'][1]/*[name()='free-memory']/text()" , "free-memory" ,
"//*[name()='datastore-contents-xml']/*[name()='memory-statistics']/*[name()='memory-statistic'][1]/*[name()='lowest-memory']/text()" , "lowest-memory" ,
"//*[name()='datastore-contents-xml']/*[name()='memory-statistics']/*[name()='memory-statistic'][1]/*[name()='highest-memory']/text()" , "highest-memory"
]
#logstash is not dectecting any of these xpaths in the config
}
mutate {
convert => {
"total-memory" => "integer"
"used-memory" => "integer"
"free-memory" => "integer"
"lowest-memory" => "integer"
"highest-memory" => "integer"
}
}
}
output {
stdout {
codec => rubydebug
}
file {
path => "%{type}_%{+dd_MM_yyyy}.log"
}
}
Desired output:
{
"ip_address" => "10.10.20.30",
"subcription-id" => 2147483650,
"event-time" => "2019-09-12 13:13:30.290000+00:00",
"host" => "127.0.0.1",
"Statistic" => "memory-statistic",
"type" => "sandbox-out",
"#version" => "1",
"#timestamp" => 2019-09-26T10:03:00.620Z,
"session-id-num" => "35"
"yang-model" => "http://cisco.com/ns/yang/Cisco-IOS-XE-memory-oper"
"name" => "Processor"
"total-memory" => 2238677360
"used-memory" => 340449924
"free-memory" => 1898227436
"lowest-usage" => 1897220640
"highest-usage" => 1264110388
}
ERROR:
[2019-09-27T09:18:55,622][ERROR][logstash.filters.ruby ] Ruby exception occurred: undefined method `match' for nil:NilClass
/home/elastic-stack/logstash-7.3.2/vendor/bundle/jruby/2.5.0/gems/awesome_print-1.7.0/lib/awesome_print/formatters/base_formatter.rb:31: warning: constant ::Fixnum is deprecated
{
"ip_address" => "10.10.20.30",
"subcription-id" => 2147483650,
"session-id-num" => "35",
"tags" => [
[0] "_rubyexception"
],
"Statistic" => "memory-statistic",
"event-time" => "2019-09-12 13:13:30.290000+00:00",
"type" => "sandbox-out",
"#version" => "1",
"host" => "127.0.0.1",
"#timestamp" => 2019-09-27T07:18:54.868Z
By the error I can already know that the problem is with the ruby filter but I do not know how to resolve it.
This data generate by Cisco Telemetry and I am trying to ingest it using Elastic Stack.
The error seems to be that the event has no message field, so you cannot call match on a non existing thing.
I see you are calling match on the message field in this ruby code:
ruby { code => 'event.set("justXml", event.get("message").match(/.+(<datastore-contents-xml.*)/m)[1])' }
However you are removing the message field from the event a few lines earlier:
mutate {
remove_field => ["headers", "host_name", "session-id","message"]
}
Solution is to remove the message field only when you don't need it anymore, I would move the remove_field mutate to the end of the filter block.
One more suggestion if I may add. You have multiple grok filters running on the same, message field:
grok {
match => { "message" => "\[%{USER:host_name} %{IP:ip_address} %{USER:session-id} %{NUMBER:session-id-num}\]"}
}
grok {
match => { "message" => "\Subscription Id \: %{BASE16NUM:subcription-id:int}"}
}
grok {
match => { "message" => "\Event time \: %{TIMESTAMP_ISO8601:event-time}"}
}
grok {
match => {"message" => "\<%{USERNAME:Statistic}\>"}
}
This can be simplified into this (you can check to the Grok filter docs:
grok {
break_on_match => false,
match => {
"message" => [
"\[%{USER:host_name} %{IP:ip_address} %{USER:session-id} %{NUMBER:session-id-num}\]",
"\Subscription Id \: %{BASE16NUM:subcription-id:int}",
"\Event time \: %{TIMESTAMP_ISO8601:event-time}",
"\<%{USERNAME:Statistic}\>"
]
}
}
This way you need only one instance of the grok filter, as it will go through the patterns in the list and because of break_on_match=>false it will not finish after the first successful match, but will make sure to extract all fields it can based on all the patterns in the list.
I have the following logstash conf file
input {
tcp {
port => 12345
codec => json
}
}
filter {
grok {
break_on_match => true
match => [
"message", "%{TIMESTAMP_ISO8601:timestamp} (verbose|info|debug) (hostd|vpxa)",
]
mutate {
add_tag => "esxi_verbose"
}
}
}
if "esxi_verbose" in [tags] {
drop{}
}
output {
stdout { codec => rubydebug }
elasticsearch {
hosts => ["localhost:9200"]
index => "logstash-%{+YYYY.MM.dd}"
}
}
I am trying to drop any verbose, debug, info messages. When I start logstash I get the error
[2019-03-03T16:53:11,731][ERROR][logstash.agent] Failed to execute action {:action=>LogStash::PipelineAction::Create/pipeline_id:main, :exception=>"LogStash::ConfigurationError", :message=>"Expected one of #, \", ', -, [, { at line 13, column 5 (byte 211) after filter {\n grok {\n break_on_match => true\n match => [\n \"message\", \"%{TIMESTAMP_ISO8601:timestamp} (verbose|info|debug) (hostd|vpxa)\",\n "
Can someone help me what I am doing wrong.
you have 3 issues in the config:
there's a comma at the end of the grok message line which is
redundant
the mutate is inside the grok filter, but it should come
after it
the 'if' statement should be inside the 'filter' section.
This is the updated and working config:
input {
tcp {
port => 12345
codec => json
}
}
filter {
grok {
break_on_match => true
match => [
"message", "%{TIMESTAMP_ISO8601:timestamp} (verbose|info|debug) (hostd|vpxa)"
]
}
mutate {
add_tag => "esxi_verbose"
}
if "esxi_verbose" in [tags] {
drop{}
}
}
output {
stdout { codec => rubydebug }
elasticsearch {
hosts => ["localhost:9200"]
index => "logstash-%{+YYYY.MM.dd}"
}
}
I have the following configuration for logstash.
There are 3 parts to this one is a generallog which we use for all applications they land in here.
second part is the application stats where in which we have a specific logger which will be configured to push the application statistics
third we have is the click stats when ever an event occurs on client side we may want to push it to the logstash on the upd address.
all 3 are udp based, we also use log4net to to send the logs to the logstash.
the base install did not have a GeoIP.dat file so got the file downloaded from the https://dev.maxmind.com/geoip/legacy/geolite/
have put the file in the /opt/logstash/GeoIPDataFile with a 777 permissions on the file and folder.
second thing is i have a country name and i need a way to show how many users form each country are viewing the application in last 24 hours.
so for that reason we also capture the country name as its in their profile in the application.
now i need a way to get the geo co-ordinates to use the tilemap in kibana.
What am i doing wrong.
if i take the geoIP { source -=> "country" section the logstash works fine.
when i check the
/opt/logstash/bin/logstash -t -f /etc/logstash/conf.d/logstash.conf
The configuration file is ok is what i receive. where am i going worng?
Any help would be great.
input {
udp {
port => 5001
type => generallog
}
udp {
port => 5003
type => applicationstats
}
udp {
port => 5002
type => clickstats
}
}
filter {
if [type] == "generallog" {
grok {
remove_field => message
match => { message => "(?m)%{TIMESTAMP_ISO8601:sourcetimestamp} \[%{NUMBER:threadid}\] %{LOGLEVEL:loglevel} +- %{IPORHOST:requesthost} - %{WORD:applicationname} - %{WORD:envname} - %{GREEDYDATA:logmessage}" }
}
if !("_grokparsefailure" in [tags]) {
mutate {
replace => [ "message" , "%{logmessage}" ]
replace => [ "host" , "%{requesthost}" ]
add_tag => "generalLog"
}
}
}
if [type] == "applicationstats" {
grok {
remove_field => message
match => { message => "(?m)%{TIMESTAMP_ISO8601:sourceTimestamp} \[%{NUMBER:threadid}\] %{LOGLEVEL:loglevel} - %{WORD:envName}\|%{IPORHOST:actualHostMachine}\|%{WORD:applicationName}\|%{NUMBER:empId}\|%{WORD:regionCode}\|%{DATA:country}\|%{DATA:applicationName}\|%{NUMBER:staffapplicationId}\|%{WORD:applicationEvent}" }
}
geoip {
source => "country"
target => "geoip"
database => "/opt/logstash/GeoIPDataFile/GeoIP.dat"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
}
mutate {
convert => [ "[geoip][coordinates]", "float"]
}
if !("_grokparsefailure" in [tags]) {
mutate {
add_tag => "applicationstats"
add_tag => [ "eventFor_%{applicationName}" ]
}
}
}
if [type] == "clickstats" {
grok {
remove_field => message
match => { message => "(?m)%{TIMESTAMP_ISO8601:sourceTimestamp} \[%{NUMBER:threadid}\] %{LOGLEVEL:loglevel} - %{IPORHOST:remoteIP}\|%{IPORHOST:fqdnHost}\|%{IPORHOST:actualHostMachine}\|%{WORD:applicationName}\|%{WORD:envName}\|(%{NUMBER:clickId})?\|(%{DATA:clickName})?\|%{DATA:clickEvent}\|%{WORD:domainName}\\%{WORD:userName}" }
}
if !("_grokparsefailure" in [tags]) {
mutate {
add_tag => "clicksStats"
add_tag => [ "eventFor_%{clickName}" ]
}
}
}
}
output {
if [type] == "applicationstats" {
elasticsearch {
hosts => "localhost:9200"
index => "applicationstats-%{+YYYY-MM-dd}"
template => "/opt/logstash/templates/udp-applicationstats.json"
template_name => "applicationstats"
template_overwrite => true
}
}
else if [type] == "clickstats" {
elasticsearch {
hosts => "localhost:9200"
index => "clickstats-%{+YYYY-MM-dd}"
template => "/opt/logstash/templates/udp-clickstats.json"
template_name => "clickstats"
template_overwrite => true
}
}
else if [type] == "generallog" {
elasticsearch {
hosts => "localhost:9200"
index => "generallog-%{+YYYY-MM-dd}"
template => "/opt/logstash/templates/udp-generallog.json"
template_name => "generallog"
template_overwrite => true
}
}
else{
elasticsearch {
hosts => "localhost:9200"
index => "logstash-%{+YYYY-MM-dd}"
}
}
}
As per the error message, the mutation which you're trying to do could be wrong. Could you please change your mutate as below:
mutate {
convert => { "geoip" => "float" }
convert => { "coordinates" => "float" }
}
I guess you've given the mutate as an array, and it's a hash type by origin. Try converting both the values individually. Your database path for geoip seems to be fine in your filter. Is that the whole error which you've mentioned in the question? If not update the question with the whole error if possible.
Refer here, for in depth explanations.