Logstash code to upload CSV file content into Elasticsearch - elasticsearch
I have CSV file and I want to import it into Elasticsearch 5.0.0 using Logstash.
This are first 2 lines of the CSV file:
Id,CameraId,ZoneId,Latitude,Longitude,Number,OriginalNumber,Country,OriginalCountry,CapturedOn,Status,Direction,Speed,Confidence,AvgDigitsHeight,MultiplateRate,ProcessingTimeOCR,Signaled,OcrImageId,EnvImageIds,CapturerId,CapturerType,IsAlarm,AlarmListIds,ReplicationId,ImagesUploaded
111,24,1,42.8,3.5,XXDFR,XXDFR,DE,DE,2017-03-04 12:06:20.0,0,1,0,99,21.0,3,16.0193003809306,0,0,[],null,null,0,[],0,0
I run this Logstash script:
input {
file {
path => ["/usr/develop/test.csv"]
type => "core2"
start_position => "beginning"
}
}
filter {
csv {
columns => [
"Id","CameraId","ZoneId","Latitude","Longitude,"Number","OriginalNumber","Country","OriginalCountry","CapturedOn","Status","Direction","Speed","Confidence","AvgDigitsHeight","MultiplateRate","ProcessingTimeOCR","Signaled","OcrImageId","EnvImageIds","CapturerId","CapturerType","IsAlarm","AlarmListIds","ReplicationId","ImagesUploaded"
]
}
}
output {
stdout { codec => rubydebug }
elasticsearch {
action => "index"
hosts => ["127.0.0.1:9200"]
index => "mytest"
document_type => "doc"
document_id => "%{Id}"
workers => 1
}
}
I get this error:
logstash.agent ] fetched an invalid config {:config=>"input
{\nfile {\npath => [\"/usr/develop/test.csv\"]\ntype =>
\"core2\"\nstart_position => \"beginning\" \n}\n}\nfilter {\ncsv
{\nseparator => \",\"\ncolumns =>
[\"Id\",\"CameraId\",\"ZoneId\",\"Latitude\",\"Longitude,\"Number\",\"OriginalNumber\",\"Country\",\"OriginalCountry\",\"CapturedOn\"]\n}\n}\noutput
{\nelasticsearch {\naction => \"index\"\nhosts =>
[\"localhost:9200\"]\nindex => \"test\"\ndocument_type =>
\"doc\"\ndocument_id => \"%{Id}\"\nworkers => 1\n}\nstdout { codec =>
rubydebug}\n}\n\n", :reason=>"Expected one of #, {, ,, ] at line 11,
column 61 (byte 225) after filter {\ncsv {\nseparator =>
\",\"\ncolumns =>
[\"Id\",\"CameraId\",\"ZoneId\",\"Latitude\",\"Longitude,\""}
Not sure if you caught this yet, but it's because you are missing a " for the column name "Longitude"
Related
Create index based on message field - appname
****Logstash.conf code *******
input {
stdin{
type => "stdin-type"
}
file{
type => "json"
path => [ "C:/prod/*.log", "C:/prod/*/**.log"]
start_position => "beginning"
tags => "prod"
}
file{
type => "json"
path => [ "C:/dev/*.log", "C:/dev/*/**.log"]
start_position => "beginning"
tags => "dev"
}
}
filter {
grok {
match => {
"message" => [ "%{JSON:payload_raw} "]
}
pattern_definitions => {
"JSON" => "{.*$"}
}
json {
source => "payload_raw"
target => "payload"
}
mutate {
remove_field => [ "payload_raw","message" ]
}
date {
match => [ "[payload][datetime]", "yyyy-MM-dd HH:mm:ss,SSS" ]
target => "#timestamp"
}
}
output {
stdout {
codec => rubydebug
}
elasticsearch {
hosts => ["localhost:9200"]
index => "%{tags}-logs"
}
}
Sample log
{datetime":"2021-08-10 04:11:37,825","servername":"VM-0001","serverip":"(null)","process":"2404","thread":"4","level":"DEBUG","appname":"Dev-Email","page":"Program.cs"}
Given the sample document your shared, your elasticsearch output needs to look like this:
elasticsearch {
hosts => ["localhost:9200"]
index => "%{appname}-logs"
}
Also know that index names are not allowed to contain uppercase letters, so Dev-Email will need to be lowercased (using the mutate/lowercase filter) before being used as the index name.
logstash don't report all the events
i could see some events are missing while reporting logs to elastic search. Take an example i am sending 5 logs event only 4 or 3 are reporting.
Basically i am using logstash 7.4 to read my log messages and store the information on elastic search 7.4. below is my logstash configuration
input {
file {
type => "web"
path => ["/Users/a0053/Downloads/logs/**/*-web.log"]
start_position => "beginning"
sincedb_path => "/tmp/sincedb_file"
codec => multiline {
pattern => "^(%{MONTHDAY}-%{MONTHNUM}-%{YEAR} %{TIME}) "
negate => true
what => previous
}
}
}
filter {
if [type] == "web" {
grok {
match => [ "message","(?<frontendDateTime>%{MONTHDAY}-%{MONTHNUM}-%{YEAR} %{TIME})%{SPACE}(\[%{DATA:thread}\])?( )?%{LOGLEVEL:level}%{SPACE}%{USERNAME:zhost}%{SPACE}%{JAVAFILE:javaClass} %{USERNAME:orgId} (?<loginId>[\w.+=:-]+#[0-9A-Za-z][0-9A-Za-z-]{0,62}(?:[.](?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*) %{GREEDYDATA:jsonstring}"]
}
json {
source => "jsonstring"
target => "parsedJson"
remove_field=>["jsonstring"]
}
mutate {
add_field => {
"actionType" => "%{[parsedJson][actionType]}"
"errorMessage" => "%{[parsedJson][errorMessage]}"
"actionName" => "%{[parsedJson][actionName]}"
"Payload" => "%{[parsedJson][Payload]}"
"pageInfo" => "%{[parsedJson][pageInfo]}"
"browserInfo" => "%{[parsedJson][browserInfo]}"
"dateTime" => "%{[parsedJson][dateTime]}"
}
}
}
}
output{
if "_grokparsefailure" in [tags]
{
elasticsearch
{
hosts => "localhost:9200"
index => "grokparsefailure-%{+YYYY.MM.dd}"
}
}
else {
elasticsearch
{
hosts => "localhost:9200"
index => "zindex"
}
}
stdout{codec => rubydebug}
}
As keep on new logs are writing to log files, i could see a difference of log counts.
Any suggestions would be appreciated.
Loading a number of xml file into logstash
i want to load a number of xml file into logstash in the same time, so what i should to add in my config file!.
Thanks guys for your support :)
this is my config file :
input {
file {
path => "D:/test*.xml",
start_position => beginning
sincedb_path => "NUL"
codec => multiline {
pattern => "<invoicing>|</invoicing>"
negate => "true"
what => "previous"
auto_flush_interval => 1
max_lines => 3000
}
}
}
filter {
xml {
source => "message"
target => "message.parsed"
store_xml => false
force_array => false
}
}
output {
stdout { codec => rubydebug }
elasticsearch {
index => "tizer005"
hosts => ["localhost:9200"]
document_type => "ChannelFiles"
}
}
Parsing XML file using Logstash
I am trying to parse an XML file in Logstash. I want to use XPath to do the parsing of documents in XML. So when I run my config file the data loads into elasticsearch but It is not in the way I want to load the data. The data loaded in elasticsearch is each line in xml document
Structure of my XML file
What I want to achieve:
create fields in elasticsearch that stores the follwing
ID =1
Name = "Finch"
My Config file:
input{
file{
path => "C:\Users\186181152\Downloads\stations.xml"
start_position => "beginning"
sincedb_path => "/dev/null"
exclude => "*.gz"
type => "xml"
}
}
filter{
xml{
source => "message"
store_xml => false
target => "stations"
xpath => [
"/stations/station/id/text()", "station_id",
"/stations/station/name/text()", "station_name"
]
}
}
output{
elasticsearch{
codec => json
hosts => "localhost"
index => "xmlns"
}
stdout{
codec => rubydebug
}
}
Output in Logstash:
{
"station_name" => "%{station_name}",
"path" => "C:\Users\186181152\Downloads\stations.xml",
"#timestamp" => 2018-02-09T04:03:12.908Z,
"station_id" => "%{station_id}",
"#version" => "1",
"host" => "BW",
"message" => "\t\r",
"type" => "xml"
}
The multiline filter allows to create xml file as a single event and we can use xml-filter or xpath to parse the xml to ingest data in elasticsearch.
In the multiline filter, we mention a pattern( in below example) that is used by logstash to scan your xml file. Once the pattern matches all the entries after that will be considered as a single event.
The following is an example of working config file for my data
input {
file {
path => "C:\Users\186181152\Downloads\stations3.xml"
start_position => "beginning"
sincedb_path => "/dev/null"
exclude => "*.gz"
type => "xml"
codec => multiline {
pattern => "<stations>"
negate => "true"
what => "previous"
}
}
}
filter {
xml {
source => "message"
store_xml => false
target => "stations"
xpath => [
"/stations/station/id/text()", "station_id",
"/stations/station/name/text()", "station_name"
]
}
}
output {
elasticsearch {
codec => json
hosts => "localhost"
index => "xmlns24"
}
stdout {
codec => rubydebug
}
}
logstash elasticsearch ouput plugin script example to add value to array filed?
Hello I am getting this error when I try to add value to existing array field in elasticseach, and my logstash output configuration is:
elasticsearch {
document_id => 1
action => "update"
hosts => ["X.X.X.X:9200"]
index => "test"
script_lang => "painless"
script_type => "inline"
script => 'ctx._source.arrat.add(event("[file][fuid]"))'
}
The error i was getting is
error"=>{"type"=>"illegal_argument_exception", "reason"=>"failed to execute script", "caused_by"=>{"type"=>"script_exception", "reason"=>"compile error", "script_stack"=>["ctx._source.arrat.add(event(\"[file][fuid]\"))", " ^---- HERE"], "script"=>"ctx._source.arrat.add(event(\"[file][fuid]\"))", "lang"=>"painless", "caused_by"=>{"type"=>"illegal_argument_exception", "reason"=>"Unknown call [event] with [1] arguments."}}}}}}.
Below is the logstash configuration
input {
beats {
port => "12109"
}
}
filter {
mutate {
id => "brolog-files-rename-raw-fields"
rename => { "[ts]" => "[file][ts]"
"[fuid]" => "[file][fuid]"
"[tx_hosts]" => "[file][tx_hosts]"
"[rx_hosts]" => "[file][rx_hosts]"
"[conn_uids]" => "[file][conn_uids]"
"[source]" => "[file][source]"
"[depth]" => "[file][depth]"
"[analyzers]" => "[file][analyzers]"
"[mime_type]" => "[file][mime_type]"
"[duration]" => "[file][duration]"
"[is_orig]" => "[file][is_orig]"
"[seen_bytes]" => "[file][seen_bytes]"
"[missing_bytes]" => "[file][missing_bytes]"
"[overflow_bytes]" => "[file][overflow_bytes]"
"[timedout]" => "[file][timedout]"
"[md5]" => "[file][md5]"
"[sha1]" => "[file][sha1]"
}
}
}
output{
stdout { codec => rubydebug}
elasticsearch {
document_id => 1
action => "update"
doc_as_upsert => "true"
hosts => ["X.X.X.X:9200"]
index => "test"
script_lang => "painless"
script_type => "inline"
script => 'ctx._source.arrat.add(event.[file][fuid])'
}
}
i am getting data in json format.