Test logstash with elasticsearch as input and output - elasticsearch

I have configured logstash with Elasticsearch as input and output paramaters as below :
input
{
elasticsearch {
hosts => ["hostname" ]
index => 'indexname'
type => 'type'
user => 'username'
password => 'password'
docinfo => true
query => '{ "query": { "match": { "first_name": "mary" } }}'
}
}
output
{
elasticsearch {
hosts => ["hostname" ]
index => 'indexname'
user => 'username'
password => 'password'
}
}
My indexed data is as below :
PUT person/person/3
{
"first_name" : "mary"
}
PUT person/person/4
{
"first_name" : "mary.m"
}
PUT person/person/5
{
"first_name" : "mary.k"
}
When I run below query on ES
GET indexname/_search
{
"query": {
"match": {
"first_name": "mary"
}
}
}
it returns
{
"took": 1,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"failed": 0
},
"hits": {
"total": 1,
"max_score": 0.2876821,
"hits": [
{
"_index": "person",
"_type": "person",
"_id": "3",
"_score": 0.2876821,
"_source": {
"first_name": "mary"
}
}
]
}
}
Although logstash pipeline has started successfully it does not log this query in ES as I had used query as "match": { "first_name": "mary"} in input section.

Since your ES runs on HTTPS, you need to add ssl => true to your elasticsearch input configuration
input {
elasticsearch {
hosts => ["hostname" ]
index => 'indexname'
type => 'type'
user => 'username'
password => 'password'
docinfo => true
ssl => true <--- add this
query => '{ "query": { "match": { "first_name": "mary" } }}'
}
}

Related

I have implemented the kafka with logstash input and elasticsearch output. its working fine in kibana.. I want to filter the data based on statuscode

This is kibana dashboard json Data.. Here i have to filter the based on response statuscode with in the message json data field..
{
"_index": "rand-topic",
"_type": "_doc",
"_id": "ulF8uH0BK9MbBSR7DPEw",
"_version": 1,
"_score": null,
"fields": {
"#timestamp": [
"2021-12-14T10:27:56.956Z"
],
"#version": [
"1"
],
"#version.keyword": [
"1"
],
"message": [
"{\"requestMethod\":\"GET\",\"headers\":{\"content-type\":\"application/json\",\"user-agent\":\"PostmanRuntime/7.28.4\",\"accept\":\"*/*\",\"postman-token\":\"977fc94b-38c8-4df4-ad73-814871a32eca\",\"host\":\"localhost:5600\",\"accept-encoding\":\"gzip, deflate, br\",\"connection\":\"keep-alive\",\"content-length\":\"44\"},\"body\":{\"category\":\"CAT\",\"noise\":\"purr\"},\"query\":{},\"requestUrl\":\"http://localhost:5600/kafka\",\"protocol\":\"HTTP/1.1\",\"remoteIp\":\"1\",\"requestSize\":302,\"userAgent\":\"PostmanRuntime/7.28.4\",\"statusCode\":200,\"response\":{\"success\":true,\"message\":\"Kafka Details are added\",\"data\":{\"kafkaData\":{\"_id\":\"61b871ac69be37078a9c1a79\",\"category\":\"DOG\",\"noise\":\"bark\",\"__v\":0},\"postData\":{\"category\":\"DOG\",\"noise\":\"bark\"}}},\"latency\":{\"seconds\":0,\"nanos\":61000000},\"responseSize\":193}"]},"sort[1639477676956]}
Expected output like this Here added the statuscode field from message field
{
"_index": "rand-topic",
"_type": "_doc",
"_id": "ulF8uH0BK9MbBSR7DPEw",
"_version": 1,
"_score": null,
"fields": {
"#timestamp": [
"2021-12-14T10:27:56.956Z"
],
"#version": [
"1"
],
"#version.keyword": [
"1"
],
"statusCode": [
200
],
"message": [
"{\"requestMethod\":\"GET\",\"headers\":{\"content-
type\":\"application/json\",\"user-
agent\":\"PostmanRuntime/7.28.4\",\"accept\":\"*/*\",\"postman-
token\":\"977fc94b-38c8-4df4-ad73-
814871a32eca\",\"host\":\"localhost:5600\",\"accept-
encoding\":\"gzip, deflate, br\",\"connection\":\"keep-
alive\",\"content-length\":\"44\"},\"body\":
{\"category\":\"CAT\",\"noise\":\"purr\"},\"query\": {}, \"requestUrl\":\"http://localhost:5600/kafka\",\"protocol\":\"HTTP/1.1\",\"remoteIp\":\"1\",\"requestSize\":302,\"userAgent\":\"PostmanRuntime/7.28.4\",\"statusCode\":200,\"response\":{\"success\":true,\"message\":\"Kafka Details are added\",\"data\":{\"kafkaData\":{\"_id\":\"61b871ac69be37078a9c1a79\",\"category\":\"DOG\",\"noise\":\"bark\",\"__v\":0},\"postData\":{\"category\":\"DOG\",\"noise\":\"bark\"}}},\"latency\":{\"seconds\":0,\"nanos\":61000000},\"responseSize\":193}"
]},"sort": [1639477676956]}
Please help me how to configure logstash filter for statusCode
input {
kafka {
topics => ["randtopic"]
bootstrap_servers => "192.168.29.138:9092"
}
}
filter{
mutate {
add_field => {
"statusCode" => "%{[status]}"
}
}
}
output {
elasticsearch {
hosts => ["192.168.29.138:9200"]
index => "rand-topic"
workers => 1
}
}
output {
if [message][0][statusCode] == "200" {
Do Somethings ....
stdout { codec => ""}
}
}

How to get distance from elasticsearch.net / NEST for a geo_point field

I would like to get in my search request the distance for a geo_point..
I already write this request that gives me closest point to my searching param.
ConnectionSettings elasticSettings = new ConnectionSettings(new Uri("http://localhost:9200"));
ElasticClient client = new ElasticClient(elasticSettings);
var searchResults = client.Search<dynamic>(s => s.Index("index1,index2,index3").From(0).Size(10).Query(
q => q.Bool(
b => b.Must(
f => f.GeoDistance(
g => g.Distance(20, DistanceUnit.Kilometers).DistanceType(GeoDistanceType.Arc).Field("geo").Location(lat, lon))))));
I tried a lot of code found on the web but I can not adapt it for my code..
I just want that elasticsearch return me the distance foreach point.
My field in elasticsearch is like that (simple string):
geo 74.875,-179.875
and in another index test, is like that (structured) : the search doesn't works like this
geo {
"lat": 74.875,
"lon": -178.625
}
Is the first or second mapping can have an impact on the query ?
Here is my mapping for the index :
{
"index1": {
"aliases": {},
"mappings": {
"properties": {
"Date": { "type": "date" },
"Value": { "type": "text" },
"geo": { "type": "geo_point" }
}
},
"settings": {
"index": {
"refresh_interval": "1s",
"number_of_shards": "4",
"provided_name": "index1",
"creation_date": "1569420798736",
"number_of_replicas": "0",
"uuid": "jqc1RRhxSC2e5yJJX2lyzw",
"version": { "created": "7030199" }
}
}
}
}
I integrate a scripfield in my query like that :
var searchResults = client.Search<dynamic>(s => s.Index("index").From(0).Size(100).ScriptFields(sf => sf.ScriptField("distance", d => d.Source("if(doc['geo'].size(){doc['geo'].arcDistance("+ lat+","+ lon + ")}"))).Query(
q => q.Bool(
b => b.Must(
f => f.GeoDistance(
g => g.Distance(20, DistanceUnit.Kilometers).DistanceType(GeoDistanceType.Arc).Field("geo").Location(lat, lon))))));
With this request, I have a "200 successfull" responses and it seems that I it returns me the distance but not the other field, and the 100 documents are null.
Valid NEST response built from a successful (200) low level call on
POST: /index1/_search?typed_keys=true
# Audit trail of this API call:
- [1] HealthyResponse: Node: http://localhost:9200/ Took:
00:00:01.0670113
# Request:
{"from":0,"query":{"bool":{"must":[{"geo_distance":
{"distance":"200km","distance_type":"arc","geo":
{"lat":57.123,"lon":-20.876}}}]}},"script_fields":{"distance":{"script":
{"source":"doc['geo'].arcDistance(57.123,-20.876)"}}},"size":100}
# Response:
{
"took": 3,
"timed_out": false,
"_shards": {
"total": 4,
"successful": 4,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 1203,
"relation": "eq"
},
"max_score": 1.0,
"hits": [
{
"_index": "index1",
"_type": "_doc",
"_id": "121197",
"_score": 1.0,
"fields": { "distance": [ 198251.11868760435 ] }
},
{
"_index": "index1",
"_type": "_doc",
"_id": "121198",
"_score": 1.0,
"fields": { "distance": [ 197018.831847128 ] }
},
...98 more
]
}
}
Thank you.
You need to use script field to return distance
"script_fields":{
"distance":{
"script":"doc['latlng'].arcDistance(params.lat,params.lng)",
"params":{
"lat":<some value>,
"lng":<some value>
}
}
}
Nest
var scriptFields = new ScriptFields
{
{
"distance", new ScriptField {
Script = new InlineScript( "if(doc['"+field+"'].size() > 0) { doc['"+field+"'].arcDistance(params.lat,params.lon) }")
{
Params=new FluentDictionary<string, object>
{
{ "lat", latitude},
{ "lon", longitude}
}
}
}
}
};

elasticsearch 5.x : how make a nest match query search

In the previous version of Nest , i knew how to do the equivalent of a basic es match query with nest:
I created an exemple index and mapping
PUT /base_well
{
"mappings": {
"person": {
"properties": {
"first_name":{
"type": "string"
},
"last_name":{
"type": "string"
},
"age":{
"type": "integer"
}
}
}
}
}
POST /base_well/person
{
"first_name":"Adrien",
"last_name" : "Mopo",
"Age" : 21
}
POST /base_well/person
{
"first_name":"Polo",
"last_name" : "Apou",
"Age" : 36
}
ES request works actually
POST /base_well/person/_search
{
"query":
{
"match":{
"first_name":"Adrien"
}
}
}
this Elasticsearch request give me this answere:
{
"took": 3,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"failed": 0
},
"hits": {
"total": 1,
"max_score": 0.2876821,
"hits": [
{
"_index": "base_well",
"_type": "person",
"_id": "AVkq9PI5ybdSs0epy_Rb",
"_score": 0.2876821,
"_source": {
"first_name": "Adrien",
"last_name": "Mopo",
"Age": 21
}
}
]
}
}
NEST equivalent that does not work anymore:
public class Person
{
public string first_name {get;set;}
public string last_name { get; set; }
public int Age { get; set; }
}
//nest equivalent does not work anymore
var uri = new Uri("http://localhost:9200");
var setting = new ConnectionSettings(uri);
setting.DisableDirectStreaming(true);
setting.DefaultIndex("base_well");
var Client = new ElasticClient(setting);
var response = Client.Search<Person>(s => s.Query(p => p.Term(q => q.first_name, "Adrien")));
var tooks = response.Took;
var hits = response.Hits;
var total = response.Total;
It gives me 0 documents results , 0 hits
Do you know how to do that in the last version?
var response = Client.Search<Person>(s => s.Query(p => p.Match(m => m.Field(f => f.first_name).Query("Marc"))));

How do I replicate the _id and _type of elasticsearch index when dumping data through Logstash

I have an "Index":samcorp with "type":"sam".
One of them looks like the below :
{
"_index": "samcorp",
"_type": "sam",
"_id": "1236",
"_version": 1,
"_score": 1,
"_source": {
"name": "Sam Smith",
"age": 22,
"confirmed": true,
"join_date": "2014-06-01"
}
}
I want to replicate the same data into a different "index" name "jamcorp" with the same "type" and same "id"
I am using Logstash to do it:
I use the below code in the configuration file of logstash I end up having wrong ids and type
input {
elasticsearch {
hosts => ["127.0.0.1:9200"]
index => "samcorp"
}
}
filter {
mutate {
remove_field => [ "#version", "#timestamp" ]
}
}
output {
elasticsearch {
hosts => ["127.0.0.1:9200"]
manage_template => false
index => "jamcorp"
document_type => "%{_type}"
document_id => "%{_id}"
}
}
I've tried all possible combinations, I gt the following output:
Output:
{
"_index": "jamcorp",
"_type": "%{_type}",
"_id": "%{_id}",
"_version": 4,
"_score": 1,
"_source": {
"name": "Sam Smith",
"age": 22,
"confirmed": true,
"join_date": "2014-06-01"
}
}
The Ouptut I require is:
{
"_index": "jamcorp",
"_type": "sam",
"_id": "1236",
"_version": 4,
"_score": 1,
"_source": {
"name": "Sam Smith",
"age": 22,
"confirmed": true,
"join_date": "2014-06-01"
}
}
Any help would be appreciated. :) Thanks
In your elasticsearch input, you need to set the docinfo parameter to true
input {
elasticsearch {
hosts => ["127.0.0.1:9200"]
index => "samcorp"
docinfo => true <--- add this
}
}
As a result the #metadata hash will be populated with the index, _type and _id of the document and you can reuse that in your filters and outputs:
output {
elasticsearch {
hosts => ["127.0.0.1:9200"]
manage_template => false
index => "jamcorp"
document_type => "%{[#metadata][_type]}" <--- use #metadata
document_id => "%{[#metadata][_id]}" <--- use #metadata
}
}

How to extract feature from the Elasticsearch _source to index

I have used logstash,elasticsearch and kibana to collect logs.
The log file is json which like this:
{"_id":{"$oid":"5540afc2cec7c68fc1248d78"},"agentId":"0000000BAB39A520","handler":"SUSIControl","sensorId":"/GPIO/GPIO00/Level","ts":{"$date":"2015-04-29T09:00:00.846Z"},"vHour":1}
{"_id":{"$oid":"5540afc2cec7c68fc1248d79"},"agentId":"0000000BAB39A520","handler":"SUSIControl","sensorId":"/GPIO/GPIO00/Dir","ts":{"$date":"2015-04-29T09:00:00.846Z"},"vHour":0}
and the code I have used in logstash:
input {
file {
type => "log"
path => ["/home/data/1/1.json"]
start_position => "beginning"
}
}
filter {
json{
source => "message"
}
}
output {
elasticsearch { embedded => true }
stdout { codec => rubydebug }
}
then the output in elasticsearch is :
{
"_index": "logstash-2015.06.29",
"_type": "log",
"_id": "AU5AG7KahwyA2bfnpJO0",
"_version": 1,
"_score": 1,
"_source": {
"message": "{"_id":{"$oid":"5540afc2cec7c68fc1248d7c"},"agentId":"0000000BAB39A520","handler":"SUSIControl","sensorId":"/GPIO/GPIO05/Dir","ts":{"$date":"2015-04-29T09:00:00.846Z"},"vHour":1}",
"#version": "1",
"#timestamp": "2015-06-29T16:17:03.040Z",
"type": "log",
"host": "song-Lenovo-IdeaPad",
"path": "/home/song/soft/data/1/Average.json",
"_id": {
"$oid": "5540afc2cec7c68fc1248d7c"
},
"agentId": "0000000BAB39A520",
"handler": "SUSIControl",
"sensorId": "/GPIO/GPIO05/Dir",
"ts": {
"$date": "2015-04-29T09:00:00.846Z"
},
"vHour": 1
}
}
But the information in the json file all in the _source not index
so that i can't use kibana to analysis them.
the kibana shows that Analysis is not available for object fields.
the _source is object fields
how to solve this problem?

Resources