How to get more than one record? - elasticsearch

Question

I am using JDBC with logstash to get data from a PostgreSQL query and export it to ES v7 here is the configuration file:
input {
jdbc {
jdbc_connection_string => "jdbc:postgresql://ldatabase_rds_path?useSSL=true"
jdbc_user => "username"
jdbc_password => "password"
jdbc_driver_library => "/home/z/Documents/postgresql-42.2.18.jar"
jdbc_driver_class => "org.postgresql.Driver"
tracking_column => "id"
tracking_column_type => "numeric"
clean_run => true
schedule => "0 */1 * * *"
statement => "SELECT id as id, type as type, z_id as z_id, sender_id as sender_id, receiver_id as receiver_id, status as status, amount as amount, fees as fees, created as created, metadata as metadata, funding_source_from_id as funding_source_from_id, funding_source_to_id as funding_source_to_id, is_parent as is_parent, destination_type as destination_type, source_type as source_type FROM payments_transfer"
}
}
output {
stdout { codec => json_lines }
elasticsearch {
hosts => ["localhost:9200"]
manage_template => false
index => "payments_transfer_data"
document_id => "%{id}"
}
}
It take much time to only get 1 record from the database!
I tried some solutions like explicitly define a mapping so I added a mapping for the data like that:
PUT payments_transfer_data/_mapping/doc?include_type_name=true
{
"properties": {
"#timestamp": {
"type": "date"
},
"#version": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"amount": {
"type": "float"
},
"created": {
"type": "date"
},
"destination_type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"z_id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"fees": {
"type": "float"
},
"funding_source_from_id": {
"type": "long"
},
"funding_source_to_id": {
"type": "long"
},
"id": {
"type": "long"
},
"is_parent": {
"type": "boolean"
},
"metadata": {
"type": "keyword"
},
"receiver_id": {
"type": "long"
},
"sender_id": {
"type": "long"
},
"source_type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"status": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
}
}
}
Here is the only record that get:
source_type:customer sender_id:3 destination_type:funding-source type:funded_transfer z_id:863a240c-2011-e911-8114-bacd823e9f1d receiver_id:65 status:processed amount:550 funding_source_from_id:332 #timestamp:Nov 8, 2020 # 16:00:08.809 fees:5.61 is_parent:false created:Jan 5, 2019 # 21:28:21.847 #version:1 id:2,160 funding_source_to_id: - metadata: - _id:2160 _type:doc _index:payments_transfer_data _score: -

Answer 1

According to the official documentation you should use tracking_column in conjunction to use_column_value. With your current setting tracking_column will not have any impact.
Have you tried using Select * from to see if everything is being pulled.