Parsing array of objects using Logstash - elasticsearch

Question

I have JMS message payload containing array of customer objects which needs to indexed into elastic search.
JMS Payload
{
"file": {
"header": {
"id": "13",
"name": "ElasticCustomer",
"description": "ElasticCustomer",
"version": "Version 1",
"exportID": "ElasticCustomer",
"exportInstance": "20200918043921264",
"extractionMode": "FULL",
"extractionDate": "2020-09-18T05:37:32"
},
"body": {
"records": [
{
"customerId": "NVS1200063862",
"URI": "https://data.novartis.net/r1/commercial/customer_management/customer_mdm/v1/customer/HCO/NVS1200063862",
"customerCountry": "200024",
"type": {
"linked_record": {
"code": "200027",
"description": "Health Care Organization",
"shortDescription": "HCO",
"remoteKeys": {
"linked_records": []
},
"technicalData": {
"lastUser": "chaoure1",
"creationTime": "2019-06-24T11:02:45",
"lastTime": "2019-06-24T11:02:45",
"primaryKey": "200027",
"foreignKey": "200027",
"linkedTable": "/root/ReferenceData"
}
}
},
"veevaRecordType": {
"recordTypeId": "012A0000000zC10IAE",
"recordTypeValue": "PH_CORE-Clinics"
},
"status": {
"linked_record": {
"code": "200036",
"description": "Active",
"shortDescription": null,
"remoteKeys": {
"linked_records": [
{
"system": "200073",
"systemKey": "STA.3",
"technicalData": {
"lastUser": "moritda3",
"creationTime": "2020-04-22T19:48:08",
"lastTime": "2020-04-22T19:48:08",
"primaryKey": "6099",
"linkedTable": "/root/RemoteKeys"
}
},
{
"system": "200072",
"systemKey": "Active",
"technicalData": {
"lastUser": "moritda3",
"creationTime": "2020-04-22T19:48:08",
"lastTime": "2020-04-22T19:48:08",
"primaryKey": "6351",
"linkedTable": "/root/RemoteKeys"
}
},
{
"system": "200071",
"systemKey": "Active",
"technicalData": {
"lastUser": "moritda3",
"creationTime": "2020-04-22T19:48:08",
"lastTime": "2020-04-22T19:48:08",
"primaryKey": "6353",
"linkedTable": "/root/RemoteKeys"
}
}
]
},
"technicalData": {
"lastUser": "chaoure1",
"creationTime": "2019-06-24T11:02:45",
"lastTime": "2019-06-24T11:02:45",
"primaryKey": "200036",
"foreignKey": "200036",
"linkedTable": "/root/ReferenceData"
}
}
},
"statusReason": {
"linked_record": {}
},
"hco": "HCO1200016052",
"hcp": null,
"healthCareOrganisation": {
"linked_records": []
},
"healthCareProfessional": {
"linked_records": []
},
"communication": {
"linked_records": [
{
"communicationId": 2739472,
"customerId": "NVS1200063862",
"communicationType": {
"linked_record": {
"code": "200146",
"description": "Communication Channel",
"shortDescription": null,
"remoteKeys": {
"linked_records": []
},
"technicalData": {
"lastUser": "chaoure1",
"creationTime": "2019-06-24T11:02:45",
"lastTime": "2019-06-24T11:02:45",
"primaryKey": "200146",
"foreignKey": "200146",
"linkedTable": "/root/ReferenceData"
}
}
},
"type": "212126",
"value": "519 2046052",
"status": {
"linked_record": {
"code": "200084",
"description": "Active",
"shortDescription": null,
"remoteKeys": {
"linked_records": []
},
"technicalData": {
"lastUser": "chaoure1",
"creationTime": "2019-06-24T11:02:45",
"lastTime": "2019-06-24T11:02:45",
"primaryKey": "200084",
"foreignKey": "200084",
"linkedTable": "/root/ReferenceData"
}
}
},
"isPrimary": false,
"technicalData": {
"lastUser": "sys_apimdmprod",
"creationTime": "2020-08-07T08:01:30",
"lastTime": "2020-08-07T08:01:30",
"primaryKey": "2739472",
"linkedTable": "/root/Communication"
}
}
]
},
"remoteKeys": {
"linked_records": []
},
"adresses": {
"linked_records": [
{
"addressId": 6748337,
"Identification": {
"customerId": "NVS1200063862",
"isPrimary": true,
"addressType": {
"linked_record": {
"code": "200148",
"description": "Primary address",
"shortDescription": null,
"remoteKeys": {
"linked_records": [
{
"system": "200072",
"systemKey": "Primary address",
"status": "AC",
"technicalData": {
"lastUser": "moritda3",
"creationTime": "2020-04-22T19:48:08",
"lastTime": "2020-04-22T19:48:08",
"primaryKey": "5412",
"linkedTable": "/root/RemoteKeys"
}
},
{
"system": "200071",
"systemKey": "Primary address",
"status": "AC",
"technicalData": {
"lastUser": "moritda3",
"creationTime": "2020-04-22T19:48:08",
"lastTime": "2020-04-22T19:48:08",
"primaryKey": "5417",
"linkedTable": "/root/RemoteKeys"
}
},
{
"system": "200073",
"systemKey": "TYS.P",
"status": "AC",
"technicalData": {
"lastUser": "moritda3",
"creationTime": "2020-04-22T19:48:08",
"lastTime": "2020-04-22T19:48:08",
"primaryKey": "5422",
"linkedTable": "/root/RemoteKeys"
}
}
]
},
"technicalData": {
"lastUser": "chaoure1",
"creationTime": "2019-06-24T11:02:45",
"lastTime": "2019-06-24T11:02:45",
"primaryKey": "200148",
"foreignKey": "200148",
"linkedTable": "/root/ReferenceData"
}
}
}
},
"AddressAttributes": {
"addressLine1": "D-590 Oxford St E",
"addressLine2": "D",
"addressLine3": null,
"city": "London",
"stateandprovince": {
"linked_record": {
"code": "213040",
"description": "ON",
"shortDescription": "ON",
"remoteKeys": {
"linked_records": [
{
"code": 6672,
"system": "200071",
"systemKey": "ON",
"status": "AC",
"technicalData": {
"lastUser": "moritda3",
"creationTime": "2020-04-22T19:48:08",
"lastTime": "2020-04-22T19:48:08",
"primaryKey": "6672",
"linkedTable": "/root/RemoteKeys"
}
},
{
"code": 6685,
"system": "200073",
"systemKey": "009",
"status": "AC",
"technicalData": {
"lastUser": "moritda3",
"creationTime": "2020-04-22T19:48:08",
"lastTime": "2020-04-22T19:48:08",
"primaryKey": "6685",
"linkedTable": "/root/RemoteKeys"
}
},
{
"code": 11365,
"system": "209336",
"systemKey": "ON",
"status": "AC",
"technicalData": {
"lastUser": "moritda3",
"creationTime": "2020-04-22T19:48:08",
"lastTime": "2020-04-22T19:48:08",
"primaryKey": "11365",
"linkedTable": "/root/RemoteKeys"
}
}
]
},
"technicalData": {
"lastUser": "chaoure1",
"creationTime": "2019-06-24T11:02:45",
"lastTime": "2019-06-24T11:02:45",
"primaryKey": "213040",
"foreignKey": "213040",
"linkedTable": "/root/ReferenceData"
}
}
},
"zip": "N5Y 3J1",
"country": {
"linked_record": {
"code": "200024",
"description": "Canada",
"shortDescription": "CA",
"remoteKeys": {
"linked_records": [
{
"system": "200071",
"systemKey": "Canada",
"status": "AC",
"technicalData": {
"lastUser": "moritda3",
"creationTime": "2020-04-22T19:48:08",
"lastTime": "2020-04-22T19:48:08",
"primaryKey": "6081",
"linkedTable": "/root/RemoteKeys"
}
},
{
"system": "200073",
"systemKey": "CA",
"status": "AC",
"technicalData": {
"lastUser": "moritda3",
"creationTime": "2020-04-22T19:48:08",
"lastTime": "2020-04-22T19:48:08",
"primaryKey": "6082",
"linkedTable": "/root/RemoteKeys"
}
}
]
},
"technicalData": {
"lastUser": "singhv2z",
"creationTime": "2019-06-24T11:02:45",
"lastTime": "2020-05-04T06:48:38",
"primaryKey": "200024",
"foreignKey": "200024",
"linkedTable": "/root/ReferenceData"
}
}
}
},
"AdditionalAttributes": {
"brick": "N5Y",
"microbrick": null,
"latitude": null,
"longitude": null,
"height": null,
"geographicCoordinates": null,
"masterAddress": null,
"effectiveDate": "2020-08-06",
"addressStatus": {
"linked_record": {
"code": "200003",
"description": "Active",
"shortDescription": null,
"remoteKeys": {
"linked_records": []
},
"technicalData": {
"lastUser": "chaoure1",
"creationTime": "2019-06-24T11:02:45",
"lastTime": "2019-06-24T11:02:45",
"primaryKey": "200003",
"foreignKey": "200003",
"linkedTable": "/root/ReferenceData"
}
}
},
"addressVerificationStatus": {
"linked_record": {
"code": "200032",
"description": "Verified",
"shortDescription": null,
"remoteKeys": {
"linked_records": []
},
"technicalData": {
"lastUser": "chaoure1",
"creationTime": "2019-06-24T11:02:45",
"lastTime": "2019-06-24T11:02:45",
"primaryKey": "200032",
"foreignKey": "200032",
"linkedTable": "/root/ReferenceData"
}
}
}
},
"technicalData": {
"lastUser": "sys_apimdmprod",
"creationTime": "2020-08-07T08:01:35",
"lastTime": "2020-08-07T08:01:35",
"primaryKey": "6748337",
"linkedTable": "/root/Address"
}
}
]
},
"education": {
"linked_records": []
},
"specialties": {
"linked_records": []
},
"consent": {
"linked_records": []
},
"credentials": {
"linked_records": []
},
"Affiliation": {
"customerParents": {
"linked_records": [
{
"affiliationId": 5977284,
"parentId": "NVS1200063862",
"childId": "NVS1000143585",
"affiliationType": {
"linked_record": {
"code": "200006",
"description": "HCO-HCP",
"shortDescription": null,
"remoteKeys": {
"linked_records": []
},
"technicalData": {
"lastUser": "chaoure1",
"creationTime": "2019-06-24T11:02:45",
"lastTime": "2019-06-24T11:02:45",
"primaryKey": "200006",
"foreignKey": "200006",
"linkedTable": "/root/ReferenceData"
}
}
},
"startDate": "2020-08-07",
"endDate": null,
"isPrimary": true,
"nanobrick": null,
"role": null,
"technicalData": {
"lastUser": "sys_apimdmprod",
"creationTime": "2020-08-07T08:01:55",
"lastTime": "2020-08-07T08:01:55",
"primaryKey": "5977284",
"linkedTable": "/root/Affiliation"
}
}
]
},
"customerChild": {
"linked_records": []
}
},
"additionalQualification": {
"linked_records": []
},
"statusComment": null,
"Tracking": {
"technicalRecordStatus": "PU",
"producerSystem": "OneKey",
"producerSystemRecordID": "WCAH00083151",
"transactionID": "WORKPLACE_execution3ab77e11c84a403aa12a33c53fafaa4520200807080051163",
"createdDate": "2020-08-07T08:01:27",
"createdBy": "OneKey",
"LastUpdatedDate": "2020-08-07T08:01:35",
"lastUpdateBy": "Api Gateway (sys_apimdmprod)",
"ApprovalDate": null,
"ApprovedBy": null,
"workflowID": null
},
"technicalData": {
"lastUser": "ebx-systemUser",
"creationTime": "2020-08-07T08:01:29",
"lastTime": "2020-09-17T15:41:04",
"primaryKey": "NVS1200063862"
}
}
]
},
"footer": {
"nb_records": 1
}
}
}
I want to create seperate upsert logstash event for each record in records array field.
My Current Logstash configuration looks like below
input
{
jms {
include_header => false
include_properties => false
include_body => true
use_jms_timestamp => false
destination => 'SpringBatchTestQueue'
pub_sub => false
jndi_name => '/JMS/CF/MDM'
jndi_context => {
'java.naming.factory.initial' => 'com.solacesystems.jndi.SolJNDIInitialContextFactory'
'java.naming.security.principal' => 'EDM_Test_User#NovartisDevVPN'
'java.naming.provider.url' => 'tcp://localhost:55555'
'java.naming.security.credentials' => 'EDM_Test_User'
}
require_jars=> ['/app/elasticsearch/jms/commons-lang-2.6.jar',
'/app/elasticsearch/jms/sol-jms-10.10.0.jar',
'/app/elasticsearch/jms/geronimo-jms_1.1_spec-1.1.1.jar']
}
}
filter{
json{
source => "message"
}
split {
field => "[file][body][records]"
}
}
output
{
elasticsearch
{
hosts => ["https://localhost:9200/"]
index => "test-%{+YYYY.MM.dd}"
document_id => "%{customerId}"
doc_as_upsert => true
ssl => true
ssl_certificate_verification => true
cacert => "/app/elasticsearch/config/ssl/Novartis_Silver_Three_Chain.pem"
}
}
Can anyone please let me know how each customerId in recordsfield can be indexed into ES using logstash?
Edit 1: As per answer suggested by Alcanzer.._id field is not populated correctly as well message contains unnecessary fields as well

Answer 1

You'll need to use the split filter to split on [file][body][records]. One use case of the split filter is to take an Array and create 1 event per element in the array.
You'd just add
split { field => "[file][body][records]" }
After your json filter.