filebeat JSON logging broken logs in Elastic search - elasticsearch

Question

I'm using filebeat>kafka>logstash>elastiserach stack of version 7.15.0.
My logs are getting broken and not writing properly into elastic which is resulting in json.erros
error.message:Key 'log' not found
error.type: json
and
error.message:Error decoding JSON: invalid character 's' after array element error.type:json
my log contains both json and nonjson contents
my docker stdout logs>>
{"log":"{\"instant\":{\"epochSecond\":1643023707,\"nanoOfSecond\":538281000},\"thread\":\"grpc-default-executor-11\",\"level\":\"INFO\",\"loggerName\":\"com.abc.ab.ab.core.service.integration.controller.NotifyCoreGrpcController\",\"message\":\"RMQ_CORE_GRPC_NOTIFY RESP : {\\\"baseBizResponse\\\":{\\\"success\\\":true,\\\"resultCode\\\":\\\"SUCCESS\\\"}} \",\"endOfBatch\":false,\"loggerFqcn\":\"org.apache.logging.slf4j.Log4jLogger\",\"contextMap\":{\"RMQ_ID\":\"2022012401445669241212121212\",\"FLOW_TYPE\":\"RMQ_CORE_GRPC_NOTIFY\",\"MERCHANT_TRANS_ID\":\"bcd4ab1e54abaha122\",\"spanId\":\"4fa1474c078afceb\",\"traceId\":\"bcd4ab1e54abaha122\"},\"threadId\":100,\"threadPriority\":5,\"dateTime\":\"2022-01-24T16:58:27.538+0530\"}\r\n","stream":"stdout","time":"2022-01-24T11:28:27.538354156Z"}
and
[244540.169s][debug][gc,age] GC(51) Desired survivor size 80740352 bytes, new threshold 15 (max threshold 15)
Filebeat conf>>
filebeat.yml: |
filebeat.inputs:
- type: container
multiline.pattern: ^[[:space:]]'
multiline.negate: false
multiline.match: after
json.keys_under_root: true
json.message_key: log
json.add_error_key: true
enabled: true
paths:
- /var/log/containers/*.log
exclude_files: ['fluentd-*', 'istio-*', 'cluster-logging-*', 'infra-*']
processors:
- add_kubernetes_metadata:
in_cluster: true
host: ${NODE_NAME}
default_matchers.enabled: false
matchers:
- logs_path:
logs_path: "/var/log/containers/"
processors:
- drop_fields:
fields:
- 'kubernetes.node.name'
output.kafka:
enabled: true
hosts: ["kafka1:9092","kafka2:9092","kafka3:9092"]
partition.round_robin:
reachable_only: false
required_acks: 1
compression: gzip
max_message_bytes: 10000000
topics:
- topic: '%{[kubernetes.labels.app]}'
default: 'app-perf-k8s-logs'
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
filebeat.modules:
- module: nginx
- module: kafka
logging.level: debug
logstash config >>
input {
kafka {
topics_pattern => ".*."
bootstrap_servers => "kafka1:9092","kafka2:9092","kafka3:9092"
client_id => "logstash"
codec => json
decorate_events => true
consumer_threads => 10
heartbeat_interval_ms => "100000"
session_timeout_ms => "300000"
poll_timeout_ms => 300000
partition_assignment_strategy => "org.apache.kafka.clients.consumer.RoundRobinAssignor"
request_timeout_ms => "400000"
group_id => "logConsumer"
auto_offset_reset => "latest"
}
}
output {
elasticsearch {
hosts => "es-logging-perf-lb.abc.com:80"
index => "filebeat-%{[#metadata][kafka][topic]}-%{+YYYY.MM.dd}"
}
Please help and suggest.