Fluentd logs not working with multiple <match> - elasticsearch

Question

We have ElasticSearch FluentD Kibana Stack in our K8s, We are using different source for taking logs and matching it to different Elasticsearch host to get our logs bifurcated .
All was working fine until one of our elastic (elastic-audit) is down and now none of logs are getting pushed which has been mentioned on the fluentd config.
Not sure how can 1 elastic cluster can affect other cluster which are configured on same fluentd configuration file? We tried to remove the impacted tag from the configuration which is in our case "AUDIT_LOG" from filter and match , after which the logs started pushing from other tags.
Need to get FluentD working with all tags in it
NAMESPACE NAME HEALTH NODES VERSION PHASE AGE
dm-elastic dm-eck green 5 7.2.0 Ready 252d
elastic-audit da-audit red 7.4.0 Ready 263d
monitoring da-eck green 3 7.2.0 Ready 264d
moss-elastic moss-eck green 7 7.2.0 Ready 252d
Error Logs
2021-06-30 08:55:41 +0000 [warn]: [elasticsearch-auditlog] Remaining retry: 5. Retry to communicate after 1024 second(s).
2021-06-30 09:29:49 +0000 [warn]: [elasticsearch-auditlog] Could not communicate to Elasticsearch, resetting connection and trying again. SSL_connect SYSCALL returned=5 errno=0 state=unknown state (OpenSSL::SSL::SSLError)
2021-06-30 09:29:49 +0000 [warn]: [elasticsearch-auditlog] Remaining retry: 4. Retry to communicate after 2048 second(s).
FLUENTD Config
containers.input.conf: |-
<source>
#id fluentd-containers.log
#type tail
path /var/log/containers/*.log
exclude_path /var/log/containers/fluentd*.log
pos_file /var/log/es-containers.log.pos
time_format %Y-%m-%dT%H:%M:%S.%NZ
tag kubernetes.*
format json
read_from_head true
</source>
system.input.conf: |-
# Example:
# 2015-12-21 23:17:22,066 [salt.state ][INFO ] Completed state [net.ipv4.ip_forward] at time 23:17:22.066081
<source>
#id startupscript.log
#type tail
format syslog
path /var/log/startupscript.log
pos_file /var/log/es-startupscript.log.pos
tag startupscript
</source>
<source>
#id kubelet.log
#type tail
format multiline
multiline_flush_interval 5s
format_firstline /^\w\d{4}/
format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
time_format %m%d %H:%M:%S.%N
path /var/log/kubelet.log
pos_file /var/log/es-kubelet.log.pos
tag kubelet
</source>
# Logs from systemd-journal for interesting services.
# TODO(random-liu): Remove this after cri container runtime rolls out.
<source>
#id journald-docker
#type systemd
matches [{ "_SYSTEMD_UNIT": "docker.service" }]
<storage>
#type local
persistent true
</storage>
read_from_head true
tag docker
</source>
<source>
#id haproxy-ingress
#type syslog
port 5140
bind 0.0.0.0
tag haproxy
<parse>
#type syslog
message_format rfc5424
rfc5424_time_format %Y-%m-%dT%H:%M:%S%z
</parse>
</source>
<source>
#id journald-kubelet
#type systemd
matches [{ "_SYSTEMD_UNIT": "kubelet.service" }]
<storage>
#type local
persistent true
</storage>
read_from_head true
tag kubelet
</source>
forward.input.conf: |-
# Takes the messages sent over TCP
<source>
#type forward
</source>
monitoring.conf: |-
# Prometheus Exporter Plugin
# input plugin that exports metrics
<source>
#type prometheus
</source>
<source>
#type monitor_agent
</source>
# input plugin that collects metrics from MonitorAgent
<source>
#type prometheus_monitor
<labels>
host ${hostname}
</labels>
</source>
# input plugin that collects metrics for output plugin
<source>
#type prometheus_output_monitor
<labels>
host ${hostname}
</labels>
</source>
# input plugin that collects metrics for in_tail plugin
<source>
#type prometheus_tail_monitor
<labels>
host ${hostname}
</labels>
</source>
output.conf: |-
<filter kubernetes.**>
#type concat
key log
separator ""
stream_identity_key tag
multiline_start_regexp /^time=/
flush_interval 5
timeout_label #NORMAL
</filter>
<match **>
#type relabel
#label #NORMAL
</match>
<label #NORMAL>
<match kubernetes.**>
#type rewrite_tag_filter
<rule>
key log
pattern (tag=(AUDIT_LOG|CUSTOMER_AUDIT_LOG)|"log_type":"AUDIT_LOG")
tag auditlog.${tag}
</rule>
<rule>
key log
pattern tag=PARTNER_AUDIT_LOG
tag partner-auditlog.${tag}
</rule>
<rule>
key log
pattern tag=MAINTENANCE_AUDIT_LOG
tag maintenance-auditlog.${tag}
</rule>
<rule>
key log
pattern ^time=".*?".*
tag daas_service.${tag}
</rule>
<rule>
key log
pattern ^time=".*?".*
tag other_service.${tag}
invert true
</rule>
</match>
# Enriches records with Kubernetes metadata
<filter {daas_service,other_service}.kubernetes.**>
#type kubernetes_metadata
</filter>
<filter {daas_service,other_service}.kubernetes.**>
#type throttle
group_key kubernetes.pod_name
group_bucket_period_s 60
group_bucket_limit 6000
</filter>
<filter daas_service.kubernetes.**>
#type kvp
parse_key log
fields_key log_field
pattern "([a-zA-Z_-]\\w*)=((['\"])(?:^(?:\\3)|[^\\\\]|\\\\.)*?(\\3)|[\\w.#$%/+-]*)"
</filter>
<filter daas_service.kubernetes.**>
#type record_modifier
<record>
dummy ${if record.has_key?('log_field') and record['log_field'].has_key?('time'); record['#timestamp']=record['log_field']['time']; record['log_field'].delete('time'); end; nil}
dummy2 ${begin; t = Time.parse record['#timestamp']; record['#timestamp'] = t.utc.strftime('%Y-%m-%dT%H:%M:%S.%3NZ'); rescue; record.delete('#timestamp'); end; nil}
</record>
remove_keys dummy,dummy2
</filter>
<filter auditlog.kubernetes.**>
#type kvp
parse_key log
pattern "([a-zA-Z_-]\\w*)=((['\"])(?:^(?:\\3)|[^\\\\]|\\\\.)*?(\\3)|[\\w.#$%/+-]*)"
</filter>
<filter auditlog.kubernetes.**>
#type record_modifier
<record>
dummy ${if record.has_key?('time'); record['#timestamp']=record['time']; record.delete('time'); end; nil}
dummy2 ${begin; t = Time.parse record['#timestamp']; record['#timestamp'] = t.utc.strftime('%Y-%m-%dT%H:%M:%S.%3NZ'); rescue; record.delete('#timestamp'); end; nil}
levelinfo ${if record.has_key?('level'); record['level']='info'; end; nil}
</record>
remove_keys dummy,dummy2,levelinfo
</filter>
<filter partner-auditlog.kubernetes.**>
#type kvp
parse_key log
pattern "([a-zA-Z_-]\\w*)=((['\"])(?:^(?:\\3)|[^\\\\]|\\\\.)*?(\\3)|[\\w.#$%/+-]*)"
</filter>
<filter partner-auditlog.kubernetes.**>
#type record_modifier
<record>
dummy ${if record.has_key?('time'); record['#timestamp']=record['time']; record.delete('time'); end; nil}
dummy2 ${begin; t = Time.parse record['#timestamp']; record['#timestamp'] = t.utc.strftime('%Y-%m-%dT%H:%M:%S.%3NZ'); rescue; record.delete('#timestamp'); end; nil}
levelinfo ${if record.has_key?('level'); record['level']='info'; end; nil}
</record>
remove_keys dummy,dummy2,levelinfo
</filter>
<filter maintenance-auditlog.kubernetes.**>
#type kvp
parse_key log
pattern "([a-zA-Z_-]\\w*)=((['\"])(?:^(?:\\3)|[^\\\\]|\\\\.)*?(\\3)|[\\w.#$%/+-]*)"
</filter>
<filter maintenance-auditlog.kubernetes.**>
#type record_modifier
<record>
dummy ${if record.has_key?('time'); record['#timestamp']=record['time']; record.delete('time'); end; nil}
dummy2 ${begin; t = Time.parse record['#timestamp']; record['#timestamp'] = t.utc.strftime('%Y-%m-%dT%H:%M:%S.%3NZ'); rescue; record.delete('#timestamp'); end; nil}
levelinfo ${if record.has_key?('level'); record['level']='info'; end; nil}
</record>
remove_keys dummy,dummy2,levelinfo
</filter>
<filter haproxy.**>
#type parser
key_name message
reserve_data true
reserve_time true
emit_invalid_record_to_error false
<parse>
#type multi_format
<pattern>
format regexp
# Examples
# 10.2.1.0:31654 [06/Nov/2019:13:21:05.569] httpsfront default-paas-secure-443/10.20.48.136:443 1/0/642 3670 SD 3/2/0/0/0 0/0
expression /^(?<remoteAddress>[\w\.]+:\d+) \[(?<requestDate>[^\]]*)\] httpsfront (?<namespace>[\w]+)-(?<service>[\w-]+)\/(?<backendAddress>[\w\.]+:\d+) (?<waitTime>\d+)\/(?<backendConnectTime>\d+)\/(?<responseTime>\d+) (?<responseBytes>\d+) (?<terminationState>[\w-]+) (?<actconn>\d+)\/(?<feconn>\d+)\/(?<beconn>\d+)\/(?<srvconn>\d+)\/(?<retries>\d+) (?<srvqueue>\d+)\/(?<backendQueue>\d+)$/
</pattern>
<pattern>
format regexp
expression /^(?<remoteAddress>[\w\.]+:\d+) \[(?<requestDate>[^\]]*)\] httpfront-(?<domain>[\w-.]+)~ (?<namespace>kube-system|[\w]+)-(?<service>[\w-]+)(-[\d]+)?\/[\w-]+ (?<requestReadTime>\d+)\/(?<waitTime>\d+)\/(?<backendConnectTime>\d+)\/(?<backendResponseTime>\d+)\/(?<responseTime>\d+) (?<statusCode>\d+) (?<responseBytes>\d+) (?<reqCookie>[\w-]+) (?<resCookie>[\w-]+) (?<terminationState>[\w-]+) (?<actconn>\d+)\/(?<feconn>\d+)\/(?<beconn>\d+)\/(?<srvconn>\d+)\/(?<retries>\d+) (?<srvqueue>\d+)\/(?<backendQueue>\d+) "(?<method>[A-Z]+) (?<url>[^ ]+) (?<httpVersion>[^ ]+)"$/
</pattern>
<pattern>
format regexp
# Examples:
# Connect from 172.20.59.142:13201 to 172.20.59.142:31916 (httpfront/HTTP)
# Connect from 10.0.1.2:33312 to 10.0.3.31:8012 (www/HTTP)
expression /^Connect from (?<remoteAddress>[\w\.]+:\d+) to (?<backendAddress>[\w\.]+:\d+) \((?<frontend>[\w]+)\/(?<mode>[\w]+)\)$/
</pattern>
<pattern>
format regexp
# Examples:
# Server kube-system-fluentd-http-http-input/server0002 is going DOWN for maintenance. 3 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue.
# Server kube-system-fluentd-http-http-input/server0001 is going DOWN for maintenance. 2 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue.
expression /^Server (?<namespace>kube-system|[\w]+)-(?<service>[\w-]+)\/[\w-]+ is going DOWN for maintenance. (?<remainingActive>\d+) active and (?<remainingBackup>\d+) backup servers left. (?<activeSessions>\d+) sessions active, (?<requeued>\d+) requeued, (?<remainingInQueue>\d+) remaining in queue.$/
</pattern>
<pattern>
format regexp
# Examples:
# "10.2.2.0:60889 [06/Nov/2019:13:54:54.904] httpfront-shared-frontend/3: SSL handshake failure"
expression /^(?<remoteAddress>[\w\.]+:\d+) \[(?<requestDate>[^\]]*)\] (?<frontend>[\w-]+\/\d+): (?<msg>[\w].*)$/
</pattern>
<pattern>
format regexp
# Examples:
# Server kube-system-fluentd-http-http-input/server0003 is DOWN, reason: Layer4 connection problem, info: \"Connection refused\", check duration: 0ms. 2 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue.
# Server kube-system-fluentd-http-http-input/server0003 is UP, reason: Layer4 check passed, check duration: 0ms. 3 active and 0 backup servers online. 0 sessions requeued, 0 total in queue.
expression /^Server (?<namespace>kube-system|[\w]+)-(?<service>[\w-]+)\/[\w-]+ is (?<status>[\w]+), reason: (?<reason>[^,]+), (info: "(?<info>[^"]+)", )?check duration: (?<checkDuration>[^.]+). (?<remainingActive>\d+) active and (?<remainingBackup>\d+) backup servers (left|online). ((?<activeSessions>\d+) sessions active, )?(?<requeued>\d+) (sessions )?requeued, (?<remainingInQueue>\d+) (remaining|total) in queue.$/
</pattern>
</parse>
</filter>
<match auditlog.kubernetes.**>
#id elasticsearch-auditlog
#type elasticsearch
#log_level info
include_tag_key true
host elastic.audit.{{ required "management Domain" .Values.mgmt_domain }}
port 443
user "#{ENV['ELASTIC_USER']}"
password "#{ENV['ELASTIC_PASSWORD']}"
scheme "https"
ssl_verify false
type_name _doc
ssl_version TLSv1_2
time_precision 3
logstash_format true
logstash_prefix auditlog
reconnect_on_error true
request_timeout 30s
bulk_message_request_threshold -1
<buffer>
#type file
path /var/log/fluentd-buffers/auditlog.system.buffer
flush_mode interval
retry_type exponential_backoff
flush_thread_count 2
flush_interval 5s
retry_max_interval 30
chunk_limit_size 10M
total_limit_size 10G
queued_chunks_limit_size 100
overflow_action throw_exception
</buffer>
</match>
<match partner-auditlog.kubernetes.**>
#id elasticsearch-partner-auditlog
#type elasticsearch
#log_level info
include_tag_key true
host elastic.audit.{{ required "management Domain" .Values.mgmt_domain }}
port 443
user "#{ENV['ELASTIC_USER']}"
password "#{ENV['ELASTIC_PASSWORD']}"
scheme "https"
ssl_verify false
ssl_version TLSv1_2
type_name _doc
time_precision 3
logstash_format true
logstash_prefix partner-auditlog
reconnect_on_error true
request_timeout 30s
bulk_message_request_threshold -1
<buffer>
#type file
path /var/log/fluentd-buffers/partner-auditlog.system.buffer
flush_mode interval
retry_type exponential_backoff
flush_thread_count 2
flush_interval 5s
retry_max_interval 30
chunk_limit_size 10M
total_limit_size 10G
queued_chunks_limit_size 100
overflow_action throw_exception
</buffer>
</match>
<match maintenance-auditlog.kubernetes.**>
#id elasticsearch-maintenance-auditlog
#type elasticsearch
#log_level info
include_tag_key true
user "#{ENV['ELASTIC_USER']}"
password "#{ENV['ELASTIC_PASSWORD']}"
scheme "https"
ssl_verify false
ssl_version TLSv1_2
type_name _doc
host elastic.audit.{{ required "management Domain" .Values.mgmt_domain }}
port 443
time_precision 3
logstash_format true
logstash_prefix maintenance-auditlog
reconnect_on_error true
request_timeout 30s
bulk_message_request_threshold -1
<buffer>
#type file
path /var/log/fluentd-buffers/maintenance-auditlog.system.buffer
flush_mode interval
retry_type exponential_backoff
flush_thread_count 2
flush_interval 5s
retry_max_interval 30
chunk_limit_size 10M
total_limit_size 1G
queued_chunks_limit_size 100
overflow_action throw_exception
</buffer>
</match>
<match haproxy.**>
#id elasticsearch-haproxy
#type elasticsearch
#log_level info
include_tag_key true
user "#{ENV['ELASTIC_USER']}"
password "#{ENV['ELASTIC_PASSWORD']}"
scheme "https"
type_name _doc
ssl_verify false
ssl_version TLSv1_2
host da-eck-es-http.monitoring
port 9200
time_precision 3
logstash_format true
logstash_prefix haproxy
reconnect_on_error true
request_timeout 30s
bulk_message_request_threshold -1
<buffer>
#type memory
flush_mode interval
retry_type exponential_backoff
flush_thread_count 2
flush_interval 5s
retry_max_interval 20
retry_max_times 10
chunk_limit_size 10M
total_limit_size 100M
queued_chunks_limit_size 100
overflow_action throw_exception
</buffer>
</match>
<match **>
#id elasticsearch
#type elasticsearch
#log_level info
include_tag_key true
type_name _doc
host da-eck-es-http.monitoring
port 9200
user "#{ENV['ELASTIC_USER']}"
password "#{ENV['ELASTIC_PASSWORD']}"
scheme "https"
ssl_verify false
ssl_version TLSv1_2
time_precision 3
logstash_format true
reconnect_on_error true
request_timeout 30s
bulk_message_request_threshold -1
<buffer>
#type memory
flush_mode interval
retry_type exponential_backoff
flush_thread_count 2
flush_interval 5s
retry_max_interval 30
chunk_limit_size 10M
retry_max_times 10
total_limit_size 1G
queued_chunks_limit_size 100
overflow_action throw_exception
</buffer>
</match>
</label>