How to parse a specific HTTP status error - elasticsearch

Question

I have a nginx-ingress-controller logs in the Elasticsearch that is not able to catch/parse HTTP error 503 (Service Unavailable) and come up with it in the "Status" field, but HTTP 200 and others work.
Elasticsearch discover - currently
I have these the annotations already configured in the nginx-ingress-controller deployment:
annotations:
co.elastic.logs/processors.0.dissect.ignore_failure: "true"
co.elastic.logs/processors.0.dissect.target_prefix: dissect
co.elastic.logs/processors.0.dissect.tokenizer: '%{levelandtimestamp} %{pid} %{class} %{message}'
co.elastic.logs/processors.0.dissect.when.regexp.message: ^[IWEF][0-9]{4}.*
co.elastic.logs/processors.1.copy_fields.fields.0.from: dissect.levelandtimestamp
co.elastic.logs/processors.1.copy_fields.fields.0.to: dissect.timestamp
co.elastic.logs/processors.1.copy_fields.ignore_missing: "true"
co.elastic.logs/processors.2.dissect.field: dissect.timestamp
co.elastic.logs/processors.2.dissect.ignore_failure: "true"
co.elastic.logs/processors.2.dissect.target_prefix: dissect
co.elastic.logs/processors.2.dissect.tokenizer: '%{time_normilized}'
co.elastic.logs/processors.2.dissect.trim_chars: IWEF
co.elastic.logs/processors.2.dissect.trim_values: left
co.elastic.logs/processors.3.timestamp.field: dissect.time_normilized
co.elastic.logs/processors.3.timestamp.ignore_missing: "true"
co.elastic.logs/processors.3.timestamp.layouts: 0102 15:04:05.999
co.elastic.logs/processors.3.timestamp.when.has_fields: dissect.time_normilized
co.elastic.logs/processors.4.drop_fields.fields: message
co.elastic.logs/processors.4.drop_fields.ignore_missing: "true"
co.elastic.logs/processors.4.drop_fields.when.regexp.message: ^[IWEF][0-9]{4}.*
co.elastic.logs/processors.5.rename.fields.0.from: dissect.message
co.elastic.logs/processors.5.rename.fields.0.to: message
co.elastic.logs/processors.5.rename.fields.1.from: dissect.class
co.elastic.logs/processors.5.rename.fields.1.to: class
co.elastic.logs/processors.5.rename.ignore_missing: "true"
co.elastic.logs/processors.5.rename.when.has_fields: dissect.message
co.elastic.logs/processors.6.add_fields.fields.level: INFO
co.elastic.logs/processors.6.add_fields.target: ""
co.elastic.logs/processors.7.add_fields.fields.level: ERROR
co.elastic.logs/processors.7.add_fields.target: ""
co.elastic.logs/processors.7.add_fields.when.contains.dissect.levelandtimestamp: E
co.elastic.logs/processors.8.add_fields.fields.level: WARN
co.elastic.logs/processors.8.add_fields.target: ""
co.elastic.logs/processors.8.add_fields.when.contains.dissect.levelandtimestamp: W
co.elastic.logs/processors.9.add_fields.fields.level: FATAL
co.elastic.logs/processors.9.add_fields.target: ""
co.elastic.logs/processors.9.add_fields.when.contains.dissect.levelandtimestamp: F
co.elastic.logs/processors.10.decode_json_fields.fields: message
co.elastic.logs/processors.10.decode_json_fields.max_depth: "1"
co.elastic.logs/processors.10.decode_json_fields.overwrite_keys: "true"
co.elastic.logs/processors.10.decode_json_fields.target: ""
co.elastic.logs/processors.11.timestamp.field: time
co.elastic.logs/processors.11.timestamp.layouts: "2006-01-02T15:04:05+00:00"
co.elastic.logs/processors.11.timestamp.when.has_fields: time
co.elastic.logs/processors.12.dissect.field: request_query
co.elastic.logs/processors.12.dissect.ignore_failure: "true"
co.elastic.logs/processors.12.dissect.target_prefix: ""
co.elastic.logs/processors.12.dissect.tokenizer: '%{request_method} %{request_uri} %{request_protocol}'
co.elastic.logs/processors.12.dissect.when.regexp.message: request_query
co.elastic.logs/processors.13.drop_fields.fields: message
co.elastic.logs/processors.13.drop_fields.ignore_missing: "true"
co.elastic.logs/processors.13.drop_fields.when.has_fields: request_query
Filebeat.yaml - ConfigMap:
Data
====
filebeat.yml:
----
http:
enabled: true
host: localhost
port: 5066
filebeat.inputs:
- type: udp
max_message_size: 10MiB
host: "0.0.0.0:9999"
fields:
event_type: "vault-audit"
fields_under_root: true
processors:
- decode_json_fields:
fields: ["message"]
target: "vault"
process_array: true
overwrite_keys: false
add_error_key: true
- copy_fields:
fields:
- from: vault.response.data.username
to: vault.response.datainfo.username
fail_on_error: false
ignore_missing: true
- drop_fields:
fields: ["vault.response.data"]
ignore_missing: true
- timestamp:
field: vault.time
layouts:
- 'Y'
- type: tcp
max_message_size: 10MiB
host: "0.0.0.0:9000"
fields:
event_type: "vault-audit"
fields_under_root: true
processors:
- decode_json_fields:
fields: ["message"]
target: "vault"
process_array: true
overwrite_keys: false
add_error_key: true
- copy_fields:
fields:
- from: vault.response.data.username
to: vault.response.datainfo.username
fail_on_error: false
ignore_missing: true
- drop_fields:
fields: ["vault.response.data"]
ignore_missing: true
- timestamp:
field: vault.time
layouts:
- 'Y'
filebeat.autodiscover:
providers:
- type: kubernetes
hints.enabled: true
hints.default_config:
type: container
paths:
- /var/lib/docker/containers/${data.kubernetes.container.id}/*.log
processors:
- add_fields:
target: kubernetes
fields:
cluster.name: "k8s-dev"
- drop_event:
when:
and:
- contains:
message: "DEBUG"
- contains:
message: "changes.SessionEntityWrapper"
- drop_event:
when:
equals:
class: "c.i.p.metrics.TelegrafMetricObserver"
- drop_event:
when:
contains:
message: "com.xxx"
- drop_event:
when:
contains:
message: "metrics.xxx.com"
- drop_event:
when:
and:
- equals:
service: "CMC"
- contains:
exception.stacktrace: "org.hibernate.HibernateException: createQuery is not valid without active transaction"
- drop_event:
when:
and:
- equals:
kubernetes.container.name: "selenoid"
- contains:
kubernetes.pod.name: "availability-tests"
- drop_event:
when:
equals:
kubernetes.labels.app: "time-nginx"
- add_cloud_metadata: ~
- rename:
ignore_missing: true
fail_on_error: false
fields:
- from: "kubernetes.labels.k8s-app"
to: "service"
- from: "kubernetes.labels.service"
to: "service"
- rename:
fields:
- from: "kubernetes.labels.tenant-alias"
to: "tenant_alias"
ignore_missing: true
fail_on_error: false
when:
not:
has_fields: ['tenant_alias']
- rename:
fields:
- from: "kubernetes.labels.tenant-id"
to: "tenant_id"
ignore_missing: true
fail_on_error: false
when:
not:
has_fields: ['tenant_id']
- script:
lang: javascript
id: lowercase
source: >
function process(event) {
var level = event.Get("level");
if(level != null) {
event.Put("level", level.toString().toLowerCase());
}
}
- drop_fields:
fields:
- dissect
- ecs
- input
- ts
- tsNs
- stream
- kubernetes.namespace_uid
- kubernetes.namespace_labels
- kubernetes.node.uid
- kubernetes.node.hostname
- kubernetes.node.labels
- kubernetes.pod.uid
- kubernetes.pod.ip
- kubernetes.statefulset
- kubernetes.replicaset
- kubernetes.container.image
- kubernetes.labels
- container.id
ignore_missing: true
logging.metrics.enabled: false
logging.json: true
logging.level: warning
output.kafka:
version: 2.0.0
codec.json:
pretty: false
# escape_html: false
client_id: "logshipper"
hosts: ["kafka-cp.xxx.com:9094"]
topic: "logging-kubernetes"
topics:
- topic: "kubernetes-audit"
when.equals:
event_type: "audit"
- topic: "vault-audit"
when.equals:
event_type: "vault-audit"
partition.round_robin:
group_events: 10
reachable_only: false
required_acks: 1
compression: gzip
max_message_bytes: 1e+06
ssl.certificate_authorities: /tmp/ca.crt
How can I parse status error 503 into Elasticsearch Discover the same way as others?

Answer 1

The error was in the log format, more specifically in log-format-upstream in the nginx-ingress-controller configmap, some fields did not have "quote".
old:
log-format-upstream: '{ "time": "$time_iso8601", "remote_addr": "$remote_addr",
"x-forward-for": "$proxy_add_x_forwarded_for", "request_time": $request_time,
"status": $status, "bytes_sent": $bytes_sent, "body_bytes_sent": $body_bytes_sent,
"request_length": $request_length, "request_host": "$http_host", "request_query":
"$request", "http_referrer": "$http_referer", "http_user_agent": "$http_user_agent",
"upstream_name": "$proxy_upstream_name", "upstream_addr": "$upstream_addr", "upstream_response_length":
$upstream_response_length, "upstream_response_time": $upstream_response_time,
"upstream_status": $upstream_status, "request_id": "$req_id" }'
new:
log-format-upstream: '{ "time": "$time_iso8601", "remote_addr": "$remote_addr",
"x-forward-for": "$proxy_add_x_forwarded_for", "request_time": "$request_time",
"status": "$status", "bytes_sent": "$bytes_sent", "body_bytes_sent": "$body_bytes_sent",
"request_length": "$request_length", "request_host": "$http_host", "request_query":
"$request", "http_referrer": "$http_referer", "http_user_agent": "$http_user_agent",
"upstream_name": "$proxy_upstream_name", "upstream_addr": "$upstream_addr", "upstream_response_length":
"$upstream_response_length", "upstream_response_time": "$upstream_response_time",
"upstream_status": "$upstream_status", "request_id": "$req_id" }'