elasticsearch query to extract namespace and log fields? - elasticsearch

Question

Using match_all I can get the following from a local es cluster:
$ curl "http://127.0.0.1:9200/_search?pretty" -H 'Content-Type: application/json' -d'{ "query": { "match_all": {} }}'
{
"took" : 9,
"timed_out" : false,
"_shards" : {
"total" : 1,
"successful" : 1,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 10000,
"relation" : "gte"
},
"max_score" : 1.0,
"hits" : [
{
"_index" : "kubernetes-2021.08.30",
"_type" : "_doc",
"_id" : "GSh-l3sBkJvTF8SCKo5f",
"_score" : 1.0,
"_source" : {
"#timestamp" : "2021-08-30T14:37:05.020Z",
"time" : "2021-08-30T14:37:05.020460752Z",
"stream" : "stderr",
"_p" : "F",
"log" : "[2021/08/30 14:37:05] [error] [net] TCP connection failed: elasticsearch-master.elk.svc.cluster.local:9200 (Connection refused)",
"kubernetes" : {
"pod_name" : "fluent-bit-49z9h",
"namespace_name" : "logging",
"pod_id" : "02428324-c3e0-459e-bcc5-0c33af8db989",
"labels" : {
"app_kubernetes_io/instance" : "fluent-bit",
"app_kubernetes_io/name" : "fluent-bit",
"controller-revision-hash" : "74556bf9cf",
"pod-template-generation" : "1"
},
"annotations" : {
"checksum/config" : "f4a875e2e4705ad60e5dcc5c306e94891f9200db72649cff4020642d9df2ecf1",
"checksum/luascripts" : "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
},
"host" : "my-kind-worker",
"container_name" : "fluent-bit",
"docker_id" : "46f6f349cc5bed659c50f9d29a94a76376f9243c076cdd29dfbd7cc60c238149",
"container_hash" : "docker.io/fluent/fluent-bit#sha256:10ea2709cef6e7059d980b4969d5f9d753ef97278a817c214cbe9120b1152082",
"container_image" : "docker.io/fluent/fluent-bit:1.8.3"
}
}
},
{
"_index" : "kubernetes-2021.08.30",
"_type" : "_doc",
"_id" : "Gih-l3sBkJvTF8SCKo5f",
"_score" : 1.0,
"_source" : {
"#timestamp" : "2021-08-30T14:37:05.020Z",
"time" : "2021-08-30T14:37:05.020491241Z",
"stream" : "stderr",
"_p" : "F",
"log" : "[2021/08/30 14:37:05] [error] [net] socket #64 could not connect to elasticsearch-master.elk.svc.cluster.local:9200",
"kubernetes" : {
"pod_name" : "fluent-bit-49z9h",
"namespace_name" : "logging",
"pod_id" : "02428324-c3e0-459e-bcc5-0c33af8db989",
"labels" : {
"app_kubernetes_io/instance" : "fluent-bit",
"app_kubernetes_io/name" : "fluent-bit",
"controller-revision-hash" : "74556bf9cf",
"pod-template-generation" : "1"
},
"annotations" : {
"checksum/config" : "f4a875e2e4705ad60e5dcc5c306e94891f9200db72649cff4020642d9df2ecf1",
"checksum/luascripts" : "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
},
"host" : "my-kind-worker",
"container_name" : "fluent-bit",
"docker_id" : "46f6f349cc5bed659c50f9d29a94a76376f9243c076cdd29dfbd7cc60c238149",
"container_hash" : "docker.io/fluent/fluent-bit#sha256:10ea2709cef6e7059d980b4969d5f9d753ef97278a817c214cbe9120b1152082",
"container_image" : "docker.io/fluent/fluent-bit:1.8.3"
}
}
}
]
}
}
For each hit I would like to just print the log and namespace_name. I have tried with:
$ curl -X GET "http://127.0.0.1:9200/_search?pretty" -H 'Content-Type: application/json' -d'
{
"_source": {
"includes": [ "log", "kubernetes.namespace_name" ],
"excludes": [ "_type" ]
}
}
'
{
"took" : 293,
"timed_out" : false,
"_shards" : {
"total" : 1,
"successful" : 1,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 10000,
"relation" : "gte"
},
"max_score" : 1.0,
"hits" : [
{
"_index" : "kubernetes-2021.08.30",
"_type" : "_doc",
"_id" : "GSh-l3sBkJvTF8SCKo5f",
"_score" : 1.0,
"_source" : {
"kubernetes" : {
"namespace_name" : "logging"
},
"log" : "[2021/08/30 14:37:05] [error] [net] TCP connection failed: elasticsearch-master.elk.svc.cluster.local:9200 (Connection refused)"
}
},
{
"_index" : "kubernetes-2021.08.30",
"_type" : "_doc",
"_id" : "Gih-l3sBkJvTF8SCKo5f",
"_score" : 1.0,
"_source" : {
"kubernetes" : {
"namespace_name" : "logging"
},
"log" : "[2021/08/30 14:37:05] [error] [net] socket #64 could not connect to elasticsearch-master.elk.svc.cluster.local:9200"
}
}
]
}
}
But even though I have "excludes": [ "_type" ] its still part of the output.
How do I trim those unwanted fields from the output?

Answer 1

You can use filter_path to modify your response result
POST/ http://localhost:9200/index-name/_search?filter_path=hits.hits._source
{
"_source": {
"includes": [
"log",
"kubernetes.namespace_name"
]
}
}
The search result will be
{
"hits": {
"hits": [
{
"_source": {
"kubernetes": {
"namespace_name": "logging"
},
"log": "[2021/08/30 14:37:05] [error] [net] socket #64 could not connect to elasticsearch-master.elk.svc.cluster.local:9200"
}
}
]
}
}