Related
I was having a problem dropping replicas when transitioning to warm nodes and I found out that the replicas need to be dropped while hot because the ultrawarm nodes have read-only S3 backed storage. So, I then added another hot stage called "hot_no_replicas" and I was seeing that in fact the replicas were being dropped as planned, However, after the warm migration, the replicas are reappearing. You can see it clearly here in the screenshot. 000022 still has a replica and is on hot. 000021, being older, has had the replica dropped and is still on hot. But 000020 being older than 000021 and having migrated to warm, now has a replica again. I will paste the logging policy and attach screenshot. Does anyone know why this is happening?
ps, I added another , probably superfluous "number_of_replicas" : 0" to the warm state to try one more time to get them removed before the warm transition.
{
"_id" : "shortterm_logging_policy",
"_version" : 37,
"_seq_no" : 1640996,
"_primary_term" : 1,
"policy" : {
"policy_id" : "shortterm_logging_policy",
"description" : "hot-warm-delete",
"last_updated_time" : 1672416827543,
"schema_version" : 14,
"error_notification" : null,
"default_state" : "hot",
"states" : [
{
"name" : "hot",
"actions" : [
{
"retry" : {
"count" : 3,
"backoff" : "exponential",
"delay" : "1m"
},
"rollover" : {
"min_size" : "1gb",
"min_doc_count" : 6000000,
"min_index_age" : "1d"
}
}
],
"transitions" : [
{
"state_name" : "hot_no_replicas",
"conditions" : {
"min_index_age" : "2d"
}
}
]
},
{
"name" : "hot_no_replicas",
"actions" : [
{
"retry" : {
"count" : 3,
"backoff" : "exponential",
"delay" : "1m"
},
"replica_count" : {
"number_of_replicas" : 0
}
}
],
"transitions" : [
{
"state_name" : "warm",
"conditions" : {
"min_index_age" : "3d"
}
}
]
},
{
"name" : "warm",
"actions" : [
{
"retry" : {
"count" : 3,
"backoff" : "exponential",
"delay" : "1m"
},
"replica_count" : {
"number_of_replicas" : 0
}
},
{
"retry" : {
"count" : 3,
"backoff" : "exponential",
"delay" : "1m"
},
"warm_migration" : { }
}
],
"transitions" : [
{
"state_name" : "delete",
"conditions" : {
"min_index_age" : "7d"
}
}
]
},
{
"name" : "delete",
"actions" : [
{
"retry" : {
"count" : 3,
"backoff" : "exponential",
"delay" : "1m"
},
"delete" : { }
}
],
"transitions" : [ ]
}
],
"ism_template" : [
{
"index_patterns" : [
"filebeat-syslog-*"
],
"priority" : 1,
"last_updated_time" : 1652815954078
},
{
"index_patterns" : [
"filebeat-mapi_logs-*"
],
"priority" : 2,
"last_updated_time" : 1666124989125
},
{
"index_patterns" : [
"filebeat-proftpd-*"
],
"priority" : 3,
"last_updated_time" : 1666126038959
}
]
}
}
Here I have run explain on 000017 which is in the same state as 000020 was.
{
"filebeat-syslog-000017" : {
"aliases" : {
"all_logs" : { },
"filebeat-syslog-write" : {
"is_write_index" : false
}
},
"mappings" : {
"properties" : {
"#timestamp" : {
"type" : "date"
},
"#version" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"agent" : {
"properties" : {
"ephemeral_id" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"hostname" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"id" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"type" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"version" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"apenv" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"app" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"beat" : {
"properties" : {
"hostname" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"version" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"close_inactive" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"ecs" : {
"properties" : {
"version" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"host" : {
"properties" : {
"name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"input" : {
"properties" : {
"type" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"log" : {
"properties" : {
"file" : {
"properties" : {
"path" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"offset" : {
"type" : "long"
}
}
},
"log_category" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"offset" : {
"type" : "long"
},
"prospector" : {
"properties" : {
"type" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"received_at" : {
"type" : "date"
},
"received_from" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"site" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"source" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"syslog_hostname" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"syslog_message" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"syslog_pid" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"syslog_program" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"syslog_timestamp" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"tags" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"timezone" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"settings" : {
"index" : {
"mapping" : {
"total_fields" : {
"limit" : "3000"
},
"ignore_malformed" : "true"
},
"refresh_interval" : "30s",
"translog" : {
"flush_threshold_size" : "1024mb"
},
"plugins" : {
"index_state_management" : {
"rollover_alias" : "filebeat-syslog-write"
}
},
"provided_name" : "filebeat-syslog-000017",
"query" : {
"default_field" : "message"
},
"creation_date" : "1672592177394",
"number_of_replicas" : "1",
"uuid" : "o6UaHr1oQaaDMQHDEe0bkA",
"version" : {
"created" : "135248027"
},
"routing" : {
"allocation" : {
"include" : {
"_tier_preference" : "data_hot"
}
}
},
"number_of_shards" : "3",
"merge" : {
"scheduler" : {
"max_thread_count" : "1"
}
}
}
}
}
}
I'm writing an elasticSearch query against an index that contains lat/long. It is indexed as correct type to work with geoSpatial queries.
I am trying to aggregate places based on the current bounding box in a mapbox map and getting the bounds. The map also has a search box where user search with some string. Combining both the search and geoBounding, I am forming the following query.
{
"from":0,
"size":100,
"track_total_hits":true,
"sort":[
{
"place_name.keyword":"asc"
}
],
"query":{
"bool":{
"must":[
{
"multi_match":{
"query":"w",
"fields":[
"place_name^3",
"properties.top_category",
"properties.brands"
],
"operator":"and"
}
},
{
"geo_bounding_box":{
"location.point":{
"top_right":{
"lat":38.89450183333278,
"lon":-90.38570942514077
},
"bottom_left":{
"lat":38.88102629071099,
"lon":-90.40970118570218
}
}
}
}
]
}
}
}
But the query returns 0 hits.
When I run just the multi_match separately and the geo_bounding_box separately both returns the results as expected. So I'm not sure what I'm missing here.
I tried this as well using one as a filter
{
"from":0,
"size":100,
"track_total_hits":true,
"sort":[
{
"place_name.keyword":"asc"
}
],
"query":{
"bool":{
"must":
{
"multi_match":{
"query":"W",
"fields":[
"place_name^3",
"properties.top_category",
"properties.brands"
],
"operator":"and"
}
},
"filter":{
"geo_bounding_box":{
"location.point":{
"top_right":{
"lat":38.89450183333278,
"lon":-90.38570942514077
},
"bottom_left":{
"lat":38.88102629071099,
"lon":-90.40970118570218
}
}
}
}
}
}
}
Getting same issue. I know the data exists because calling with only the geo_bounding_box returns this data. But not when I combine it with a multi_match, the data should match because the place name matches.
Update: Added the index mapping
{
"places_here_integration" : {
"mappings" : {
"properties" : {
"location" : {
"properties" : {
"DMA_id" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"DMA_name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"county_id" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"county_name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"geo_point" : {
"type" : "geo_point",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"point" : {
"type" : "geo_shape"
},
"polygon" : {
"type" : "geo_shape"
},
"state_name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"state_usps" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"place_name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"properties" : {
"properties" : {
"address" : {
"properties" : {
"city" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"region" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"state" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"street_address" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"zip_code" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"brands" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"created_datetime" : {
"type" : "date"
},
"created_user" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"ids" : {
"properties" : {
"building_id" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"parkingarea_id" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"place_id" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"related_place_ids" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"tenantspace_id" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"naics_code" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"operating_information" : {
"properties" : {
"operating_hours" : {
"properties" : {
"fri" : {
"properties" : {
"close" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"open" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"mon" : {
"properties" : {
"close" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"open" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"sat" : {
"properties" : {
"close" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"open" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"sun" : {
"properties" : {
"close" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"open" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"thu" : {
"properties" : {
"close" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"open" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"tue" : {
"properties" : {
"close" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"open" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"wed" : {
"properties" : {
"close" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"open" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
}
}
},
"operating_hours_note" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"phone_number" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"place_category_name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"sub_category" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"top_category" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"type" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
}
}
}
}
}
I have an index that contains these fields
{
"pluginindex-2022.10.19" : {
"mappings" : {
"doc" : {
"properties" : {
"#timestamp" : {
"type" : "date"
},
"action" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"kafka" : {
"properties" : {
"topic" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"pluginId" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"pluginName" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"pluginVersion" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"timestamp" : {
"type" : "date"
},
"traceId" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"userId" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
}
}
}
}
And I wanna create Visualize like
my target Chart
The abscissa of this picture is the range number of userId grouping aggregation. For example, if there are four items with userId equal to 1 in this index, the range is 0~5.
But now it looks not support import the result of aggregation.
And I try to use alias and its not support aggregate.
Anybody help! A grateful thanks!
I created aggregations includes a terms aggregate with a sum aggregate for each term bucket and will sort on the sum value. This works fine. However if I add a nested terms aggregation
which is the second commented block in the code snippet. It fails with this error message "{"type":"aggregation_execution_exception","reason":"Invalid aggregation order path [AggCPUTimeMS].}" and after I remove the "Order" line it works just fine. Is it not possible to sort the bucket by a sum if there is sub aggregation?
To Clarify what I need:
I want top level bucket group by "System" and I pick top n (this is why I need to order on this level) then inside each "System" bucket I create second level buckets group by "QueryHash". So it is not 2 groups at the same level, it is nested groups.
Another question is about the first commented block: what is the difference between "Order" and "BucketSort"? My guess is "Order" will sort the buckets and pick the top n while "BucketSort" will sort the buckets already picked, randomly or not.
var response = client.Search<SearchResultBucket>(search => search
.Size(0)
.RequestConfiguration(r => r.DisableDirectStreaming())
.Aggregations(aggContainer => aggContainer
.Terms("topLevelAggregation", termsAgg => termsAgg
.Field(new Field("System.keyword"))
.Size(5)
.Aggregations(aggContainer => aggContainer
.Sum("AggCPUTimeMS", sumAgg => sumAgg
.Field(new Field("CpuTimeMilliseconds"))
)
//.BucketSort("sum_bucket_sort", bs => bs
// .Sort(s => s
// .Descending("AggCPUTimeMS")
// )
//)
)
.Order(o => o.Descending("AggCPUTimeMS"))
//.Aggregations(aggContainer => aggContainer
// .Terms("SubAggregation1", termsAgg => termsAgg
// .Field(new Field("QueryHash.keyword"))
// )
//)
)
)
Mapping here:
{
"idx-au2-prod-sqlcpumonitoring-prod-2022.06.17-000043" : {
"mappings" : {
"dynamic_templates" : [
{
"message_field" : {
"path_match" : "message",
"match_mapping_type" : "string",
"mapping" : {
"norms" : false,
"type" : "text"
}
}
},
{
"string_fields" : {
"match" : "*",
"match_mapping_type" : "string",
"mapping" : {
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
},
"norms" : false,
"type" : "text"
}
}
}
],
"properties" : {
"#timestamp" : {
"type" : "date"
},
"#version" : {
"type" : "keyword"
},
"BatchResult" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
},
"norms" : false
},
"ClientAppName" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
},
"norms" : false
},
"ClientHostname" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
},
"norms" : false
},
"ClientProcessId" : {
"type" : "long"
},
"CollectSystemDateUtc" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
},
"norms" : false
},
"CollectSystemTimeUtc" : {
"type" : "date"
},
"CpuTimeMilliseconds" : {
"type" : "long"
},
"CurrentVersion" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
},
"norms" : false
},
"DatabaseName" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
},
"norms" : false
},
"Duration" : {
"type" : "long"
},
"EventName" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
},
"norms" : false
},
"ExeDate" : {
"type" : "date"
},
"IncrementalId" : {
"type" : "long"
},
"LogicalReads" : {
"type" : "long"
},
"Owner" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
},
"norms" : false
},
"OwnerId" : {
"type" : "long"
},
"PhysicalReads" : {
"type" : "long"
},
"Qty" : {
"type" : "long"
},
"QueryHash" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
},
"norms" : false
},
"QueryPlanHash" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
},
"norms" : false
},
"QueryType" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
},
"norms" : false
},
"QueryTypeId" : {
"type" : "long"
},
"ResourceGroupID" : {
"type" : "long"
},
"ResourcePoolID" : {
"type" : "long"
},
"RowCount" : {
"type" : "long"
},
"ServerInstanceName" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
},
"norms" : false
},
"ServerPrincipalName" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
},
"norms" : false
},
"SessionID" : {
"type" : "long"
},
"SourceTableName" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
},
"norms" : false
},
"SqlText" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 32766
}
},
"norms" : false
},
"System" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
},
"norms" : false
},
"SystemId" : {
"type" : "long"
},
"TaskTime" : {
"type" : "long"
},
"Writes" : {
"type" : "long"
},
"geoip" : {
"dynamic" : "true",
"properties" : {
"ip" : {
"type" : "ip"
},
"latitude" : {
"type" : "half_float"
},
"location" : {
"type" : "geo_point"
},
"longitude" : {
"type" : "half_float"
}
}
}
}
}
}
}
document example:
{
"_index": "idx-au2-prod-sqlcpumonitoring-prod-2022.06.17-000043",
"_id": "lDoxiYEBssAtKosd_uKJM",
"_version": 1,
"_score": 1,
"_source": {
"QueryType": "Regular",
"SqlText": "BACKUP LOG #DbName TO DISK = #FilePath WITH INIT, NOSKIP, RETAINDAYS=1, NAME = 'LogBackup', NO_COMPRESSION;\n\t\t\t",
"LogicalReads": 417,
"QueryTypeId": 1,
"QueryHash": "0",
"CollectSystemDateUtc": "2022.06.22",
"DatabaseName": "FO8PRD",
"ResourceGroupID": 280,
"System": "Unknown",
"SessionID": 3027,
"#version": "1",
"Writes": 8,
"ClientAppName": "ServiceRunner",
"ClientProcessId": 40792,
"Owner": "CORE",
"CollectSystemTimeUtc": "2022-06-22T02:02:50.293",
"QueryPlanHash": "0",
"Duration": 1133,
"SystemId": 14,
"Qty": 1,
"#timestamp": "2022-06-22T02:15:45.711Z",
"CpuTimeMilliseconds": 16,
"EventName": "sp_statement_completed",
"PhysicalReads": 99,
"ResourcePoolID": 257,
"TaskTime": 43216568,
"CurrentVersion": "2.17.41",
"RowCount": 8,
"IncrementalId": 18894454167,
"ClientHostname": "SPRC-015",
"ServerInstanceName": "INSTANCE1",
"ExeDate": "2022-06-16T23:00:00+10",
"OwnerId": 3,
"ServerPrincipalName": "Admin"
}
I'm currently trying to find all children of parents that match certain query using the following has_parent query:
GET my_index*/_search
{
"query": {
"has_parent": {
"parent_type": "threat",
"query": {
"term": {
"type.keyword": {
"value": "ip"
}
}
}
}
}
}
But it returns no hits, even with a match_all query.
The mapping of the index is as follows:
"my_index" : {
"mappings" : {
"doc" : {
"properties" : {
"#timestamp" : {
"type" : "date"
},
"#version" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"asn_info" : {
"properties" : {
"as_org" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"asn" : {
"type" : "long"
}
}
},
"campaign" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"category" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"category_description" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"confidence" : {
"type" : "float"
},
"criticity" : {
"type" : "float"
},
"detection_date" : {
"type" : "float"
},
"feed" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"feeds" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"geo" : {
"properties" : {
"city_name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"country_code2" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"country_code3" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"country_name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"latitude" : {
"type" : "float"
},
"longitude" : {
"type" : "float"
}
}
},
"hierarchy" : {
"type" : "join",
"eager_global_ordinals" : true,
"relations" : {
"threat" : "date"
}
},
"host" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"ip" : {
"type" : "long"
},
"ip_address" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"port" : {
"type" : "long"
},
"subcategory" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"tags" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"timestamp" : {
"type" : "date"
},
"type" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
}
}
}
As you can see, the hierarchy field is a join field, with "threat" defined as parent of "date". I don't see any problem with this. Does anyone know what could be happening?