Elasticsearch aggregation on multiple fields across multiple indexes - elasticsearch

I have two indexes - one for Application model and the other for Databases model (many-to-many relational).
Each document is denormalized to contain attributes from the other model
Application
|_ vendor_name
|_ databases
|_ db_1
|_ db_2
Database
|_ database_applications
|_ app_1
|_vendor_name
|_ app_2
|_ vendor_name
Executing a multi-index search for a vendor name - it seems I'm getting the proper results from both Indexes.
The challenge is properly aggregating on the vendor_name field
using the following aggregation seems to work when the result is only from Database. I also tried field: '*vendor_name' but doesn't seem to work.
What am I missing? Should the model be changed?
aggregation:
vendor_name: {
terms: {
field: "database_applications.vendor_name"
}
}
UPDATE 1:
As per #Andrie-Stefan - Here's a more accurate representation of both indexes mappings (abbreviated for shortness):
Database
{
"company-company_databases": {
"aliases": {},
"mappings": {
"company_database": {
"properties": {
"company_applications": {
"properties": {
"application_id": {
"type": "long"
},
"application_name": {
"type": "string"
},
"business_owner": {
"type": "string"
},
"company_system_applications": {
"properties": {
"allow_add_request": {
"type": "string"
},
"allow_remove_request": {
"type": "string"
},
"asset_type": {
"type": "string"
},
"company_application_id": {
"type": "long"
},
"company_application_name": {
"type": "string"
},
"company_business_owner": {
"type": "string"
},
"company_division_id": {
"type": "long"
},
"company_it_app_steward": {
"type": "string"
},
"company_notes": {
"type": "string"
},
"company_system_id": {
"type": "long"
},
"company_vendor": {
"type": "string"
},
"id": {
"type": "long"
},
"it_app_steward": {
"type": "string"
},
"it_owner": {
"type": "string"
},
"last_modified": {
"type": "string"
},
"last_modified_by": {
"type": "string"
},
"media_location": {
"type": "string"
},
"media_source": {
"type": "string"
},
"name": {
"type": "string"
},
"owned_by": {
"type": "string"
},
"status": {
"type": "string"
},
"status_id": {
"type": "long"
},
"system_application": {
"properties": {
"division": {
"type": "string"
},
"id": {
"type": "long"
},
"name": {
"type": "string"
},
"owner_id": {
"type": "string"
},
"status": {
"type": "string"
},
"steward_id": {
"type": "string"
},
"vendor_name": {
"type": "string"
},
"vendor_url_web_site": {
"type": "string"
},
"version": {
"type": "string"
}
}
},
"vendor_name": {
"type": "string",
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed"
}
}
},
"version": {
"type": "string"
}
}
},
"division_id": {
"type": "long"
},
"it_app_steward": {
"type": "string"
},
"notes": {
"type": "string"
},
"software_inventory_id": {
"type": "long"
},
"vendor": {
"type": "string"
}
}
},
"company_instances": {
"properties": {
"business_environment_id": {
"type": "long"
},
"cgi_service_id": {
"type": "long"
},
"char_set": {
"type": "string"
},
"confirmed_license_purchase_dt": {
"type": "string"
},
"company_server": {
"properties": {
"business_environment_id": {
"type": "long"
},
"division_id": {
"type": "long"
},
"domain": {
"type": "string"
},
"hw_platform_id": {
"type": "long"
},
"ip_address": {
"type": "string"
},
"location_id": {
"type": "long"
},
"no_of_cpu": {
"type": "long"
},
"notes": {
"type": "string"
},
"os_platform_id": {
"type": "long"
},
"os_version": {
"type": "string"
},
"server_id": {
"type": "long"
},
"server_name": {
"type": "string"
}
}
},
"description": {
"type": "string"
},
"division_id": {
"type": "long"
},
"edition_id": {
"type": "long"
},
"instance_id": {
"type": "long"
},
"instance_name": {
"type": "string"
},
"itap_have_access": {
"type": "string"
},
"listener_port": {
"type": "long"
},
"notes": {
"type": "string"
},
"patch_number": {
"type": "string"
},
"rdbms_type_id": {
"type": "long"
},
"server_id": {
"type": "long"
},
"service_level_id": {
"type": "long"
},
"version": {
"type": "string"
}
}
},
"db_security_model_id": {
"type": "long"
},
"schema_or_db": {
"type": "string"
},
"schema_or_db_id": {
"type": "long"
},
"schema_or_db_type_id": {
"type": "long"
}
}
}
},
"settings": {
"index": {
"creation_date": "1442976578465",
"uuid": "TxQZoNSpR5qa2Y2ERZzuYw",
"number_of_replicas": "1",
"number_of_shards": "5",
"version": {
"created": "1070299"
}
}
},
"warmers": {}
}
}
Application
{
"applications": {
"aliases": {},
"mappings": {
"application": {
"properties": {
"application_view": {
"properties": {
"app_name": {
"type": "string"
},
"app_status": {
"type": "string"
},
"app_steward_name": {
"type": "string"
},
"app_suite": {
"type": "string"
},
"app_vendor_name": {
"type": "string"
},
"app_version": {
"type": "string"
},
"assignment_group": {
"type": "string"
},
"business_domain_name": {
"type": "string"
},
"exception": {
"type": "string"
},
"id": {
"type": "long"
},
"it_owner_name": {
"type": "string"
},
"service_level": {
"type": "string"
}
}
},
"assignment_group": {
"type": "string"
},
"company_databases": {
"properties": {
"backup_history_info": {
"type": "string"
},
"company_applications": {
"properties": {
"alternate_name": {
"type": "string"
},
"application_id": {
"type": "long"
},
"application_name": {
"type": "string"
},
"business_owner": {
"type": "string"
},
"company_system_applications": {
"properties": {
"aka": {
"type": "string"
},
"allow_add_request": {
"type": "string"
},
"allow_remove_request": {
"type": "string"
},
"asset_type": {
"type": "string"
},
"contract_number": {
"type": "string"
},
"cost_level": {
"type": "string"
},
"company_alternate_name": {
"type": "string"
},
"company_application_id": {
"type": "long"
},
"company_application_name": {
"type": "string"
},
"company_business_owner": {
"type": "string"
},
"company_division_id": {
"type": "long"
},
"company_it_app_steward": {
"type": "string"
},
"company_notes": {
"type": "string"
},
"company_system_id": {
"type": "long"
},
"company_vendor": {
"type": "string"
},
"description": {
"type": "string"
},
"display_in_catalog": {
"type": "string"
},
"id": {
"type": "long"
},
"inform_of_removal": {
"type": "string"
},
"is_restricted": {
"type": "string"
},
"it_app_steward": {
"type": "string"
},
"it_owner": {
"type": "string"
},
"last_modified": {
"type": "string"
},
"last_modified_by": {
"type": "string"
},
"media_location": {
"type": "string"
},
"media_source": {
"type": "string"
},
"name": {
"type": "string"
},
"os_environment": {
"type": "string"
},
"owned_by": {
"type": "string"
},
"retirement_date": {
"type": "date",
"format": "dateOptionalTime"
},
"status": {
"type": "string"
},
"status_id": {
"type": "long"
},
"suite_name": {
"type": "string"
},
"system_application": {
"properties": {
"assignment_group": {
"type": "string"
},
"division": {
"type": "string"
},
"id": {
"type": "long"
},
"name": {
"type": "string"
},
"owner_id": {
"type": "string"
},
"status": {
"type": "string"
},
"steward_id": {
"type": "string"
},
"suite": {
"type": "string"
},
"vendor_name": {
"type": "string"
},
"vendor_url_web_site": {
"type": "string"
},
"version": {
"type": "string"
}
}
},
"vendor_name": {
"type": "string"
},
"version": {
"type": "string"
}
}
},
"division_id": {
"type": "long"
},
"it_app_steward": {
"type": "string"
},
"notes": {
"type": "string"
},
"software_inventory_id": {
"type": "long"
},
"vendor": {
"type": "string"
}
}
},
"company_instances": {
"properties": {
"business_environment_id": {
"type": "long"
},
"cgi_service_id": {
"type": "long"
},
"char_set": {
"type": "string"
},
"confirmed_license_purchase_dt": {
"type": "string"
},
"company_server": {
"properties": {
"business_environment_id": {
"type": "long"
},
"division_id": {
"type": "long"
},
"domain": {
"type": "string"
},
"hw_platform_id": {
"type": "long"
},
"ip_address": {
"type": "string"
},
"location_id": {
"type": "long"
},
"no_of_cpu": {
"type": "long"
},
"notes": {
"type": "string"
},
"os_platform_id": {
"type": "long"
},
"os_version": {
"type": "string"
},
"server_id": {
"type": "long"
},
"server_name": {
"type": "string"
}
}
},
"description": {
"type": "string"
},
"division_id": {
"type": "long"
},
"edition_id": {
"type": "long"
},
"instance_id": {
"type": "long"
},
"instance_name": {
"type": "string"
},
"itap_have_access": {
"type": "string"
},
"listener_port": {
"type": "long"
},
"location_id": {
"type": "long"
},
"notes": {
"type": "string"
},
"patch_number": {
"type": "string"
},
"rdbms_type_id": {
"type": "long"
},
"server_id": {
"type": "long"
},
"service_level_id": {
"type": "long"
},
"version": {
"type": "string"
}
}
},
"db_security_model_id": {
"type": "long"
},
"notes": {
"type": "string"
},
"schema_or_db": {
"type": "string"
},
"schema_or_db_id": {
"type": "long"
},
"schema_or_db_type_id": {
"type": "long"
}
}
},
"division": {
"type": "string",
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed"
}
}
},
"id": {
"type": "long"
},
"name": {
"type": "string"
},
"owner": {
"properties": {
"email_address": {
"type": "string"
},
"id": {
"type": "long"
},
"name": {
"type": "string",
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed"
}
}
},
"search_type": {
"type": "string"
},
"user_id": {
"type": "string"
}
}
},
"owner_id": {
"type": "string"
},
"status": {
"type": "string",
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed"
}
}
},
"steward": {
"properties": {
"email_address": {
"type": "string"
},
"id": {
"type": "long"
},
"name": {
"type": "string",
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed"
}
}
},
"search_type": {
"type": "string"
},
"user_id": {
"type": "string"
}
}
},
"steward_id": {
"type": "string"
},
"suite": {
"type": "string"
},
"vendor_name": {
"type": "string",
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed"
}
}
},
"vendor_url_web_site": {
"type": "string"
},
"version": {
"type": "string"
}
}
}
},
"settings": {
"index": {
"creation_date": "1442970067540",
"uuid": "O7DTaCSESbqhjJpv62T0Wg",
"number_of_replicas": "1",
"number_of_shards": "5",
"version": {
"created": "1070299"
}
}
},
"warmers": {}
}
}

Fields can only be aggregated across indices if they are named alike. There is no wildcard syntax for aggregation fields.
Here is what your mapping currently defines:
INDEX: company-company_databases
TYPE: company_database
FIELD NAMES:
company_applications.company_system_applications.vendor_name
company_applications.company_system_applications.system_application.vendor_name
INDEX: applications
TYPE: application
FIELD NAMES:
company_databases.company_applications.company_system_applications.vendor_name
company_databases.company_applications.company_system_applications.system_application.vendor_name
As far as Elasticsearch is concerned, these fields have nothing in common (even though part of the path is vendor_name).
If your goal is to aggregate vendor_name across a query that spans the two indices, think about restructuring your indices/mappings to accomplish this.
Note that Elasticsearch doesn't model many-to-many relationships
If you can get away with duplicating Database info across applications, you might be able to re-formulate your relationships as a hierarchy, e.g.:
INDEX: applications
--
TYPE: application
FIELDS: vendor_name, etc...
--
TYPE: database_application
FIELDS: vendor_name, databases.<inner fields>, etc...
--
Then you'd be able to aggregate across types on the same field path vendor_name with the added bonus of querying a single applications index.

Related

Problem with rotating (ILM) Cloudflare indices on ELK cluster

The problem I have is that my Cloudflare indices report the following ILM errors:
on index with alias: illegal_argument_exception: rollover target [cloudflare] does not point to a write index
on index without alias: illegal_argument_exception: index.lifecycle.rollover_alias [cloudflare] does not point to index [cloudflare-2022.08.13-000001]
Basically what I was able to find out is that when a new index is created, it doesn't receive the alias from rollover_alias:
{
"settings": {
"index": {
"lifecycle": {
"name": "cloudflare",
"rollover_alias": "cloudflare"
},
option which makes the rollover fail. When I assign the alias manually to all indices affected, rollover and ILM starts to work again but I want to understand why does it happen and find out a permanent solution to this problem. Otherwise I will have to check this cluster manually and force moving the data from HOT to WARM nodes when the HOT storage fills up.
The setup on Cloudflare is based on this guide, in other words Cloudflare pushes the logs to S3 bucket, then AWS Lambda pushes them to ELK (elastic.co).
Cloudflare index template in question:
"cloudflare": {
"index_patterns": [
"cloudflare-*"
],
"mappings": {
"properties": {
"observer.ip": {
"type": "ip"
},
"cloudflare.parent.ray_id": {
"type": "keyword"
},
"cloudflare.worker.subrequest_count": {
"type": "long"
},
"cloudflare.origin.ip": {
"type": "ip"
},
"cloudflare.edge.rate.limit.id": {
"type": "long"
},
"user_agent.version": {
"type": "keyword"
},
"cloudflare.device.type": {
"type": "keyword"
},
"cloudflare.edge.pathing.op": {
"type": "keyword"
},
"user_agent.os.version": {
"type": "keyword"
},
"source.port": {
"type": "long"
},
"cloudflare.edge.server.ip": {
"type": "ip"
},
"cloudflare.security_level": {
"type": "keyword"
},
"observer.vendor": {
"type": "keyword"
},
"event.dataset": {
"type": "keyword"
},
"cloudflare.worker.cpu_time": {
"type": "long"
},
"http.response.status_code": {
"type": "long"
},
"user_agent.minor": {
"type": "keyword"
},
"cloudflare.cache.response.status": {
"type": "long"
},
"user_agent.patch": {
"type": "keyword"
},
"#timestamp": {
"type": "date"
},
"cloudflare.edge.colo.id": {
"type": "integer"
},
"user_agent.os.full": {
"type": "keyword"
},
"source.address": {
"type": "keyword"
},
"user_agent.build": {
"type": "keyword"
},
"source.as.number": {
"type": "long"
},
"cloudflare.edge.start.timestamp": {
"type": "date"
},
"cloudflare.waf.rule.id": {
"type": "keyword"
},
"cloudflare.origin.ssl.protocol": {
"type": "keyword"
},
"http.request.bytes": {
"type": "long"
},
"source.geo.country_iso_code": {
"type": "keyword"
},
"cloudflare.edge.pathing.src": {
"type": "keyword"
},
"cloudflare.edge.response.bytes": {
"type": "long"
},
"cloudflare.edge.response.status": {
"type": "long"
},
"cloudflare.waf.rule.message": {
"type": "keyword"
},
"cloudflare.origin.response.time": {
"type": "long"
},
"url.path": {
"fields": {
"path": {
"index": true,
"eager_global_ordinals": false,
"fielddata": false,
"index_options": "positions",
"index_phrases": false,
"norms": true,
"type": "text",
"store": false
}
},
"type": "keyword"
},
"cloudflare.edge.response.compression_ratio": {
"type": "float"
},
"cloudflare.worker.subrequest": {
"type": "boolean"
},
"cloudflare.cache.response.bytes": {
"type": "long"
},
"cloudflare.waf.profile": {
"type": "keyword"
},
"cloudflare.waf.flags": {
"type": "keyword"
},
"cloudflare.firewall.matches.actions": {
"type": "keyword"
},
"cloudflare.http.response.status_code": {
"type": "long"
},
"user_agent.os.platform": {
"type": "keyword"
},
"cloudflare.waf.matched_var": {
"type": "keyword"
},
"user_agent.os_minor": {
"type": "keyword"
},
"cloudflare.worker.status": {
"type": "keyword"
},
"#version": {
"type": "keyword"
},
"cloudflare.firewall.matches.rule_ids": {
"type": "keyword"
},
"user_agent.os_major": {
"type": "keyword"
},
"cloudflare.origin.response.bytes": {
"type": "long"
},
"source.ip": {
"type": "ip"
},
"http.response.bytes": {
"type": "long"
},
"cloudflare.client.ssl.protocol": {
"type": "keyword"
},
"url.full": {
"type": "keyword"
},
"client.address": {
"type": "keyword"
},
"user_agent.os_name": {
"type": "keyword"
},
"cloudflare.edge.end.timestamp": {
"type": "date"
},
"cloudflare.origin.response.http.last_modified": {
"ignore_malformed": true,
"type": "date"
},
"user_agent.original": {
"type": "keyword"
},
"cloudflare.cache.tiered.fill": {
"type": "boolean"
},
"cloudflare.origin.response.http.expires": {
"type": "date",
"format": "E, d MMM uuuu HH:mm:ss 'UTC'"
},
"user_agent.name": {
"type": "keyword"
},
"cloudflare.waf.action": {
"type": "keyword"
},
"cloudflare.cache.status": {
"type": "keyword"
},
"cloudflare.edge.request.host": {
"type": "keyword"
},
"source.geo": {
"type": "object",
"properties": {
"region_code": {
"type": "keyword"
},
"longitude": {
"type": "float"
},
"region_iso_code": {
"type": "keyword"
},
"region_name": {
"type": "keyword"
},
"country_code2": {
"type": "keyword"
},
"ip": {
"type": "ip"
},
"continent_code": {
"type": "keyword"
},
"postal_code": {
"type": "keyword"
},
"country_code3": {
"type": "keyword"
},
"latitude": {
"type": "float"
},
"city_name": {
"type": "keyword"
},
"dma_code": {
"type": "long"
},
"country_name": {
"type": "keyword"
},
"continent_name": {
"type": "keyword"
},
"timezone": {
"type": "keyword"
},
"location": {
"type": "geo_point"
}
}
},
"cloudflare.edge.rate.limit.action": {
"type": "keyword"
},
"cloudflare.client.ssl.cipher": {
"type": "keyword"
},
"user_agent.os.name": {
"type": "keyword"
},
"cloudflare.edge.pathing.status": {
"type": "keyword"
},
"cloudflare.zone_id": {
"type": "integer"
},
"client.port": {
"type": "long"
},
"observer.type": {
"type": "keyword"
},
"http.request.referrer": {
"type": "keyword"
},
"user_agent.major": {
"type": "keyword"
},
"event.end": {
"type": "date"
},
"cloudflare.client.request.protocol": {
"type": "keyword"
},
"user_agent.device.name": {
"type": "keyword"
},
"destination.ip": {
"type": "ip"
},
"url.domain": {
"type": "keyword"
},
"http.request.method": {
"type": "keyword"
},
"cloudflare.firewall.matches.sources": {
"type": "keyword"
},
"cloudflare.edge.response.content_type": {
"type": "keyword"
},
"cloudflare.ray_id": {
"type": "keyword"
},
"event.start": {
"type": "date"
},
"ecs.version": {
"type": "keyword"
},
"client.ip": {
"type": "ip"
},
"cloudflare.edge.colo.code": {
"type": "keyword"
},
"http.version": {
"type": "keyword"
},
"cloudflare.client.ip.class": {
"type": "keyword"
},
"server.ip": {
"type": "ip"
},
"user_agent.os.kernel": {
"type": "keyword"
}
}
},
"aliases": {},
"order": 0,
"settings": {
"index": {
"number_of_replicas": "1",
"mapping": {
"ignore_malformed": "true"
},
"number_of_shards": "1",
"lifecycle": {
"rollover_alias": "cloudflare",
"name": "cloudflare"
},
"routing": {
"allocation": {
"include": {
"_tier_preference": null
}
}
}
}
}
}
}
ILM Policy in quesion:
{
"cloudflare": {
"policy": {
"phases": {
"cold": {
"actions": {
"set_priority": {
"priority": 0
}
},
"min_age": "30d"
},
"warm": {
"actions": {
"set_priority": {
"priority": 50
}
},
"min_age": "0ms"
},
"hot": {
"actions": {
"rollover": {
"max_age": "1d"
},
"set_priority": {
"priority": 100
}
},
"min_age": "0ms"
},
"delete": {
"actions": {
"delete": {
"delete_searchable_snapshot": true
}
},
"min_age": "60d"
}
}
},
"modified_date": "2021-11-02T17:18:34.417Z",
"in_use_by": {
"indices": [
"cloudflare-2022.07.09-000001",
"cloudflare-2022.07.08-000001",
"cloudflare-2022.07.04-000001",
"cloudflare-2022.07.06-000001",
"cloudflare-2022.07.07-000001",
"cloudflare-2022.07.05-000001",
"cloudflare-2022.06.10-000001",
"cloudflare-2022.06.12-000001",
"cloudflare-2022.06.11-000001",
"cloudflare-2022.06.13-000001",
"cloudflare-2022.08.02-000001",
"cloudflare-2022.08.03-000001",
"cloudflare-2022.08.01-000001",
"cloudflare-2022.08.04-000001",
"cloudflare-2022.08.08-000001",
"cloudflare-2022.06.18-000001",
"cloudflare-2022.08.06-000001",
"cloudflare-2022.06.07-000001",
"cloudflare-2022.06.16-000001",
"cloudflare-2022.06.14-000001",
"cloudflare-2022.06.09-000001",
"cloudflare-2022.06.05-000001",
"cloudflare-2022.06.03-000001",
"cloudflare-2022.05.23-000001",
"cloudflare-2022.05.21-000001",
"cloudflare-2022.07.02-000001",
"cloudflare-2022.07.11-000001",
"cloudflare-2022.07.13-000001",
"cloudflare-2022.08.01-000017",
"cloudflare-2022.07.17-000001",
"cloudflare-2022.07.18-000001",
"cloudflare-2022.05.28-000001",
"cloudflare-2022.05.27-000001",
"cloudflare-2022.05.24-000001",
"cloudflare-2022.06.01-000001",
"cloudflare-2022.06.22-000001",
"cloudflare-2022.08.02-000023",
"cloudflare-2022.08.03-000024",
"cloudflare-2022.08.02-000021",
"cloudflare-2022.06.23-000001",
"cloudflare-2022.08.02-000022",
"cloudflare-2022.08.12-000001",
"cloudflare-2022.08.06-000027",
"cloudflare-2022.08.13-000001",
"cloudflare-2022.08.07-000028",
"cloudflare-2022.06.19-000001",
"cloudflare-2022.08.16-000001",
"cloudflare-2022.06.26-000001",
"cloudflare-2022.08.09-000001",
"cloudflare-2022.08.05-000001",
"cloudflare-2022.08.02-000020",
"cloudflare-2022.06.15-000001",
"cloudflare-2022.05.20-000001",
"cloudflare-2022.06.08-000001",
"cloudflare-2022.07.10-000001",
"cloudflare-2022.06.04-000001",
"cloudflare-2022.07.03-000001",
"cloudflare-2022.05.31-000001",
"cloudflare-2022.07.14-000001",
"cloudflare-2022.07.25-000004",
"cloudflare-2022.07.21-000001",
"cloudflare-2022.07.25-000001",
"cloudflare-2022.08.02-000018",
"cloudflare-2022.08.02-000019",
"cloudflare-2022.07.29-000001",
"cloudflare-2022.07.26-000001",
"cloudflare-2022.07.27-000009",
"cloudflare-2022.07.30-000015",
"cloudflare-2022.07.30-000014",
"cloudflare-2022.07.31-000016",
"cloudflare-2022.07.30-000013",
"cloudflare-2022.07.27-000010",
"cloudflare-2022.06.30-000001",
"cloudflare-2022.07.28-000011",
"cloudflare-2022.08.17-000001",
"cloudflare-2022.07.29-000012",
"cloudflare-2022.06.27-000001",
"cloudflare-2022.06.29-000001",
"cloudflare-2022.06.25-000001",
"cloudflare-2022.05.30-000001",
"cloudflare-2022.07.26-000008",
"cloudflare-2022.07.22-000001",
"cloudflare-2022.07.26-000007",
"cloudflare-2022.07.31-000001",
"cloudflare-2022.07.26-000006",
"cloudflare-2022.07.24-000001",
"cloudflare-2022.07.26-000005",
"cloudflare-2022.07.20-000001",
"cloudflare-2022.07.24-000003",
"cloudflare-2022.07.28-000001",
"cloudflare-2022.05.29-000001",
"cloudflare-2022.07.16-000001",
"cloudflare-2022.07.19-000001",
"cloudflare-2022.07.15-000001",
"cloudflare-2022.08.09-000030",
"cloudflare-2022.05.25-000001",
"cloudflare-2022.05.26-000001",
"cloudflare-2022.06.02-000001",
"cloudflare-2022.06.21-000001",
"cloudflare-2022.06.20-000001",
"cloudflare-2022.06.24-000001",
"cloudflare-2022.08.05-000026",
"cloudflare-2022.08.04-000025",
"cloudflare-2022.08.14-000001",
"cloudflare-2022.08.10-000001",
"cloudflare-2022.08.15-000001",
"cloudflare-2022.08.11-000001",
"cloudflare-2022.08.08-000029",
"cloudflare-2022.08.07-000001",
"cloudflare-2022.06.28-000001",
"cloudflare-2022.06.17-000001",
"cloudflare-2022.06.06-000001",
"cloudflare-2022.05.22-000001",
"cloudflare-2022.07.01-000001",
"cloudflare-2022.07.12-000001",
"cloudflare-2022.07.30-000001",
"cloudflare-2022.07.27-000001",
"cloudflare-2022.07.23-000001",
"cloudflare-2022.07.23-000002"
],
"data_streams": [],
"composable_templates": []
},
"version": 12
}
}
Elastic version: v7.16.2 provided by elastic.co on AWS

Elastic Search Multiple fields filter

when it comes to elasticsearch, i am new to it. That's why, my question is if i'd like to filter multiple fields, would aggregate might solve the problem? or what should i use?
{
"aggs": {
"filtered": {
"filter": {
"bool": {
"must": [
{ "term" : {"shop_slug" : "sharkys-dhammazedi.6326"}},
{ "term" :
{"slug" : "anchovy-essence"}}
]
}
}
}
}}
HERE Is The Mapping
"product_catalogue": {
"mappings": {
"products": {
"properties": {
"buy_online": {
"type": "long"
},
"category": {
"properties": {
"id": {
"type": "long"
},
"name": {
"type": "string"
},
"slug": {
"type": "string"
},
"tree": {
"properties": {
"2": {
"type": "string"
},
"3": {
"type": "string"
},
"5": {
"type": "string"
},
"10": {
"type": "string"
},
"11": {
"type": "string"
},
"12": {
"type": "string"
},
"13": {
"type": "string"
},
"14": {
"type": "string"
},
"15": {
"type": "string"
},
"16": {
"type": "string"
},
"17": {
"type": "string"
},
"18": {
"type": "string"
},
"19": {
"type": "string"
},
"20": {
"type": "string"
},
"21": {
"type": "string"
},
"22": {
"type": "string"
},
"23": {
"type": "string"
},
"24": {
"type": "string"
},
"25": {
"type": "string"
},
"26": {
"type": "string"
},
"27": {
"type": "string"
},
"28": {
"type": "string"
},
"29": {
"type": "string"
},
"30": {
"type": "string"
},
"31": {
"type": "string"
},
"32": {
"type": "string"
},
"33": {
"type": "string"
},
"34": {
"type": "string"
},
"35": {
"type": "string"
},
"36": {
"type": "string"
},
"37": {
"type": "string"
},
"38": {
"type": "string"
},
"39": {
"type": "string"
},
"40": {
"type": "string"
},
"41": {
"type": "string"
},
"42": {
"type": "string"
},
"43": {
"type": "string"
},
"46": {
"type": "string"
},
"62": {
"type": "string"
},
"72": {
"type": "string"
},
"73": {
"type": "string"
},
"74": {
"type": "string"
},
"75": {
"type": "string"
},
"77": {
"type": "string"
},
"137": {
"type": "string"
},
"139": {
"type": "string"
},
"140": {
"type": "string"
},
"141": {
"type": "string"
}
}
}
}
},
"click_collect": {
"type": "long"
},
"company_id": {
"type": "string"
},
"company_name": {
"type": "string"
},
"company_slug": {
"type": "string",
"index": "not_analyzed"
},
"condition": {
"type": "string"
},
"created_at": {
"type": "date",
"format": "dateOptionalTime"
},
"currency_iso": {
"type": "string"
},
"delivery_available": {
"type": "boolean"
},
"delivery_type": {
"properties": {
"id": {
"type": "long"
},
"name": {
"type": "string"
},
"pivot": {
"properties": {
"deliverytype_id": {
"type": "long"
},
"pricing": {
"type": "string"
},
"shop_id": {
"type": "long"
}
}
}
}
},
"description": {
"type": "string",
"analyzer": "snowball"
},
"group_id": {
"type": "string"
},
"image_url": {
"type": "string"
},
"listed_shop": {
"type": "integer"
},
"location": {
"type": "geo_point"
},
"mainPhoto": {
"type": "string"
},
"mm_description": {
"type": "string"
},
"mm_title": {
"type": "string"
},
"postcode": {
"type": "string"
},
"prices": {
"properties": {
"retail": {
"properties": {
"converted": {
"properties": {
"currencyIso": {
"type": "string"
},
"decimal": {
"type": "string"
},
"formatted": {
"type": "string"
}
}
},
"original": {
"properties": {
"currencyIso": {
"type": "string"
},
"decimal": {
"type": "string"
},
"formatted": {
"type": "string"
}
}
}
}
}
}
},
"shop_category": {
"type": "long"
},
"shop_id": {
"type": "long"
},
"shop_name": {
"type": "string"
},
"shop_slug": {
"type": "string",
"index": "not_analyzed"
},
"shop_stock_count": {
"type": "integer"
},
"shop_type": {
"properties": {
"id": {
"type": "long"
},
"name": {
"type": "string"
}
}
},
"slug": {
"type": "string"
},
"title": {
"type": "string",
"analyzer": "snowball"
},
"user_stock_count": {
"type": "integer"
},
"variant": {
"properties": {
"Colour": {
"type": "string"
},
"Disk Space": {
"type": "string"
},
"Length": {
"type": "string"
}
}
}
}
}
}
}
This query didn't seems to work out . Infact, it only filter one of the fields but can't resolve to filter both fields which is shop_slug and slug. Please kindly answer Thanks.

How to aggregate data with filter on text fields in elasticsearch?

Having read all the tutorials on ES website, I cannot achieve my goal.
We are using ES 1.7.6 and I want to have only one single instance of documents matching my criteria. But what I get from ES is all the data matching the filter and the aggregation statistics.
GET _search
{
"size":1000,
"query":{
"bool":{
"should":[
{
"match":{
"isoMessage.fields.39":{
"query":"00"
}
}
}
]
}
},
"aggs":{
"group_by_CATI":{
"terms":{
"field":"isoMessage.fields.41"
}
}
}
}
Note that the index of isoMessage.fields.41 is set to not_analyzed.
Thanks for any help;
UPDATE: The mapping
{
"cm": {
"mappings": {
"Event": {
"properties": {
"deleted": {
"type": "boolean"
},
"id": {
"type": "string"
},
"isoMessage": {
"properties": {
"fields": {
"properties": {
"2": {
"type": "string"
},
"3": {
"type": "string"
},
"4": {
"type": "string"
},
"7": {
"type": "string"
},
"11": {
"type": "string"
},
"12": {
"type": "string"
},
"13": {
"type": "string"
},
"14": {
"type": "string"
},
"22": {
"type": "string"
},
"25": {
"type": "string"
},
"32": {
"type": "string"
},
"37": {
"type": "string"
},
"39": {
"type": "string"
},
"41": {
"type": "string"
},
"42": {
"type": "string"
},
"48": {
"type": "string"
},
"49": {
"type": "string"
},
"60": {
"type": "string"
},
"63": {
"type": "string"
},
"100": {
"type": "string"
},
"128": {
"type": "string"
}
}
},
"isReversal": {
"type": "boolean"
},
"isReversalDone": {
"type": "boolean"
},
"messageSpec": {
"type": "string"
},
"mti": {
"type": "string"
},
"request": {
"type": "boolean"
},
"response": {
"type": "boolean"
}
}
},
"msg": {
"type": "string"
},
"occurDate": {
"type": "long"
},
"receiver": {
"type": "string"
},
"rrn": {
"type": "string"
},
"sender": {
"type": "string",
"index": "not_analyzed"
},
"txId": {
"type": "string"
},
"version": {
"type": "long"
}
}
}
}
}
}
If I understand correctly, and you want to get only the aggregation result:
Change your size field from:
"size":1000
to
"size":0
in order to set the _search query displayed results limit. It won't affect the aggregation result, though.

Elastic search nested document mapping

I created my index and mapping, but appears that the mapping for nested documents is changed after I start adding documents to the index.
Before I start adding documents the mapping is:
{"products_1_1": {
"mappings": {
"product": {
"properties": {
"description": {
"type": "string"
},
"metaDescription": {
"type": "string"
},
"metaTitle": {
"type": "string"
},
"mis_spells": {
"type": "string"
},
"name": {
"type": "string"
},
"productId": {
"type": "integer"
},
"categories": {
"type": "nested",
"include_in_parent": true,
"properties": {
"default_category": {
"type": "integer",
"index": "no"
},
"filter_name": {
"type": "string",
"index": "not_analyzed"
},
"id": {
"type": "integer"
},
"name": {
"type": "string"
},
"parent_categories_ids": {
"type": "string",
"index": "no"
},
"parent_categories_names": {
"type": "string",
"index": "no"
},
"parent_categories_url": {
"type": "string",
"index": "no"
},
"parent_category": {
"type": "integer",
"index": "no"
},
"tags": {
"type": "string"
},
"url": {
"type": "string",
"index": "no"
}
}
}
}
}
}
}
}
After I start inserting documents the mapping becomes:
{"products_1_1": {
"mappings": {
"product": {
"properties": {
"description": {
"type": "string"
},
"metaDescription": {
"type": "string"
},
"metaTitle": {
"type": "string"
},
"mis_spells": {
"type": "string"
},
"name": {
"type": "string"
},
"productId": {
"type": "integer"
},
"categories": {
"type": "nested",
"include_in_parent": true,
"properties": {
"0": {
"properties": {
"default_category": {
"type": "long"
},
"filter_name": {
"type": "string"
},
"id": {
"type": "long"
},
"name": {
"type": "string"
},
"parent_categories_ids": {
"type": "string"
},
"parent_categories_names": {
"type": "string"
},
"parent_categories_url": {
"type": "string"
},
"parent_category": {
"type": "long"
},
"tags": {
"type": "string"
},
"url": {
"type": "string"
}
}
},
"1": {
"properties": {
"default_category": {
"type": "long"
},
"filter_name": {
"type": "string"
},
"id": {
"type": "long"
},
"name": {
"type": "string"
},
"parent_categories_ids": {
"type": "string"
},
"parent_categories_names": {
"type": "string"
},
"parent_categories_url": {
"type": "string"
},
"parent_category": {
"type": "long"
},
"tags": {
"type": "string"
},
"url": {
"type": "string"
}
}
},
"2": {
"properties": {
"default_category": {
"type": "long"
},
"filter_name": {
"type": "string"
},
"id": {
"type": "long"
},
"name": {
"type": "string"
},
"parent_categories_ids": {
"type": "string"
},
"parent_categories_names": {
"type": "string"
},
"parent_categories_url": {
"type": "string"
},
"parent_category": {
"type": "long"
},
"tags": {
"type": "string"
},
"url": {
"type": "string"
}
}
},
...
"default_category": {
"type": "integer",
"index": "no"
},
"filter_name": {
"type": "string",
"index": "not_analyzed"
},
"id": {
"type": "integer"
},
"name": {
"type": "string"
},
"parent_categories_ids": {
"type": "string",
"index": "no"
},
"parent_categories_names": {
"type": "string",
"index": "no"
},
"parent_categories_url": {
"type": "string",
"index": "no"
},
"parent_category": {
"type": "integer",
"index": "no"
},
"tags": {
"type": "string"
},
"url": {
"type": "string",
"index": "no"
}
}
}
}
}
}
}
}
Does anybody have any idea why my mapping will be altered this way?
Thanks!

elasticsearch equivalent of join

I'm new to elastic search and I'm trying to figure out how to do the equivalent of a SQL join. Here are my 2 mappings:
{
"home_panel": {
"mappings": {
"levis": {
"properties": {
"created_at": {
"type": "date",
"format": "dateOptionalTime"
},
"current": {
"type": "string"
},
"event_uuid": {
"type": "string"
},
"panels": {
"properties": {
"action": {
"type": "string"
},
"heightratio": {
"type": "double"
},
"type": {
"type": "string"
},
"url": {
"type": "string"
},
"videourl": {
"type": "string"
}
}
},
"status": {
"type": "string"
},
"updated_at": {
"type": "date",
"format": "dateOptionalTime"
},
"uuid": {
"type": "string"
}
}
}
}
}
}
And:
{
"event": {
"mappings": {
"levis": {
"properties": {
"date": {
"type": "date",
"format": "dateOptionalTime"
},
"event_uuid": {
"type": "long"
},
"name": {
"type": "string"
},
"ticketmaster_game_event_name": {
"type": "string"
},
"ticketmaster_parking_event_name": {
"type": "string"
},
"time": {
"type": "date",
"format": "dateOptionalTime"
}
}
}
}
}
}
I'd ideally like to do something like select * from HomePanel InnerJoin Event On home_panel.event_uuid = event.event_uuid so I can see the event in the home panel object. I'm not clear how to do this in elastic search. Is this even possible?

Resources