Related
The problem I have is that my Cloudflare indices report the following ILM errors:
on index with alias: illegal_argument_exception: rollover target [cloudflare] does not point to a write index
on index without alias: illegal_argument_exception: index.lifecycle.rollover_alias [cloudflare] does not point to index [cloudflare-2022.08.13-000001]
Basically what I was able to find out is that when a new index is created, it doesn't receive the alias from rollover_alias:
{
"settings": {
"index": {
"lifecycle": {
"name": "cloudflare",
"rollover_alias": "cloudflare"
},
option which makes the rollover fail. When I assign the alias manually to all indices affected, rollover and ILM starts to work again but I want to understand why does it happen and find out a permanent solution to this problem. Otherwise I will have to check this cluster manually and force moving the data from HOT to WARM nodes when the HOT storage fills up.
The setup on Cloudflare is based on this guide, in other words Cloudflare pushes the logs to S3 bucket, then AWS Lambda pushes them to ELK (elastic.co).
Cloudflare index template in question:
"cloudflare": {
"index_patterns": [
"cloudflare-*"
],
"mappings": {
"properties": {
"observer.ip": {
"type": "ip"
},
"cloudflare.parent.ray_id": {
"type": "keyword"
},
"cloudflare.worker.subrequest_count": {
"type": "long"
},
"cloudflare.origin.ip": {
"type": "ip"
},
"cloudflare.edge.rate.limit.id": {
"type": "long"
},
"user_agent.version": {
"type": "keyword"
},
"cloudflare.device.type": {
"type": "keyword"
},
"cloudflare.edge.pathing.op": {
"type": "keyword"
},
"user_agent.os.version": {
"type": "keyword"
},
"source.port": {
"type": "long"
},
"cloudflare.edge.server.ip": {
"type": "ip"
},
"cloudflare.security_level": {
"type": "keyword"
},
"observer.vendor": {
"type": "keyword"
},
"event.dataset": {
"type": "keyword"
},
"cloudflare.worker.cpu_time": {
"type": "long"
},
"http.response.status_code": {
"type": "long"
},
"user_agent.minor": {
"type": "keyword"
},
"cloudflare.cache.response.status": {
"type": "long"
},
"user_agent.patch": {
"type": "keyword"
},
"#timestamp": {
"type": "date"
},
"cloudflare.edge.colo.id": {
"type": "integer"
},
"user_agent.os.full": {
"type": "keyword"
},
"source.address": {
"type": "keyword"
},
"user_agent.build": {
"type": "keyword"
},
"source.as.number": {
"type": "long"
},
"cloudflare.edge.start.timestamp": {
"type": "date"
},
"cloudflare.waf.rule.id": {
"type": "keyword"
},
"cloudflare.origin.ssl.protocol": {
"type": "keyword"
},
"http.request.bytes": {
"type": "long"
},
"source.geo.country_iso_code": {
"type": "keyword"
},
"cloudflare.edge.pathing.src": {
"type": "keyword"
},
"cloudflare.edge.response.bytes": {
"type": "long"
},
"cloudflare.edge.response.status": {
"type": "long"
},
"cloudflare.waf.rule.message": {
"type": "keyword"
},
"cloudflare.origin.response.time": {
"type": "long"
},
"url.path": {
"fields": {
"path": {
"index": true,
"eager_global_ordinals": false,
"fielddata": false,
"index_options": "positions",
"index_phrases": false,
"norms": true,
"type": "text",
"store": false
}
},
"type": "keyword"
},
"cloudflare.edge.response.compression_ratio": {
"type": "float"
},
"cloudflare.worker.subrequest": {
"type": "boolean"
},
"cloudflare.cache.response.bytes": {
"type": "long"
},
"cloudflare.waf.profile": {
"type": "keyword"
},
"cloudflare.waf.flags": {
"type": "keyword"
},
"cloudflare.firewall.matches.actions": {
"type": "keyword"
},
"cloudflare.http.response.status_code": {
"type": "long"
},
"user_agent.os.platform": {
"type": "keyword"
},
"cloudflare.waf.matched_var": {
"type": "keyword"
},
"user_agent.os_minor": {
"type": "keyword"
},
"cloudflare.worker.status": {
"type": "keyword"
},
"#version": {
"type": "keyword"
},
"cloudflare.firewall.matches.rule_ids": {
"type": "keyword"
},
"user_agent.os_major": {
"type": "keyword"
},
"cloudflare.origin.response.bytes": {
"type": "long"
},
"source.ip": {
"type": "ip"
},
"http.response.bytes": {
"type": "long"
},
"cloudflare.client.ssl.protocol": {
"type": "keyword"
},
"url.full": {
"type": "keyword"
},
"client.address": {
"type": "keyword"
},
"user_agent.os_name": {
"type": "keyword"
},
"cloudflare.edge.end.timestamp": {
"type": "date"
},
"cloudflare.origin.response.http.last_modified": {
"ignore_malformed": true,
"type": "date"
},
"user_agent.original": {
"type": "keyword"
},
"cloudflare.cache.tiered.fill": {
"type": "boolean"
},
"cloudflare.origin.response.http.expires": {
"type": "date",
"format": "E, d MMM uuuu HH:mm:ss 'UTC'"
},
"user_agent.name": {
"type": "keyword"
},
"cloudflare.waf.action": {
"type": "keyword"
},
"cloudflare.cache.status": {
"type": "keyword"
},
"cloudflare.edge.request.host": {
"type": "keyword"
},
"source.geo": {
"type": "object",
"properties": {
"region_code": {
"type": "keyword"
},
"longitude": {
"type": "float"
},
"region_iso_code": {
"type": "keyword"
},
"region_name": {
"type": "keyword"
},
"country_code2": {
"type": "keyword"
},
"ip": {
"type": "ip"
},
"continent_code": {
"type": "keyword"
},
"postal_code": {
"type": "keyword"
},
"country_code3": {
"type": "keyword"
},
"latitude": {
"type": "float"
},
"city_name": {
"type": "keyword"
},
"dma_code": {
"type": "long"
},
"country_name": {
"type": "keyword"
},
"continent_name": {
"type": "keyword"
},
"timezone": {
"type": "keyword"
},
"location": {
"type": "geo_point"
}
}
},
"cloudflare.edge.rate.limit.action": {
"type": "keyword"
},
"cloudflare.client.ssl.cipher": {
"type": "keyword"
},
"user_agent.os.name": {
"type": "keyword"
},
"cloudflare.edge.pathing.status": {
"type": "keyword"
},
"cloudflare.zone_id": {
"type": "integer"
},
"client.port": {
"type": "long"
},
"observer.type": {
"type": "keyword"
},
"http.request.referrer": {
"type": "keyword"
},
"user_agent.major": {
"type": "keyword"
},
"event.end": {
"type": "date"
},
"cloudflare.client.request.protocol": {
"type": "keyword"
},
"user_agent.device.name": {
"type": "keyword"
},
"destination.ip": {
"type": "ip"
},
"url.domain": {
"type": "keyword"
},
"http.request.method": {
"type": "keyword"
},
"cloudflare.firewall.matches.sources": {
"type": "keyword"
},
"cloudflare.edge.response.content_type": {
"type": "keyword"
},
"cloudflare.ray_id": {
"type": "keyword"
},
"event.start": {
"type": "date"
},
"ecs.version": {
"type": "keyword"
},
"client.ip": {
"type": "ip"
},
"cloudflare.edge.colo.code": {
"type": "keyword"
},
"http.version": {
"type": "keyword"
},
"cloudflare.client.ip.class": {
"type": "keyword"
},
"server.ip": {
"type": "ip"
},
"user_agent.os.kernel": {
"type": "keyword"
}
}
},
"aliases": {},
"order": 0,
"settings": {
"index": {
"number_of_replicas": "1",
"mapping": {
"ignore_malformed": "true"
},
"number_of_shards": "1",
"lifecycle": {
"rollover_alias": "cloudflare",
"name": "cloudflare"
},
"routing": {
"allocation": {
"include": {
"_tier_preference": null
}
}
}
}
}
}
}
ILM Policy in quesion:
{
"cloudflare": {
"policy": {
"phases": {
"cold": {
"actions": {
"set_priority": {
"priority": 0
}
},
"min_age": "30d"
},
"warm": {
"actions": {
"set_priority": {
"priority": 50
}
},
"min_age": "0ms"
},
"hot": {
"actions": {
"rollover": {
"max_age": "1d"
},
"set_priority": {
"priority": 100
}
},
"min_age": "0ms"
},
"delete": {
"actions": {
"delete": {
"delete_searchable_snapshot": true
}
},
"min_age": "60d"
}
}
},
"modified_date": "2021-11-02T17:18:34.417Z",
"in_use_by": {
"indices": [
"cloudflare-2022.07.09-000001",
"cloudflare-2022.07.08-000001",
"cloudflare-2022.07.04-000001",
"cloudflare-2022.07.06-000001",
"cloudflare-2022.07.07-000001",
"cloudflare-2022.07.05-000001",
"cloudflare-2022.06.10-000001",
"cloudflare-2022.06.12-000001",
"cloudflare-2022.06.11-000001",
"cloudflare-2022.06.13-000001",
"cloudflare-2022.08.02-000001",
"cloudflare-2022.08.03-000001",
"cloudflare-2022.08.01-000001",
"cloudflare-2022.08.04-000001",
"cloudflare-2022.08.08-000001",
"cloudflare-2022.06.18-000001",
"cloudflare-2022.08.06-000001",
"cloudflare-2022.06.07-000001",
"cloudflare-2022.06.16-000001",
"cloudflare-2022.06.14-000001",
"cloudflare-2022.06.09-000001",
"cloudflare-2022.06.05-000001",
"cloudflare-2022.06.03-000001",
"cloudflare-2022.05.23-000001",
"cloudflare-2022.05.21-000001",
"cloudflare-2022.07.02-000001",
"cloudflare-2022.07.11-000001",
"cloudflare-2022.07.13-000001",
"cloudflare-2022.08.01-000017",
"cloudflare-2022.07.17-000001",
"cloudflare-2022.07.18-000001",
"cloudflare-2022.05.28-000001",
"cloudflare-2022.05.27-000001",
"cloudflare-2022.05.24-000001",
"cloudflare-2022.06.01-000001",
"cloudflare-2022.06.22-000001",
"cloudflare-2022.08.02-000023",
"cloudflare-2022.08.03-000024",
"cloudflare-2022.08.02-000021",
"cloudflare-2022.06.23-000001",
"cloudflare-2022.08.02-000022",
"cloudflare-2022.08.12-000001",
"cloudflare-2022.08.06-000027",
"cloudflare-2022.08.13-000001",
"cloudflare-2022.08.07-000028",
"cloudflare-2022.06.19-000001",
"cloudflare-2022.08.16-000001",
"cloudflare-2022.06.26-000001",
"cloudflare-2022.08.09-000001",
"cloudflare-2022.08.05-000001",
"cloudflare-2022.08.02-000020",
"cloudflare-2022.06.15-000001",
"cloudflare-2022.05.20-000001",
"cloudflare-2022.06.08-000001",
"cloudflare-2022.07.10-000001",
"cloudflare-2022.06.04-000001",
"cloudflare-2022.07.03-000001",
"cloudflare-2022.05.31-000001",
"cloudflare-2022.07.14-000001",
"cloudflare-2022.07.25-000004",
"cloudflare-2022.07.21-000001",
"cloudflare-2022.07.25-000001",
"cloudflare-2022.08.02-000018",
"cloudflare-2022.08.02-000019",
"cloudflare-2022.07.29-000001",
"cloudflare-2022.07.26-000001",
"cloudflare-2022.07.27-000009",
"cloudflare-2022.07.30-000015",
"cloudflare-2022.07.30-000014",
"cloudflare-2022.07.31-000016",
"cloudflare-2022.07.30-000013",
"cloudflare-2022.07.27-000010",
"cloudflare-2022.06.30-000001",
"cloudflare-2022.07.28-000011",
"cloudflare-2022.08.17-000001",
"cloudflare-2022.07.29-000012",
"cloudflare-2022.06.27-000001",
"cloudflare-2022.06.29-000001",
"cloudflare-2022.06.25-000001",
"cloudflare-2022.05.30-000001",
"cloudflare-2022.07.26-000008",
"cloudflare-2022.07.22-000001",
"cloudflare-2022.07.26-000007",
"cloudflare-2022.07.31-000001",
"cloudflare-2022.07.26-000006",
"cloudflare-2022.07.24-000001",
"cloudflare-2022.07.26-000005",
"cloudflare-2022.07.20-000001",
"cloudflare-2022.07.24-000003",
"cloudflare-2022.07.28-000001",
"cloudflare-2022.05.29-000001",
"cloudflare-2022.07.16-000001",
"cloudflare-2022.07.19-000001",
"cloudflare-2022.07.15-000001",
"cloudflare-2022.08.09-000030",
"cloudflare-2022.05.25-000001",
"cloudflare-2022.05.26-000001",
"cloudflare-2022.06.02-000001",
"cloudflare-2022.06.21-000001",
"cloudflare-2022.06.20-000001",
"cloudflare-2022.06.24-000001",
"cloudflare-2022.08.05-000026",
"cloudflare-2022.08.04-000025",
"cloudflare-2022.08.14-000001",
"cloudflare-2022.08.10-000001",
"cloudflare-2022.08.15-000001",
"cloudflare-2022.08.11-000001",
"cloudflare-2022.08.08-000029",
"cloudflare-2022.08.07-000001",
"cloudflare-2022.06.28-000001",
"cloudflare-2022.06.17-000001",
"cloudflare-2022.06.06-000001",
"cloudflare-2022.05.22-000001",
"cloudflare-2022.07.01-000001",
"cloudflare-2022.07.12-000001",
"cloudflare-2022.07.30-000001",
"cloudflare-2022.07.27-000001",
"cloudflare-2022.07.23-000001",
"cloudflare-2022.07.23-000002"
],
"data_streams": [],
"composable_templates": []
},
"version": 12
}
}
Elastic version: v7.16.2 provided by elastic.co on AWS
I am totally new to elastic search. So please forgive me if this is a stupid Question and my Questions might have been answered somewhere else already but I couldn't find it.
I want to use Elastic Search as a search engine for PDF'S and docx's in my network. I used fscrawler to ingest the PDF's to elastic search. Since the documents I want to ingest are in several languages I wanted to use n-graming for stemming. To do so I wanted to update my mapping like this
PUT test/_mappings/_all
{
"mappings": {
"title": {
"properties": {
"title": {
"type": "text",
"fields": {
"de": {
"type": "string",
"analyzer": "german"
},
"en": {
"type": "string",
"analyzer": "english"
},
"general": {
"type": "string",
"analyzer": "trigrams"
}
}
}
}
}
}
}
And now I get this Errormessage
{ "error": {
"root_cause": [
{
"type": "mapper_parsing_exception",
"reason": "Root mapping definition has unsupported parameters: [mappings : {title={properties={title={type=text,
fields={de={type=string, analyzer=german}, en={type=string,
analyzer=english}, general={type=string, analyzer=trigrams}}}}}}]"
}
],
"type": "mapper_parsing_exception",
"reason": "Root mapping definition has unsupported parameters: [mappings : {title={properties={title={type=text,
fields={de={type=string, analyzer=german}, en={type=string,
analyzer=english}, general={type=string, analyzer=trigrams}}}}}}]"
}, "status": 400 }
Do you have any idea how i can fix this? Or do you have an idea how I can ingest the files with the right mapping without using fscrawler?
those are my settings
{
"test": {
"settings": {
"index": {
"mapping": {
"total_fields": {
"limit": "2000"
}
},
"number_of_shards": "5",
"provided_name": "test",
"creation_date": "1542031632596",
"analysis": {
"filter": {
"trigrams_filter": {
"type": "ngram",
"min_gram": "3",
"max_gram": "3"
}
},
"analyzer": {
"fscrawler_path": {
"tokenizer": "fscrawler_path"
},
"trigrams": {
"filter": [
"lowercase",
"trigrams_filter"
],
"type": "custom",
"tokenizer": "standard"
}
},
"tokenizer": {
"fscrawler_path": {
"type": "path_hierarchy"
}
}
},
"number_of_replicas": "1",
"uuid": "7L3QE5_xRACECVbTFlFY-Q",
"version": {
"created": "6040399"
}
}
}
}
}
My mapping
{
"test": {
"mappings": {
"_doc": {
"dynamic_templates": [
{
"raw_as_text": {
"path_match": "meta.raw.*",
"mapping": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
}
}
}
],
"properties": {
"attachment": {
"type": "binary"
},
"attributes": {
"properties": {
"group": {
"type": "keyword"
},
"owner": {
"type": "keyword"
}
}
},
"content": {
"type": "text"
},
"file": {
"properties": {
"checksum": {
"type": "keyword"
},
"content_type": {
"type": "keyword"
},
"created": {
"type": "date",
"format": "dateOptionalTime"
},
"extension": {
"type": "keyword"
},
"filename": {
"type": "keyword",
"store": true
},
"filesize": {
"type": "long"
},
"indexed_chars": {
"type": "long"
},
"indexing_date": {
"type": "date",
"format": "dateOptionalTime"
},
"last_accessed": {
"type": "date",
"format": "dateOptionalTime"
},
"last_modified": {
"type": "date",
"format": "dateOptionalTime"
},
"url": {
"type": "keyword",
"index": false
}
}
},
"meta": {
"properties": {
"altitude": {
"type": "text"
},
"author": {
"type": "text"
},
"comments": {
"type": "text"
},
"contributor": {
"type": "text"
},
"coverage": {
"type": "text"
},
"created": {
"type": "date",
"format": "dateOptionalTime"
},
"creator_tool": {
"type": "keyword"
},
"date": {
"type": "date",
"format": "dateOptionalTime"
},
"description": {
"type": "text"
},
"format": {
"type": "text"
},
"identifier": {
"type": "text"
},
"keywords": {
"type": "text"
},
"language": {
"type": "keyword"
},
"latitude": {
"type": "text"
},
"longitude": {
"type": "text"
},
"metadata_date": {
"type": "date",
"format": "dateOptionalTime"
},
"modifier": {
"type": "text"
},
"print_date": {
"type": "date",
"format": "dateOptionalTime"
},
"publisher": {
"type": "text"
},
"rating": {
"type": "byte"
},
"relation": {
"type": "text"
},
"rights": {
"type": "text"
},
"source": {
"type": "text"
},
"title": {
"type": "text"
},
"type": {
"type": "text"
}
}
},
"path": {
"properties": {
"real": {
"type": "keyword",
"fields": {
"fulltext": {
"type": "text"
},
"tree": {
"type": "text",
"analyzer": "fscrawler_path",
"fielddata": true
}
}
},
"root": {
"type": "keyword"
},
"virtual": {
"type": "keyword",
"fields": {
"fulltext": {
"type": "text"
},
"tree": {
"type": "text",
"analyzer": "fscrawler_path",
"fielddata": true
}
}
}
}
}
}
}
}
}
}
when it comes to elasticsearch, i am new to it. That's why, my question is if i'd like to filter multiple fields, would aggregate might solve the problem? or what should i use?
{
"aggs": {
"filtered": {
"filter": {
"bool": {
"must": [
{ "term" : {"shop_slug" : "sharkys-dhammazedi.6326"}},
{ "term" :
{"slug" : "anchovy-essence"}}
]
}
}
}
}}
HERE Is The Mapping
"product_catalogue": {
"mappings": {
"products": {
"properties": {
"buy_online": {
"type": "long"
},
"category": {
"properties": {
"id": {
"type": "long"
},
"name": {
"type": "string"
},
"slug": {
"type": "string"
},
"tree": {
"properties": {
"2": {
"type": "string"
},
"3": {
"type": "string"
},
"5": {
"type": "string"
},
"10": {
"type": "string"
},
"11": {
"type": "string"
},
"12": {
"type": "string"
},
"13": {
"type": "string"
},
"14": {
"type": "string"
},
"15": {
"type": "string"
},
"16": {
"type": "string"
},
"17": {
"type": "string"
},
"18": {
"type": "string"
},
"19": {
"type": "string"
},
"20": {
"type": "string"
},
"21": {
"type": "string"
},
"22": {
"type": "string"
},
"23": {
"type": "string"
},
"24": {
"type": "string"
},
"25": {
"type": "string"
},
"26": {
"type": "string"
},
"27": {
"type": "string"
},
"28": {
"type": "string"
},
"29": {
"type": "string"
},
"30": {
"type": "string"
},
"31": {
"type": "string"
},
"32": {
"type": "string"
},
"33": {
"type": "string"
},
"34": {
"type": "string"
},
"35": {
"type": "string"
},
"36": {
"type": "string"
},
"37": {
"type": "string"
},
"38": {
"type": "string"
},
"39": {
"type": "string"
},
"40": {
"type": "string"
},
"41": {
"type": "string"
},
"42": {
"type": "string"
},
"43": {
"type": "string"
},
"46": {
"type": "string"
},
"62": {
"type": "string"
},
"72": {
"type": "string"
},
"73": {
"type": "string"
},
"74": {
"type": "string"
},
"75": {
"type": "string"
},
"77": {
"type": "string"
},
"137": {
"type": "string"
},
"139": {
"type": "string"
},
"140": {
"type": "string"
},
"141": {
"type": "string"
}
}
}
}
},
"click_collect": {
"type": "long"
},
"company_id": {
"type": "string"
},
"company_name": {
"type": "string"
},
"company_slug": {
"type": "string",
"index": "not_analyzed"
},
"condition": {
"type": "string"
},
"created_at": {
"type": "date",
"format": "dateOptionalTime"
},
"currency_iso": {
"type": "string"
},
"delivery_available": {
"type": "boolean"
},
"delivery_type": {
"properties": {
"id": {
"type": "long"
},
"name": {
"type": "string"
},
"pivot": {
"properties": {
"deliverytype_id": {
"type": "long"
},
"pricing": {
"type": "string"
},
"shop_id": {
"type": "long"
}
}
}
}
},
"description": {
"type": "string",
"analyzer": "snowball"
},
"group_id": {
"type": "string"
},
"image_url": {
"type": "string"
},
"listed_shop": {
"type": "integer"
},
"location": {
"type": "geo_point"
},
"mainPhoto": {
"type": "string"
},
"mm_description": {
"type": "string"
},
"mm_title": {
"type": "string"
},
"postcode": {
"type": "string"
},
"prices": {
"properties": {
"retail": {
"properties": {
"converted": {
"properties": {
"currencyIso": {
"type": "string"
},
"decimal": {
"type": "string"
},
"formatted": {
"type": "string"
}
}
},
"original": {
"properties": {
"currencyIso": {
"type": "string"
},
"decimal": {
"type": "string"
},
"formatted": {
"type": "string"
}
}
}
}
}
}
},
"shop_category": {
"type": "long"
},
"shop_id": {
"type": "long"
},
"shop_name": {
"type": "string"
},
"shop_slug": {
"type": "string",
"index": "not_analyzed"
},
"shop_stock_count": {
"type": "integer"
},
"shop_type": {
"properties": {
"id": {
"type": "long"
},
"name": {
"type": "string"
}
}
},
"slug": {
"type": "string"
},
"title": {
"type": "string",
"analyzer": "snowball"
},
"user_stock_count": {
"type": "integer"
},
"variant": {
"properties": {
"Colour": {
"type": "string"
},
"Disk Space": {
"type": "string"
},
"Length": {
"type": "string"
}
}
}
}
}
}
}
This query didn't seems to work out . Infact, it only filter one of the fields but can't resolve to filter both fields which is shop_slug and slug. Please kindly answer Thanks.
Having read all the tutorials on ES website, I cannot achieve my goal.
We are using ES 1.7.6 and I want to have only one single instance of documents matching my criteria. But what I get from ES is all the data matching the filter and the aggregation statistics.
GET _search
{
"size":1000,
"query":{
"bool":{
"should":[
{
"match":{
"isoMessage.fields.39":{
"query":"00"
}
}
}
]
}
},
"aggs":{
"group_by_CATI":{
"terms":{
"field":"isoMessage.fields.41"
}
}
}
}
Note that the index of isoMessage.fields.41 is set to not_analyzed.
Thanks for any help;
UPDATE: The mapping
{
"cm": {
"mappings": {
"Event": {
"properties": {
"deleted": {
"type": "boolean"
},
"id": {
"type": "string"
},
"isoMessage": {
"properties": {
"fields": {
"properties": {
"2": {
"type": "string"
},
"3": {
"type": "string"
},
"4": {
"type": "string"
},
"7": {
"type": "string"
},
"11": {
"type": "string"
},
"12": {
"type": "string"
},
"13": {
"type": "string"
},
"14": {
"type": "string"
},
"22": {
"type": "string"
},
"25": {
"type": "string"
},
"32": {
"type": "string"
},
"37": {
"type": "string"
},
"39": {
"type": "string"
},
"41": {
"type": "string"
},
"42": {
"type": "string"
},
"48": {
"type": "string"
},
"49": {
"type": "string"
},
"60": {
"type": "string"
},
"63": {
"type": "string"
},
"100": {
"type": "string"
},
"128": {
"type": "string"
}
}
},
"isReversal": {
"type": "boolean"
},
"isReversalDone": {
"type": "boolean"
},
"messageSpec": {
"type": "string"
},
"mti": {
"type": "string"
},
"request": {
"type": "boolean"
},
"response": {
"type": "boolean"
}
}
},
"msg": {
"type": "string"
},
"occurDate": {
"type": "long"
},
"receiver": {
"type": "string"
},
"rrn": {
"type": "string"
},
"sender": {
"type": "string",
"index": "not_analyzed"
},
"txId": {
"type": "string"
},
"version": {
"type": "long"
}
}
}
}
}
}
If I understand correctly, and you want to get only the aggregation result:
Change your size field from:
"size":1000
to
"size":0
in order to set the _search query displayed results limit. It won't affect the aggregation result, though.
I have two indexes - one for Application model and the other for Databases model (many-to-many relational).
Each document is denormalized to contain attributes from the other model
Application
|_ vendor_name
|_ databases
|_ db_1
|_ db_2
Database
|_ database_applications
|_ app_1
|_vendor_name
|_ app_2
|_ vendor_name
Executing a multi-index search for a vendor name - it seems I'm getting the proper results from both Indexes.
The challenge is properly aggregating on the vendor_name field
using the following aggregation seems to work when the result is only from Database. I also tried field: '*vendor_name' but doesn't seem to work.
What am I missing? Should the model be changed?
aggregation:
vendor_name: {
terms: {
field: "database_applications.vendor_name"
}
}
UPDATE 1:
As per #Andrie-Stefan - Here's a more accurate representation of both indexes mappings (abbreviated for shortness):
Database
{
"company-company_databases": {
"aliases": {},
"mappings": {
"company_database": {
"properties": {
"company_applications": {
"properties": {
"application_id": {
"type": "long"
},
"application_name": {
"type": "string"
},
"business_owner": {
"type": "string"
},
"company_system_applications": {
"properties": {
"allow_add_request": {
"type": "string"
},
"allow_remove_request": {
"type": "string"
},
"asset_type": {
"type": "string"
},
"company_application_id": {
"type": "long"
},
"company_application_name": {
"type": "string"
},
"company_business_owner": {
"type": "string"
},
"company_division_id": {
"type": "long"
},
"company_it_app_steward": {
"type": "string"
},
"company_notes": {
"type": "string"
},
"company_system_id": {
"type": "long"
},
"company_vendor": {
"type": "string"
},
"id": {
"type": "long"
},
"it_app_steward": {
"type": "string"
},
"it_owner": {
"type": "string"
},
"last_modified": {
"type": "string"
},
"last_modified_by": {
"type": "string"
},
"media_location": {
"type": "string"
},
"media_source": {
"type": "string"
},
"name": {
"type": "string"
},
"owned_by": {
"type": "string"
},
"status": {
"type": "string"
},
"status_id": {
"type": "long"
},
"system_application": {
"properties": {
"division": {
"type": "string"
},
"id": {
"type": "long"
},
"name": {
"type": "string"
},
"owner_id": {
"type": "string"
},
"status": {
"type": "string"
},
"steward_id": {
"type": "string"
},
"vendor_name": {
"type": "string"
},
"vendor_url_web_site": {
"type": "string"
},
"version": {
"type": "string"
}
}
},
"vendor_name": {
"type": "string",
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed"
}
}
},
"version": {
"type": "string"
}
}
},
"division_id": {
"type": "long"
},
"it_app_steward": {
"type": "string"
},
"notes": {
"type": "string"
},
"software_inventory_id": {
"type": "long"
},
"vendor": {
"type": "string"
}
}
},
"company_instances": {
"properties": {
"business_environment_id": {
"type": "long"
},
"cgi_service_id": {
"type": "long"
},
"char_set": {
"type": "string"
},
"confirmed_license_purchase_dt": {
"type": "string"
},
"company_server": {
"properties": {
"business_environment_id": {
"type": "long"
},
"division_id": {
"type": "long"
},
"domain": {
"type": "string"
},
"hw_platform_id": {
"type": "long"
},
"ip_address": {
"type": "string"
},
"location_id": {
"type": "long"
},
"no_of_cpu": {
"type": "long"
},
"notes": {
"type": "string"
},
"os_platform_id": {
"type": "long"
},
"os_version": {
"type": "string"
},
"server_id": {
"type": "long"
},
"server_name": {
"type": "string"
}
}
},
"description": {
"type": "string"
},
"division_id": {
"type": "long"
},
"edition_id": {
"type": "long"
},
"instance_id": {
"type": "long"
},
"instance_name": {
"type": "string"
},
"itap_have_access": {
"type": "string"
},
"listener_port": {
"type": "long"
},
"notes": {
"type": "string"
},
"patch_number": {
"type": "string"
},
"rdbms_type_id": {
"type": "long"
},
"server_id": {
"type": "long"
},
"service_level_id": {
"type": "long"
},
"version": {
"type": "string"
}
}
},
"db_security_model_id": {
"type": "long"
},
"schema_or_db": {
"type": "string"
},
"schema_or_db_id": {
"type": "long"
},
"schema_or_db_type_id": {
"type": "long"
}
}
}
},
"settings": {
"index": {
"creation_date": "1442976578465",
"uuid": "TxQZoNSpR5qa2Y2ERZzuYw",
"number_of_replicas": "1",
"number_of_shards": "5",
"version": {
"created": "1070299"
}
}
},
"warmers": {}
}
}
Application
{
"applications": {
"aliases": {},
"mappings": {
"application": {
"properties": {
"application_view": {
"properties": {
"app_name": {
"type": "string"
},
"app_status": {
"type": "string"
},
"app_steward_name": {
"type": "string"
},
"app_suite": {
"type": "string"
},
"app_vendor_name": {
"type": "string"
},
"app_version": {
"type": "string"
},
"assignment_group": {
"type": "string"
},
"business_domain_name": {
"type": "string"
},
"exception": {
"type": "string"
},
"id": {
"type": "long"
},
"it_owner_name": {
"type": "string"
},
"service_level": {
"type": "string"
}
}
},
"assignment_group": {
"type": "string"
},
"company_databases": {
"properties": {
"backup_history_info": {
"type": "string"
},
"company_applications": {
"properties": {
"alternate_name": {
"type": "string"
},
"application_id": {
"type": "long"
},
"application_name": {
"type": "string"
},
"business_owner": {
"type": "string"
},
"company_system_applications": {
"properties": {
"aka": {
"type": "string"
},
"allow_add_request": {
"type": "string"
},
"allow_remove_request": {
"type": "string"
},
"asset_type": {
"type": "string"
},
"contract_number": {
"type": "string"
},
"cost_level": {
"type": "string"
},
"company_alternate_name": {
"type": "string"
},
"company_application_id": {
"type": "long"
},
"company_application_name": {
"type": "string"
},
"company_business_owner": {
"type": "string"
},
"company_division_id": {
"type": "long"
},
"company_it_app_steward": {
"type": "string"
},
"company_notes": {
"type": "string"
},
"company_system_id": {
"type": "long"
},
"company_vendor": {
"type": "string"
},
"description": {
"type": "string"
},
"display_in_catalog": {
"type": "string"
},
"id": {
"type": "long"
},
"inform_of_removal": {
"type": "string"
},
"is_restricted": {
"type": "string"
},
"it_app_steward": {
"type": "string"
},
"it_owner": {
"type": "string"
},
"last_modified": {
"type": "string"
},
"last_modified_by": {
"type": "string"
},
"media_location": {
"type": "string"
},
"media_source": {
"type": "string"
},
"name": {
"type": "string"
},
"os_environment": {
"type": "string"
},
"owned_by": {
"type": "string"
},
"retirement_date": {
"type": "date",
"format": "dateOptionalTime"
},
"status": {
"type": "string"
},
"status_id": {
"type": "long"
},
"suite_name": {
"type": "string"
},
"system_application": {
"properties": {
"assignment_group": {
"type": "string"
},
"division": {
"type": "string"
},
"id": {
"type": "long"
},
"name": {
"type": "string"
},
"owner_id": {
"type": "string"
},
"status": {
"type": "string"
},
"steward_id": {
"type": "string"
},
"suite": {
"type": "string"
},
"vendor_name": {
"type": "string"
},
"vendor_url_web_site": {
"type": "string"
},
"version": {
"type": "string"
}
}
},
"vendor_name": {
"type": "string"
},
"version": {
"type": "string"
}
}
},
"division_id": {
"type": "long"
},
"it_app_steward": {
"type": "string"
},
"notes": {
"type": "string"
},
"software_inventory_id": {
"type": "long"
},
"vendor": {
"type": "string"
}
}
},
"company_instances": {
"properties": {
"business_environment_id": {
"type": "long"
},
"cgi_service_id": {
"type": "long"
},
"char_set": {
"type": "string"
},
"confirmed_license_purchase_dt": {
"type": "string"
},
"company_server": {
"properties": {
"business_environment_id": {
"type": "long"
},
"division_id": {
"type": "long"
},
"domain": {
"type": "string"
},
"hw_platform_id": {
"type": "long"
},
"ip_address": {
"type": "string"
},
"location_id": {
"type": "long"
},
"no_of_cpu": {
"type": "long"
},
"notes": {
"type": "string"
},
"os_platform_id": {
"type": "long"
},
"os_version": {
"type": "string"
},
"server_id": {
"type": "long"
},
"server_name": {
"type": "string"
}
}
},
"description": {
"type": "string"
},
"division_id": {
"type": "long"
},
"edition_id": {
"type": "long"
},
"instance_id": {
"type": "long"
},
"instance_name": {
"type": "string"
},
"itap_have_access": {
"type": "string"
},
"listener_port": {
"type": "long"
},
"location_id": {
"type": "long"
},
"notes": {
"type": "string"
},
"patch_number": {
"type": "string"
},
"rdbms_type_id": {
"type": "long"
},
"server_id": {
"type": "long"
},
"service_level_id": {
"type": "long"
},
"version": {
"type": "string"
}
}
},
"db_security_model_id": {
"type": "long"
},
"notes": {
"type": "string"
},
"schema_or_db": {
"type": "string"
},
"schema_or_db_id": {
"type": "long"
},
"schema_or_db_type_id": {
"type": "long"
}
}
},
"division": {
"type": "string",
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed"
}
}
},
"id": {
"type": "long"
},
"name": {
"type": "string"
},
"owner": {
"properties": {
"email_address": {
"type": "string"
},
"id": {
"type": "long"
},
"name": {
"type": "string",
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed"
}
}
},
"search_type": {
"type": "string"
},
"user_id": {
"type": "string"
}
}
},
"owner_id": {
"type": "string"
},
"status": {
"type": "string",
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed"
}
}
},
"steward": {
"properties": {
"email_address": {
"type": "string"
},
"id": {
"type": "long"
},
"name": {
"type": "string",
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed"
}
}
},
"search_type": {
"type": "string"
},
"user_id": {
"type": "string"
}
}
},
"steward_id": {
"type": "string"
},
"suite": {
"type": "string"
},
"vendor_name": {
"type": "string",
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed"
}
}
},
"vendor_url_web_site": {
"type": "string"
},
"version": {
"type": "string"
}
}
}
},
"settings": {
"index": {
"creation_date": "1442970067540",
"uuid": "O7DTaCSESbqhjJpv62T0Wg",
"number_of_replicas": "1",
"number_of_shards": "5",
"version": {
"created": "1070299"
}
}
},
"warmers": {}
}
}
Fields can only be aggregated across indices if they are named alike. There is no wildcard syntax for aggregation fields.
Here is what your mapping currently defines:
INDEX: company-company_databases
TYPE: company_database
FIELD NAMES:
company_applications.company_system_applications.vendor_name
company_applications.company_system_applications.system_application.vendor_name
INDEX: applications
TYPE: application
FIELD NAMES:
company_databases.company_applications.company_system_applications.vendor_name
company_databases.company_applications.company_system_applications.system_application.vendor_name
As far as Elasticsearch is concerned, these fields have nothing in common (even though part of the path is vendor_name).
If your goal is to aggregate vendor_name across a query that spans the two indices, think about restructuring your indices/mappings to accomplish this.
Note that Elasticsearch doesn't model many-to-many relationships
If you can get away with duplicating Database info across applications, you might be able to re-formulate your relationships as a hierarchy, e.g.:
INDEX: applications
--
TYPE: application
FIELDS: vendor_name, etc...
--
TYPE: database_application
FIELDS: vendor_name, databases.<inner fields>, etc...
--
Then you'd be able to aggregate across types on the same field path vendor_name with the added bonus of querying a single applications index.