Elastic Search queries not working with curl - elasticsearch

Running the command:
curl -XGET http://127.0.0.1:9200/30556/_search -d '{
"query": {
"constant_score" : {
"filter" : {
"term" : { "portal_type" : "Folder"}
}
}
}
}'
yields 0 results. The output is:
{"took":1,"timed_out":false,"_shards":{"total":5,"successful":5,"failed":0},"hits":{"total":0,"max_score":null,"hits":[]}}
If fact, I can't get any queries to yield results.
However, when I run the same query using the head plugin, it works fine.
I'm on elasticsearch 0.20.2 on Mac OS X. I'm starting elastic search using the command:
bin/elasticsearch -f
Is there something obvious I'm missing? Seems like I have the correct syntax and I don't get any errors.
Mapping:
{
"30556": {
"portal_catalog": {
"properties": {
"CreationDate": {
"type": "date",
"format": "dateOptionalTime"
},
"Creator": {
"type": "string"
},
"Date": {
"type": "date",
"format": "dateOptionalTime"
},
"Description": {
"type": "string"
},
"ModificationDate": {
"type": "date",
"format": "dateOptionalTime"
},
"SearchableText": {
"type": "string"
},
"Title": {
"type": "string"
},
"Type": {
"type": "string"
},
"UID": {
"type": "string"
},
"allowedRolesAndUsers": {
"type": "string"
},
"created": {
"type": "date",
"format": "dateOptionalTime"
},
"effective": {
"type": "date",
"format": "dateOptionalTime"
},
"effectiveRange": {
"dynamic": "true",
"properties": {
"effectiveRange1": {
"type": "date",
"format": "dateOptionalTime"
},
"effectiveRange2": {
"type": "date",
"format": "dateOptionalTime"
}
}
},
"exclude_from_nav": {
"type": "boolean"
},
"expires": {
"type": "date",
"format": "dateOptionalTime"
},
"getId": {
"type": "string"
},
"getObjPositionInParent": {
"type": "long"
},
"getObjSize": {
"type": "string"
},
"id": {
"type": "string"
},
"is_default_page": {
"type": "boolean"
},
"is_folderish": {
"type": "boolean"
},
"listCreators": {
"type": "string"
},
"meta_type": {
"type": "string"
},
"modified": {
"type": "date",
"format": "dateOptionalTime"
},
"object_provides": {
"type": "string"
},
"path": {
"dynamic": "true",
"properties": {
"depth": {
"type": "long"
},
"path": {
"type": "string"
}
}
},
"portal_type": {
"type": "string"
},
"review_state": {
"type": "string"
},
"sortable_title": {
"type": "string"
},
"total_comments": {
"type": "long"
}
}
}
}
}
Example Indexed Document:
{
"_index": "30556",
"_type": "portal_catalog",
"_id": "30613",
"_score": 1,
"_source": {
"sortable_title": "news",
"exclude_from_nav": false,
"meta_type": "ATFolder",
"Date": "2013-01-14T09:24:56-06:00",
"CreationDate": "2013-01-14T09:24:56-06:00",
"path": {
"depth": 2,
"path": "/el/news"
},
"allowedRolesAndUsers": [
"Anonymous"
],
"portal_type": "Folder",
"id": "news",
"UID": "3116b6c7ec384a9393f238fdde778612",
"expires": "2499-12-31T00:00:00-06:00",
"Subject": [],
"is_folderish": true,
"is_default_page": false,
"effectiveRange": {
"effectiveRange1": "1000-01-01T00:00:00-06:00",
"effectiveRange2": "2499-12-31T00:00:00-06:00"
},
"commentators": [],
"created": "2013-01-14T09:24:56-06:00",
"getRawRelatedItems": [],
"cmf_uid": [],
"Creator": "admin",
"end": [],
"modified": "2013-01-14T09:24:56-06:00",
"Description": "Site News",
"ModificationDate": "2013-01-14T09:24:56-06:00",
"total_comments": 0,
"in_reply_to": [],
"getIcon": "",
"effective": "1000-01-01T00:00:00-06:00",
"SearchableText": "news News Site News ",
"getObjPositionInParent": 61,
"object_provides": [
"collective.syndication.interfaces.ISyndicatable",
"Products.ATContentTypes.interfaces.folder.IATFolder",
"Products.CMFCore.interfaces._content.IContentish",
"z3c.relationfield.interfaces.IHasIncomingRelations",
"webdav.interfaces.IWriteLock"
],
"last_comment_date": null,
"review_state": "published",
"start": [],
"Type": "Folder",
"listCreators": [
"admin"
],
"getId": "news",
"getObjSize": "1 kB",
"Title": "News"
}

Try to use lower case index names.
Does it work?
If not, can you provide your indexed document and mapping if any?
UPDATE:
You use default analyzer so your field is broken into tokens which are lowercased.
A TermFilter is not analyzed So it does not match.
You can lowercase you TermFilter or use a MatchQuery which is analyzed or change your mapping and set the field to not_analyzed.

Related

Update Mapping of existing Index in Elasticsearch

I am totally new to elastic search. So please forgive me if this is a stupid Question and my Questions might have been answered somewhere else already but I couldn't find it.
I want to use Elastic Search as a search engine for PDF'S and docx's in my network. I used fscrawler to ingest the PDF's to elastic search. Since the documents I want to ingest are in several languages I wanted to use n-graming for stemming. To do so I wanted to update my mapping like this
PUT test/_mappings/_all
{
"mappings": {
"title": {
"properties": {
"title": {
"type": "text",
"fields": {
"de": {
"type": "string",
"analyzer": "german"
},
"en": {
"type": "string",
"analyzer": "english"
},
"general": {
"type": "string",
"analyzer": "trigrams"
}
}
}
}
}
}
}
And now I get this Errormessage
{ "error": {
"root_cause": [
{
"type": "mapper_parsing_exception",
"reason": "Root mapping definition has unsupported parameters: [mappings : {title={properties={title={type=text,
fields={de={type=string, analyzer=german}, en={type=string,
analyzer=english}, general={type=string, analyzer=trigrams}}}}}}]"
}
],
"type": "mapper_parsing_exception",
"reason": "Root mapping definition has unsupported parameters: [mappings : {title={properties={title={type=text,
fields={de={type=string, analyzer=german}, en={type=string,
analyzer=english}, general={type=string, analyzer=trigrams}}}}}}]"
}, "status": 400 }
Do you have any idea how i can fix this? Or do you have an idea how I can ingest the files with the right mapping without using fscrawler?
those are my settings
{
"test": {
"settings": {
"index": {
"mapping": {
"total_fields": {
"limit": "2000"
}
},
"number_of_shards": "5",
"provided_name": "test",
"creation_date": "1542031632596",
"analysis": {
"filter": {
"trigrams_filter": {
"type": "ngram",
"min_gram": "3",
"max_gram": "3"
}
},
"analyzer": {
"fscrawler_path": {
"tokenizer": "fscrawler_path"
},
"trigrams": {
"filter": [
"lowercase",
"trigrams_filter"
],
"type": "custom",
"tokenizer": "standard"
}
},
"tokenizer": {
"fscrawler_path": {
"type": "path_hierarchy"
}
}
},
"number_of_replicas": "1",
"uuid": "7L3QE5_xRACECVbTFlFY-Q",
"version": {
"created": "6040399"
}
}
}
}
}
My mapping
{
"test": {
"mappings": {
"_doc": {
"dynamic_templates": [
{
"raw_as_text": {
"path_match": "meta.raw.*",
"mapping": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
}
}
}
],
"properties": {
"attachment": {
"type": "binary"
},
"attributes": {
"properties": {
"group": {
"type": "keyword"
},
"owner": {
"type": "keyword"
}
}
},
"content": {
"type": "text"
},
"file": {
"properties": {
"checksum": {
"type": "keyword"
},
"content_type": {
"type": "keyword"
},
"created": {
"type": "date",
"format": "dateOptionalTime"
},
"extension": {
"type": "keyword"
},
"filename": {
"type": "keyword",
"store": true
},
"filesize": {
"type": "long"
},
"indexed_chars": {
"type": "long"
},
"indexing_date": {
"type": "date",
"format": "dateOptionalTime"
},
"last_accessed": {
"type": "date",
"format": "dateOptionalTime"
},
"last_modified": {
"type": "date",
"format": "dateOptionalTime"
},
"url": {
"type": "keyword",
"index": false
}
}
},
"meta": {
"properties": {
"altitude": {
"type": "text"
},
"author": {
"type": "text"
},
"comments": {
"type": "text"
},
"contributor": {
"type": "text"
},
"coverage": {
"type": "text"
},
"created": {
"type": "date",
"format": "dateOptionalTime"
},
"creator_tool": {
"type": "keyword"
},
"date": {
"type": "date",
"format": "dateOptionalTime"
},
"description": {
"type": "text"
},
"format": {
"type": "text"
},
"identifier": {
"type": "text"
},
"keywords": {
"type": "text"
},
"language": {
"type": "keyword"
},
"latitude": {
"type": "text"
},
"longitude": {
"type": "text"
},
"metadata_date": {
"type": "date",
"format": "dateOptionalTime"
},
"modifier": {
"type": "text"
},
"print_date": {
"type": "date",
"format": "dateOptionalTime"
},
"publisher": {
"type": "text"
},
"rating": {
"type": "byte"
},
"relation": {
"type": "text"
},
"rights": {
"type": "text"
},
"source": {
"type": "text"
},
"title": {
"type": "text"
},
"type": {
"type": "text"
}
}
},
"path": {
"properties": {
"real": {
"type": "keyword",
"fields": {
"fulltext": {
"type": "text"
},
"tree": {
"type": "text",
"analyzer": "fscrawler_path",
"fielddata": true
}
}
},
"root": {
"type": "keyword"
},
"virtual": {
"type": "keyword",
"fields": {
"fulltext": {
"type": "text"
},
"tree": {
"type": "text",
"analyzer": "fscrawler_path",
"fielddata": true
}
}
}
}
}
}
}
}
}
}

Sorting in Elasticsearch ignoring the date part of field

I have a mapping of date field as :
"created": {
"type" : "date",
"format": "yyyy-MM-dd HH:mm:ss"
},
Now when i sort based on the above field:
"sort":[
{"created":{"order":"asc"}}
],
It takes only time part of the date while sorting and ignores the date part.
{
"_index": "somehting",
"_type": "UserActivity",
"_id": "81574",
"_score": null,
"_source": {
"created": "2016-03-29 00:00:07",
"appCode": "appcode",
"userId": "100008057363993"
},
"sort": [
"00:00:07"
]
},
How do sort based on the whole date?
Please note i cannot use scripting as its disabled on production server and i cannot re-index..
Adding the total mapping:
{
"someIndex": {
"mappings": {
"UserActivity": {
"_timestamp": {
"enabled": true,
"store": true,
"format": "yyyy-MM-dd HH:mm:ss"
},
"properties": {
"_table": {
"type": "string"
},
"_tableat": {
"type": "string"
},
"activity": {
"properties": {
"_table": {
"type": "string"
},
"_tableat": {
"type": "string"
},
"clientId": {
"type": "integer"
},
"code": {
"type": "string"
},
"created": {
"type": "date",
"format": "yyyy-MM-dd HH:mm:ss"
},
"foreignName": {
"type": "string"
},
"frequency": {
"type": "integer"
},
"id": {
"type": "long"
},
"lastUpdated": {
"type": "date",
"format": "yyyy-MM-dd HH:mm:ss"
},
"name": {
"type": "string"
},
"points": {
"type": "long"
},
"strategy": {
"type": "string"
}
}
},
"activityId": {
"type": "string"
},
"appCode": {
"type": "string"
},
"clientId": {
"type": "long"
},
"created": {
"type": "string"
},
"details": {
"type": "string"
},
"foreignName": {
"type": "string"
},
"id": {
"type": "long"
},
"lastUpdated": {
"type": "date",
"format": "yyyy-MM-dd HH:mm:ss"
},
"notes": {
"type": "string"
},
"userId": {
"type": "long"
}
}
}
}
}
}

Searching on fields of a nested object on elasticsearch

I have this mapping on ES 1.7.3:
{
"customer": {
"aliases": {},
"mappings": {
"customer": {
"properties": {
"addresses": {
"type": "nested",
"include_in_parent": true,
"properties": {
"address1": {
"type": "string"
},
"address2": {
"type": "string"
},
"address3": {
"type": "string"
},
"country": {
"type": "string"
},
"latitude": {
"type": "double",
"index": "not_analyzed"
},
"longitude": {
"type": "double",
"index": "not_analyzed"
},
"postcode": {
"type": "string"
},
"state": {
"type": "string"
},
"town": {
"type": "string"
},
"unit": {
"type": "string"
}
}
},
"companyNumber": {
"type": "string"
},
"id": {
"type": "string",
"index": "not_analyzed"
},
"name": {
"type": "string"
},
"status": {
"type": "string"
},
"timeCreated": {
"type": "date",
"format": "dateOptionalTime"
},
"timeUpdated": {
"type": "date",
"format": "dateOptionalTime"
}
}
}
},
"settings": {
"index": {
"refresh_interval": "1s",
"number_of_shards": "5",
"creation_date": "1472372294516",
"store": {
"type": "fs"
},
"uuid": "RxJdXvPWSXGpKz8pdcF91Q",
"version": {
"created": "1050299"
},
"number_of_replicas": "1"
}
},
"warmers": {}
}
}
The spring application generates this query:
{
"query": {
"bool": {
"should": {
"query_string": {
"query": "(addresses.\\*:sample* AND NOT status:ARCHIVED)",
"fields": [
"type",
"name",
"companyNumber",
"status",
"addresses.unit",
"addresses.address1",
"addresses.address2",
"addresses.address3",
"addresses.town",
"addresses.state",
"addresses.postcode",
"addresses.country"
],
"default_operator": "or",
"analyze_wildcard": true
}
}
}
}
}
on which "addresses.*:sample*" is the only input.
"query": "(sample* AND NOT status:ARCHIVED)"
Code above works but searches all fields of the customer object.
Since I want to search only on address fields I used the "addresses.*"
Query works only if the fields of the address object are of String type and before I added longitude and latitude fields of double type on address object. Now the error occurs because of these two new fields.
Error:
Parse Failure [Failed to parse source [{
"query": {
"bool": {
"should": {
"query_string": {
"query": "(addresses.\\*:sample* AND NOT status:ARCHIVED)",
"fields": [
"type",
"name",
"companyNumber","country",
"state",
"status",
"addresses.unit",
"addresses.address1",
"addresses.address2",
"addresses.address3",
"addresses.town",
"addresses.state",
"addresses.postcode",
"addresses.country",
],
"default_operator": "or",
"analyze_wildcard": true
}
}
}
}
}
]]
NumberFormatException[For input string: "sample"
Is there a way to search "String" fields within a nested object using addresses.* only?
The solution was to add "lenient": true. As per the documentation: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html
lenient - If set to true will cause format based failures (like providing text to a numeric field) to be ignored.

Elasticsearch MapperParsingException

I am trying to index following data to elasticsearch,
{
"_id": "5619578c1983757a72efef15",
"aseg": {},
"cs": {
"source": "None",
"ss": "In Transit",
"sr": "Weight Captured",
"act": "+B",
"pid": "BAG21678106",
"st": "UD",
"dest": "Bharatpur_DC (Rajasthan)",
"u": "J",
"sl": "Jaipur_Hub (Rajasthan)",
"ud": "2015-10-12T14:59:44.270000",
"sd": "2015-10-12T14:59:44.270000"
},
"nsl": [
{
"dt": [
2015,
10,
10
],
"code": "X-PPONM"
},
{
"dt": [
2015,
10,
11
],
"code": "X-UCI"
},
]
}
but in return i am getting this error
MapperParsingException[failed to parse [cs.nsl]]; nested: ElasticsearchIllegalArgumentException[unknown property [dt]];
I checked the mapping, mapping is correct, nsl nested inside cs dict has a different mapping than nsl at root level.
"cs": {
"properties": {
"act": {
"type": "string"
},
"add": {
"type": "string"
},
"asr": {
"type": "string"
},
"bucket": {
"type": "string"
},
"dest": {
"type": "string",
"index": "not_analyzed"
},
"dwbn": {
"type": "string"
},
"lcld": {
"type": "string"
},
"lat": {
"type": "string"
},
"lon": {
"type": "string"
},
"loc": {
"type": "double"
},
"nsl": {
"type": "string",
"index": "not_analyzed"
},
"ntd": {
"type": "date",
"format": "dateOptionalTime"
},
"pbs": {
"type": "string"
},
"pid": {
"type": "string"
},
"pupid": {
"type": "string"
},
"sd": {
"type": "date",
"format": "dateOptionalTime"
},
"sl": {
"type": "string",
"index": "not_analyzed"
},
"source": {
"properties": {
"source": {
"type": "string"
},
"source_id": {
"type": "string"
},
"source_type": {
"type": "string"
}
}
},
"sr": {
"type": "string"
},
"ss": {
"type": "string",
"index": "not_analyzed"
},
"st": {
"type": "string"
},
"u": {
"type": "string",
"index": "not_analyzed"
},
"ud": {
"type": "date",
"format": "dateOptionalTime"
},
"vh": {
"type": "string"
}
}
},
and for nsl at root level mapping is as follow
"nsl": {
"properties" : {
"code" : {
"type" : "string",
"index": "not_analyzed"
},
"dt" : {
"type" : "string",
"index": "not_analyzed"
}
}
},
this is happening for only a few records, rest all are syncing fine.
there isn't any changes in payload.
Futher nsl is a sparse key inside cs.
In your mapping nsl is as follows -
"nsl": {
"type": "string",
"index": "not_analyzed"
},
As per mapping , Elasticsearch is expecting a concrete string value to the nsl field but its a object array in the document you have provided.
Elasticsearch once it has a mapping , its definite. You cant insert an object data into a string field.
I tried your document without pre-setting any mapping as follows:
{
"aseg": {},
"cs": {
"source": "None",
"ss": "In Transit",
"sr": "Weight Captured",
"act": "+B",
"pid": "BAG21678106",
"st": "UD",
"dest": "Bharatpur_DC (Rajasthan)",
"u": "J",
"nsl":"foo",
"sl": "Jaipur_Hub (Rajasthan)",
"ud": "2015-10-12T14:59:44.270000",
"sd": "2015-10-12T14:59:44.270000"
},
"nsl": [
{
"dt": [
2015,
10,
10
],
"code": "X-PPONM"
},
{
"dt": [
2015,
10,
11
],
"code": "X-UCI"
}
]
}
And the ES created the mapping as follows:
"nsl": {
"properties": {
"dt": {
"type": "long"
},
"code": {
"type": "string"
}
}
}
As you can see ES put the "dt" type as "long" which is the internal representation of a date type. So, may be need to change that type?
Also, without seeing the successful document it is difficult to guess but I believe those documents do not have the "dt" field value.
Of course, you are free to put "not_analyzed" as you see fit for any field.

elasticsearch equivalent of join

I'm new to elastic search and I'm trying to figure out how to do the equivalent of a SQL join. Here are my 2 mappings:
{
"home_panel": {
"mappings": {
"levis": {
"properties": {
"created_at": {
"type": "date",
"format": "dateOptionalTime"
},
"current": {
"type": "string"
},
"event_uuid": {
"type": "string"
},
"panels": {
"properties": {
"action": {
"type": "string"
},
"heightratio": {
"type": "double"
},
"type": {
"type": "string"
},
"url": {
"type": "string"
},
"videourl": {
"type": "string"
}
}
},
"status": {
"type": "string"
},
"updated_at": {
"type": "date",
"format": "dateOptionalTime"
},
"uuid": {
"type": "string"
}
}
}
}
}
}
And:
{
"event": {
"mappings": {
"levis": {
"properties": {
"date": {
"type": "date",
"format": "dateOptionalTime"
},
"event_uuid": {
"type": "long"
},
"name": {
"type": "string"
},
"ticketmaster_game_event_name": {
"type": "string"
},
"ticketmaster_parking_event_name": {
"type": "string"
},
"time": {
"type": "date",
"format": "dateOptionalTime"
}
}
}
}
}
}
I'd ideally like to do something like select * from HomePanel InnerJoin Event On home_panel.event_uuid = event.event_uuid so I can see the event in the home panel object. I'm not clear how to do this in elastic search. Is this even possible?

Resources