How to index files with Elasticsearch, Ingest-Mapper and NEST - elasticsearch

Question

actually I am trying to index some files (most important PDFs) with Elasticsearch and NEST client from ASP.net Core 1.0.
I found some code snippets and tried to use it for my purpose.
Those are the code segments that I use to set up the Elasticsearch index and the Pipeline:
private void SetupElasticSearch()
{
ConnectionSettings settings = new ConnectionSettings(new Uri(_appSettings.ElasticSearchSettings.Url))
.MapDefaultTypeIndices(m => m
.Add(typeof(FSDocumentFile), _appSettings.ElasticSearchSettings.IndexName)
).OnRequestCompleted(response =>
{
_logger.LogInformation(String.Format("{0} {1}", response.HttpMethod, response.Uri));
if (response.RequestBodyInBytes != null)
{
_logger.LogInformation(Encoding.UTF8.GetString(response.RequestBodyInBytes));
}
// log out the response and the response body, if one exists for the type of response
_logger.LogInformation(String.Format("{0}", response.HttpStatusCode));
if (response.ResponseBodyInBytes != null)
{
_logger.LogInformation(Encoding.UTF8.GetString(response.ResponseBodyInBytes));
}
});
ElasticClient client = new ElasticClient(settings);
CreateElasticSearchIndex(client);
CreatePipeline(client);
}
private void CreateElasticSearchIndex(ElasticClient Client)
{
Client.DeleteIndex(_appSettings.ElasticSearchSettings.IndexName);
Client.CreateIndex(_appSettings.ElasticSearchSettings.IndexName, c => c
.Settings(s => s
.Analysis(a => a
.Analyzers(ad => ad
.Custom("windows_path_hierarchy_analyzer", ca => ca
.Tokenizer("windows_path_hierarchy_tokenizer")
)
)
.Tokenizers(t => t
.PathHierarchy("windows_path_hierarchy_tokenizer", ph => ph
.Delimiter('\\')
)
)
)
)
.Mappings(m => m
.Map<FSDocumentFile>(mp => mp
.AllField(all => all
.Enabled(false)
)
.Properties(ps => ps
.Number(n => n
.Name(nn => nn.Id)
)
.Text(s => s
.Name(n => n.Comment)
//.Analyzer("windows_path_hierarchy_analyzer")
)
//.Text(s => s
// .Name(n => n.Content)
//)
.Object<Attachment>(a => a
.Name(n => n.FileData)
.Properties(p => p
.Text(t => t
.Name(n => n.Name)
)
.Text(t => t
.Name(n => n.Content)
)
.Text(t => t
.Name(n => n.ContentType)
)
.Number(n => n
.Name(nn => nn.ContentLength)
)
.Date(d => d
.Name(n => n.Date)
)
.Text(t => t
.Name(n => n.Author)
)
.Text(t => t
.Name(n => n.Title)
)
.Text(t => t
.Name(n => n.Keywords)
)
)
)
)
)
)
);
}
private void CreatePipeline(ElasticClient Client)
{
Client.PutPipeline("attachments", p => p
.Description("Document attachment pipeline")
.Processors(pr => pr
.Attachment<FSDocumentFile>(a => a
.Field(f => f.FileData.Content)
.TargetField(f => f.Content)
)
.Remove<FSDocumentFile>(r => r
.Field(f => f.FileData)
)
)
);
}
This is the definition of the the FSFileInfo classe that is used for indexing:
[ElasticsearchType(Name = "FSDocumentFile")]
public class FSDocumentFile
{
public int Id { get; set; }
/// <summary>
/// FileData Base64 encoded
/// </summary>
public string Content { get; set; }
[Attachment(Store = true)]
public Attachment FileData { get; set; }
public string Comment { get; set; }
}
This is the code that I use to index the file:
FSDocumentFile fsFile = new FSDocumentFile()
{
Id = df.DocumentFileID,
FileData = new Attachment()
{
Content = Convert.ToBase64String(fd.FileBytes),
ContentType = "application/pdf",
ContentLength = fd.FileBytes.Count(),
Name = fileName
},
Comment = "TEst Comment" + df.DocumentFileID.ToString()
};
ElasticClient client = new ElasticClient(settings);
Result callResult = client.Index<FSDocumentFile>(fsFile, fi => fi.Pipeline("attachments")).Result;
It always results in an error on Elasticsearch that says:
[2016-11-28T15:40:28,311][ERROR][o.e.a.i.IngestActionFilter] [mU3hlQ7] failed to execute pipeline [attachments]
org.elasticsearch.ElasticsearchException: java.lang.IllegalArgumentException: ElasticsearchParseException[Error parsing document in field [fileData.content]]; nested: IllegalArgumentException[field [c
ontent] not present as part of path [fileData.content]];
at org.elasticsearch.ingest.CompoundProcessor.newCompoundProcessorException(CompoundProcessor.java:156) ~[elasticsearch-5.0.0-rc1.jar:5.0.0-rc1]
at org.elasticsearch.ingest.CompoundProcessor.execute(CompoundProcessor.java:107) ~[elasticsearch-5.0.0-rc1.jar:5.0.0-rc1]
at org.elasticsearch.ingest.Pipeline.execute(Pipeline.java:58) ~[elasticsearch-5.0.0-rc1.jar:5.0.0-rc1]
at org.elasticsearch.ingest.PipelineExecutionService.innerExecute(PipelineExecutionService.java:166) ~[elasticsearch-5.0.0-rc1.jar:5.0.0-rc1]
at org.elasticsearch.ingest.PipelineExecutionService.access$000(PipelineExecutionService.java:41) ~[elasticsearch-5.0.0-rc1.jar:5.0.0-rc1]
at org.elasticsearch.ingest.PipelineExecutionService$1.doRun(PipelineExecutionService.java:65) [elasticsearch-5.0.0-rc1.jar:5.0.0-rc1]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:504) [elasticsearch-5.0.0-rc1.jar:5.0.0-rc1]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-5.0.0-rc1.jar:5.0.0-rc1]
at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) [?:1.8.0_112]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) [?:1.8.0_112]
at java.lang.Thread.run(Unknown Source) [?:1.8.0_112]
Caused by: java.lang.IllegalArgumentException: ElasticsearchParseException[Error parsing document in field [fileData.content]]; nested: IllegalArgumentException[field [content] not present as part of
path [fileData.content]];
Can someone please assist me? Or maybe point me to a good tutorial how to use Elasticsearch, Ingest-Mapper and NEST client together?
Thanks a lot in advance.