elastic, can I use my own global ordinals with a painless terms aggregation script? - elasticsearch

Question

Here's what one of my document might look like
{
"CC":{"colors":["Blue","Green","Yellow"]},
"CN":{"colors":["White","Green","Blue"]},
"WA":{"colors":["Orange","Green","Blue"]},
...
}
I want a terms aggregation, on the intersection of two fields CC.colors and CN.colors. That is, for this document, that field will have ["Green", "Blue"] in the intersection, and I want a term aggregation on this intersection.
As far as I understand, there are two ways to do it.
1) A painless script in terms aggregation, which returns the intersection of these two arrays for each document.
2) A new field created during index time, maybe called CC_CN.colors, which holds intersection for all docs.
I can't go ahead with 2 because my combinations will be too many. I can have any need during search time, like CC_CN, or CC_WA, or WA_CN_CC etc.
For 1), it works, but gets painfully slow. One reason is that 1) cannot use global ordinals.
Is there any trick, that I can ask elastic to build a custom global ordinal for my painless terms aggregation? I know there are just 25 colors in my system, so can give all colors to elastic somewhere, and "assure" them that I'll not return anything else but these colors from my aggregation?
Or, if I encode and store numbers instead of strings in index, would this be faster for elastic? e.g. 0 instead of "Black", 1 instead of "Green" etc.?
Other than intersection, my other use cases involve union etc. as well. Thanks for reading!

Answer 1

To answer it myself, we ended up asking for these arrays in _source and performing union/intersection in Ruby.
It is also possible to do this in painless, and that offers a bit better performance. Elastic uses map to do aggregation, and I couldn't figure out any way to use global ordinals. I don't think its possible.
We wrote code that generates painless code to perform intersection and union between arrays.
For any future wanderer, here's what the generated code looks like:
This is for union:
Stream stream = [].stream();
String[] stream_keys = new String[] {'CC.colors', 'CN.colors'};
for (int i = 0; i < stream_keys.length; ++i) {
if (doc.containsKey(stream_keys[i])) {
stream = Stream.concat(stream, doc[stream_keys[i]].stream());
}
}
stream = stream.distinct();
And this is for intersection (stream, list_0_stream and list_1_stream intersection):
List list_0 = list_0_stream.collect(Collectors.toList());
List list_1 = list_1_stream.collect(Collectors.toList());
return stream.filter(list_0::contains).filter(list_1::contains).toArray();
The performance seems to be acceptable.

Answer 2

There are 2 method offer to you
Cost alot of my time
^ ^
ColorMap={"Blue":0,"Green":1,"Yellow":2,"White":3,"Orange":4};
ReverseColorMap=["Blue","Green","Yellow","White","Orange"];
var All={
"CC":{"colors":["Blue","Green","Yellow"]},
"CN":{"colors":["White","Green","Blue"]},
"WA":{"colors":["Orange","Green","Blue"]}
};
//Cover Encode
function EncodeColor(T1){
var T2 = JSON.parse(JSON.stringify(T1));//Clone Original
for(var i in T2){
for(var j in T2[i]["colors"]){
T2[i]["colors"][j]=ColorMap[T2[i]["colors"][j]];
}
}
return T2;
}
var NewAll=EncodeColor(All);
console.log(All);
console.log(NewAll);
function SortColor(T1){
for(var i in T1){
T1[i]["colors"].sort((a, b) => {
return a-b;
});
}
}
function BuildSameColor(T1){
var CombineNew={};
var Name_Temp=[];
for(var i in T1){
Name_Temp.push(i);
}
for(var i =0;i<Name_Temp.length;i++){
for(var j =i+1;j<Name_Temp.length;j++){//j=i+1 because CC_CC not valid CC_CN is valid etc...
CombineNew[Name_Temp[i]+"_"+Name_Temp[j]]={"colors":T1[Name_Temp[i]]["colors"].concat(T1[Name_Temp[j]]["colors"])};//combine color array
}
}
SortColor(CombineNew);//Sort Result
//Sort will reduce compare time(later) when color is a lot
for(var i in CombineNew){
var NewAr=[];
for(var j=0;j<CombineNew[i]["colors"].length-1;j++){
if(CombineNew[i]["colors"][j]==CombineNew[i]["colors"][j+1]){
NewAr.push(CombineNew[i]["colors"][j]);
}
}
CombineNew[i]["colors"]=NewAr;
}
return CombineNew;
}
var TTT=BuildSameColor(NewAll);
console.log(TTT);
//Then Decode Color
function DecodeColor(T1){
var T2 = JSON.parse(JSON.stringify(T1));//Clone Original
for(var i in T2){
for(var j in T2[i]["colors"]){
T2[i]["colors"][j]=ReverseColorMap[T2[i]["colors"][j]];
}
}
return T2;
}
var TTTQQ=DecodeColor(TTT);
console.log(TTTQQ);
//This Also work any length of color
var Examp={
"CC":{"colors":["Blue","Green","Yellow","Orange"]},
"CN":{"colors":["White","Green","Blue"]},
"WA":{"colors":["Orange","Green","Blue"]}
};
var E_Examp=EncodeColor(Examp);
var Com_E_E_Examp=BuildSameColor(E_Examp);
var D_Examp=DecodeColor(Com_E_E_Examp);
console.log(D_Examp);
Enjoin it!!!!!!!!
ColorMap={"Blue":0,"Green":1,"Yellow":2,"White":3,"Orange":4};
ReverseColorMap=["Blue","Green","Yellow","White","Orange"];
var All={
"CC":{"colors":["Blue","Green","Yellow"]},
"CN":{"colors":["White","Green","Blue"]},
"WA":{"colors":["Orange","Green","Blue"]}
};
//Cover Encode
function EncodeColor(T1){
var T2 = JSON.parse(JSON.stringify(T1));//Clone Original
for(var i in T2){
for(var j in T2[i]["colors"]){
T2[i]["colors"][j]=ColorMap[T2[i]["colors"][j]];
}
}
return T2;
}
var NewAll=EncodeColor(All);
console.log(All);
console.log(NewAll);
function SortColor(T1){
for(var i in T1){
T1[i]["colors"].sort((a, b) => {
return a-b;
});
}
}
function StaticSearch(T1,Name1,Name2){
if(Name1 in T1 && Name2 in T1 ){
var Temp= T1[Name1]["colors"].concat(T1[Name2]["colors"]);
var T2=[];
Temp.sort((a, b) => {
return a-b;
});
for(var i=0;i<Temp.length-1;i++){
if(Temp[i]==Temp[i+1]){
T2.push(Temp[i]);
}
}
var ReturnObj={};
ReturnObj[Name1+"_"+ Name2]={};
ReturnObj[Name1+"_"+ Name2]["colors"]=T2;
return ReturnObj;
}
return null;
}
var SearchResult=StaticSearch(NewAll,"CC","CN");
console.log(SearchResult);
//Then Decode Color/*
function DecodeColor(T1){
var T2 = JSON.parse(JSON.stringify(T1));//Clone Original
for(var i in T2){
for(var j in T2[i]["colors"]){
T2[i]["colors"][j]=ReverseColorMap[T2[i]["colors"][j]];
}
}
return T2;
}
var TTTQQ=DecodeColor(SearchResult);
console.log(TTTQQ);
//This Also work any length of color
var Examp={
"CC":{"colors":["Blue","Green","Yellow","Orange"]},
"CN":{"colors":["White","Green","Blue"]},
"WA":{"colors":["Orange","Green","Blue"]}
};
var E_Examp=EncodeColor(Examp);
var Com_E_E_Examp=StaticSearch(E_Examp,"WA","CC");
var D_Examp=DecodeColor(Com_E_E_Examp);
console.log(D_Examp);
Another Method Static Search