Related
I'd like to index a geojson data to elasticsearch using curl
The geojson data looks like this:
{
"type": "FeatureCollection",
"name": "telco_new_development",
"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } },
"features": [
{ "type": "Feature", "properties": { "ogc_fid": 1, "name": "Yarrabilba", "carrier_name": "OptiComm", "uid": "35", "development_name": "Yarrabilba", "stage": "None", "developer_name": "Refer to Carrier", "development_nature": "Residential", "development_type": "Sub-division", "estimated_number_of_lots_or_units": "18500", "status": "Ready for service", "developer_application_date": "Check with carrier", "contract_date": "TBC", "estimated_service_date": "30 Jul 2013", "technology_type": "FTTP", "last_modified_date": "8 Jul 2020" }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 153.101112, -27.797998 ], [ 153.09786, -27.807122 ], [ 153.097715, -27.816313 ], [ 153.100598, -27.821068 ], [ 153.103789, -27.825047 ], [ 153.106079, -27.830225 ], [ 153.108248, -27.836107 ], [ 153.110692, -27.837864 ], [ 153.116288, -27.840656 ], [ 153.119923, -27.844818 ], [ 153.122317, -27.853523 ], [ 153.127785, -27.851777 ], [ 153.131234, -27.85115 ], [ 153.135634, -27.849741 ], [ 153.138236, -27.848668 ], [ 153.141703, -27.847075 ], [ 153.152205, -27.84496 ], [ 153.155489, -27.843381 ], [ 153.158613, -27.841546 ], [ 153.161937, -27.84059 ], [ 153.156361, -27.838492 ], [ 153.157097, -27.83451 ], [ 153.15036, -27.832705 ], [ 153.151126, -27.827536 ], [ 153.15169, -27.822564 ], [ 153.148492, -27.820801 ], [ 153.148375, -27.817969 ], [ 153.139019, -27.815804 ], [ 153.139814, -27.808556 ], [ 153.126486, -27.80576 ], [ 153.124679, -27.803584 ], [ 153.120764, -27.802953 ], [ 153.121397, -27.797353 ], [ 153.100469, -27.79362 ], [ 153.099828, -27.793327 ], [ 153.101112, -27.797998 ] ] ] ] } },
{ "type": "Feature", "properties": { "ogc_fid": 2, "name": "Elliot Springs", "carrier_name": "OptiComm", "uid": "63", "development_name": "Elliot Springs", "stage": "None", "developer_name": "Refer to Carrier", "development_nature": "Residential", "development_type": "Sub-division", "estimated_number_of_lots_or_units": "11674", "status": "Ready for service", "developer_application_date": "Check with carrier", "contract_date": "TBC", "estimated_service_date": "29 Nov 2018", "technology_type": "FTTP", "last_modified_date": "8 Jul 2020" }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 146.862725, -19.401424 ], [ 146.865987, -19.370253 ], [ 146.872767, -19.370901 ], [ 146.874484, -19.354706 ], [ 146.874913, -19.354301 ], [ 146.877059, -19.356811 ], [ 146.87972, -19.35835 ], [ 146.889161, -19.359321 ], [ 146.900062, -19.367581 ], [ 146.884955, -19.38507 ], [ 146.88341, -19.402558 ], [ 146.862725, -19.401424 ] ] ] ] } },
...
However, my curl is returns an error called The bulk request must be terminated by a newline [\\n]
curl -H 'Content-Type: application/x-ndjson' -XPOST 'localhost:9200/geo/building/_bulk?pretty' --data-binary #building.geojson
{
"error" : {
"root_cause" : [
{
"type" : "illegal_argument_exception",
"reason" : "The bulk request must be terminated by a newline [\\n]"
}
],
"type" : "illegal_argument_exception",
"reason" : "The bulk request must be terminated by a newline [\\n]"
},
"status" : 400
}
Any suggestion?
your format is not suitable for _bulk like that, as it's missing the structure it expects. https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html goes into that
you need;
to update your json file to have something like { "index" : { "_index" : "INDEX-NAME-HERE" } } before each of the documents
each document also needs to be on a single line
each line needs a \n at the end of it so that the bulk API knows when the action/record ends
I am trying to loop through the array from the count of the benefits element in my json. I keep getting numeric value error although get count function in apex json returns a number.
If we change the loop to for 1 in 1..2 - it works fine.
If we hardcode the array index to 1 then the code works but if we try to pass a variable using %d p0 then i get a numeric value error - ORA - 06502.
Below is the code:
j apex_json.t_values;
l_count_chg number;
l_count_pers number;
l_count_ben number;
l_members wwv_flow_t_varchar2;
l_paths apex_t_varchar2;
l_paths2 apex_t_varchar2;
l_paths3 apex_t_varchar2;
v_get_person varchar2(10);
v_get_benefit varchar2(10);
v_get_benopt varchar2(10);
r_count number;
begin
apex_json.parse(j,'{
"PolicyUpdate": {
"contractNumber": 12345,
"effectiveDate": "2022-04-01",
"planChanges": [
{
"changeCode": 1,
"roleplayerId": "pers1",
"person": {
"id": "pers1",
"surname": "Hazy",
"firstName": "Smith",
"benefits": [
{
"Id": "ben1",
"typeCode": "C1",
"benefitName": "Funeral Benefit",
"roleplayerId": "pers1",
"coverAmount": 10000,
"premiumAmount": 47.47,
"StartDate": "2021-04-01",
"Options": {
"Option": [
{
"Label": "Risk Benefit Classification",
"Value": "Main Life"
},
{
"Label": "Waiting Period (DOC to Event)",
"Value": "6 Months"
},
{
"Label": "Paid Up Benefit (Y/N)",
"Value": "No"
}
]
}
},
{
"Id": "ben2",
"typeCode": "C1",
"benefitName": "Tombstone Benefit",
"roleplayerId": "pers1",
"coverAmount": 5000,
"premiumAmount": 10.47,
"StartDate": "2021-04-01"
}
]
}
},
{
"changeCode": 2,
"roleplayerId": "pers2"
}
]
}
}');
dbms_output.put_line('Contract Number : ' ||
apex_json.get_varchar2(p_path => 'PolicyUpdate.contractNumber'));
dbms_output.put_line('Effective Adte : ' ||
apex_json.get_varchar2(p_path => 'PolicyUpdate.effectiveDate'));
dbms_output.put_line('----------------------------------------');
l_count_chg := apex_json.get_count(p_path => 'PolicyUpdate.planChanges',p_values=>j);
dbms_output.put_line('change count '||l_count_chg);
for i in 1..l_count_chg loop
dbms_output.put_line('changeCode '||apex_json.get_varchar2(p_path => 'PolicyUpdate.planChanges[%d].changeCode' ,p_values=>j,p0 => i));
dbms_output.put_line('roleplayerId '||apex_json.get_varchar2(p_path => 'PolicyUpdate.planChanges[%d].roleplayerId',p_values=>j,p0 => i));
dbms_output.put_line('firstname '||apex_json.get_varchar2(p_path => 'PolicyUpdate.planChanges[%d].person.firstName',p_values=>j,p0 => i));
l_count_ben := apex_json.get_count(p_path => 'PolicyUpdate.planChanges[%d].person.benefits',p_values=>j);
dbms_output.put_line('benefit count '|| l_count_ben);
for x in 1..l_count_ben loop
dbms_output.put_line('ben id '||apex_json.get_varchar2(p_path => 'PolicyUpdate.planChanges[%d].person.benefits[%d].Id',p_values=>j,p0 => i,p1 => x));
end loop;
end loop;
end;
For me the count didn't return a number (just no value) so I made some changes - I found it safest to check if the path exists before getting the count:
declare
j apex_json.t_values;
l_count_chg number;
l_count_pers number;
l_count_ben number;
l_members wwv_flow_t_varchar2;
l_paths apex_t_varchar2;
l_paths2 apex_t_varchar2;
l_paths3 apex_t_varchar2;
v_get_person varchar2(10);
v_get_benefit varchar2(10);
v_get_benopt varchar2(10);
r_count number;
begin
apex_json.parse(j,'{
"PolicyUpdate": {
"contractNumber": 12345,
"effectiveDate": "2022-04-01",
"planChanges": [
{
"changeCode": 1,
"roleplayerId": "pers1",
"person": {
"id": "pers1",
"surname": "Hazy",
"firstName": "Smith",
"benefits": [
{
"Id": "ben1",
"typeCode": "C1",
"benefitName": "Funeral Benefit",
"roleplayerId": "pers1",
"coverAmount": 10000,
"premiumAmount": 47.47,
"StartDate": "2021-04-01",
"Options": {
"Option": [
{
"Label": "Risk Benefit Classification",
"Value": "Main Life"
},
{
"Label": "Waiting Period (DOC to Event)",
"Value": "6 Months"
},
{
"Label": "Paid Up Benefit (Y/N)",
"Value": "No"
}
]
}
},
{
"Id": "ben2",
"typeCode": "C1",
"benefitName": "Tombstone Benefit",
"roleplayerId": "pers1",
"coverAmount": 5000,
"premiumAmount": 10.47,
"StartDate": "2021-04-01"
}
]
}
},
{
"changeCode": 2,
"roleplayerId": "pers2"
}
]
}
}');
dbms_output.put_line('Contract Number : ' ||
apex_json.get_varchar2(p_path => 'PolicyUpdate.contractNumber',p_values => j));
dbms_output.put_line('Effective Adte : ' ||
apex_json.get_varchar2(p_path => 'PolicyUpdate.effectiveDate',p_values => j));
dbms_output.put_line('----------------------------------------');
l_count_chg := apex_json.get_count(p_path => 'PolicyUpdate.planChanges',p_values=>j);
dbms_output.put_line('change count '||l_count_chg);
for i in 1..l_count_chg loop
dbms_output.put_line('changeCode '||apex_json.get_varchar2(p_path => 'PolicyUpdate.planChanges[%d].changeCode' ,p_values=>j,p0 => i));
dbms_output.put_line('roleplayerId '||apex_json.get_varchar2(p_path => 'PolicyUpdate.planChanges[%d].roleplayerId',p_values=>j,p0 => i));
dbms_output.put_line('firstname '||apex_json.get_varchar2(p_path => 'PolicyUpdate.planChanges[%d].person.firstName',p_values=>j,p0 => i));
IF apex_json.does_exist(p_path => 'PolicyUpdate.planChanges[%d].person.benefits',p0 => i,p_values=>j) THEN
l_count_ben := apex_json.get_count(p_path => 'PolicyUpdate.planChanges[%d].person.benefits',p0 => i,p_values=>j);
dbms_output.put_line('benefit count '|| l_count_ben);
for x in 1..l_count_ben loop
dbms_output.put_line('ben id '||apex_json.get_varchar2(p_path => 'PolicyUpdate.planChanges[%d].person.benefits[%d].Id',p_values=>j,p0 => i,p1 => x));
end loop;
END IF;
end loop;
end;
Koen's suggestion of testing with EXISTS is excellent.
He also added this, but didn't spell out that your original version seemed to be missing the parameter:
p0 => i
when calculating l_count_ben
So the get_count value would have been unknown.
Is there a way to validate this array(), Nothing works so far:
[
{
"transaction": {
"user_id": 6,
"month": 12,
"year": 2084
},
"entities": [
{
"name": "Allan Botsford",
"value": 3,
"is_total": false,
"type": "CASH"
},
{
"name": "Luisa Schiller Sr.",
"value": 6266,
"is_total": false,
"type": "CASH"
},
{
"name": "Susie Deckow MD",
"value": 506700,
"is_total": false,
"type": "CASH"
}
]
},
{
"transaction": {
"user_id": 7,
"month": 5,
"year": 2002
},
"entities": [
{
"name": "Raquel Jast",
"value": 7,
"is_total": false,
"type": "CASH"
},
{
"name": "Wendell Herman I",
"value": 4480,
"is_total": false,
"type": "CASH"
},
{
"name": "Oceane Greenfelder DDS",
"value": 46344,
"is_total": false,
"type": "CASH"
}
]
}
]
I can validate the transaction with the following rules:
[
'*.transaction.month' => 'required|numeric',
'*.transaction.year' => 'required|numeric',
'*.transaction.transaction_date' => 'sometimes|date_format:Y-m-d'
]
The problem is in the nested entities array because the following rules are ignored:
return [
'*.entities.*.is_total' => 'required|boolean',
'*.entities.*.name' => 'required|string',
'*.entities.*.value' => 'required|numeric',
'*.entities.*.type' => ['required', Rule::in(CashTemporaryInvestment::TYPES)]
]
I don't find any hint on laravel documentation. I will appreciate any help. I am using Laravel 7
Laravel Validator wildcard will work using the following preg_match() as seen in the Validator class
$pattern = str_replace('\*', '([^\.]+)', preg_quote($this->getPrimaryAttribute($attribute), '/'));
So your pattern will work if you can access the attributes using
[0]['entities'][0]['is_total'] for your rule '*.entities.*.is_total' => 'required|boolean',
what would be the json mapping to insert geo data into elasticsearch ??
if the sample json data as follows:
{ "type": "Feature", "properties": { "ID": 631861455.000000, "address": "1206 UPPER", "city": "la vegas", "state": "AL", "zip_code": "15656", "OGR_GEOMETRY": "POLYGON" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ -86.477551331, 32.490605650000099 ], [ -86.477637350999899, 32.4903921820001 ], [ -86.478257247, 32.490565591000099 ], [ -86.478250466, 32.490580239000103 ], [ -86.478243988, 32.490593680000096 ], [ -86.47823751, 32.490607122 ], [ -86.478231749, 32.490619100000096 ], [ -86.478224637, 32.490634065000101 ], [ -86.47821823699999, 32.490647540000097 ], [ -86.478211847999901, 32.490661035000095 ], [ -86.478205478999897, 32.490674526000099 ], [ -86.478202107999891, 32.490681666000093 ], [ -86.478199132, 32.4906880240001 ], [ -86.478192825999898, 32.490701523 ], [ -86.478186533, 32.490715047 ], [ -86.47818320899999, 32.490722209000097 ], [ -86.47818027999989, 32.490728569000098 ], [ -86.478174063, 32.490742125000097 ], [ -86.47816785099999, 32.490755654000097 ], [ -86.47816255799999, 32.490767236000096 ], [ -86.478159053999889, 32.490774513000105 ], [ -86.477551331, 32.490605650000099 ] ] ] } }
Look at Geo point mapping.
You need to define mapping.
How is d3.nest() used with geojson files?
My geojson data is formatted as follows:
"features": [
{ "type": "Feature", "properties": { "neighborhood": "Allerton", "boroughCode": "2", "borough": "Bronx", "#id": "http:\/\/nyc.pediacities.com\/Resource\/Neighborhood\/Allerton" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ -73.848597000000183, 40.871670000000115 ], [ -73.845822536836778, 40.870239076236174 ], [ -73.854559184633743, 40.859953835764252 ], [ -73.854665433068263, 40.859585694988056 ], [ -73.856388703358959, 40.857593635304482 ], [ -73.868881809153407, 40.857223150158326 ], [ -73.868317552728243, 40.857862062258313 ], [ -73.869553714672321, 40.857784095600181 ], [ -73.871024857620654, 40.857309948816905 ], [ -73.870480549987164, 40.865413584098484 ], [ -73.87055489856489, 40.869702798589863 ], [ -73.86721594442561, 40.869689663636713 ], [ -73.85745, 40.869533000000182 ], [ -73.855550000000108, 40.871813000000145 ], [ -73.853597967576576, 40.873288368674203 ], [ -73.848597000000183, 40.871670000000115 ] ] ] } }
But my nest command:
var nested_data = d3.nest()
.key(function(d, i) { console.log(d); return d.features.properties.neighborhood; })
.entries(map);
returns an empty array.
I want to nest my data to more easily filter it. Is this advised?
Assuming your geojson looks like the below
var map = {
type: "FeatureCollection",
"features": [
{
"type": "Feature",
"properties": {
"neighborhood": "Allerton",
"boroughCode": "2",
"borough": "Bronx",
"#id": "http:\/\/nyc.pediacities.com\/Resource\/Neighborhood\/Allerton"
},
"geometry": { /* various coordinates, etc */ }
]
}
So, what you want to do is:
d3.nest()
.key(function(d, i) {
return d.properties.neighborhood;
})
.entries(map.features);
You want to pass map.features since that's your array.