ElasticSearch version 7.13
Index already exist and I want to reindex with mapping, the field is a boolean. But when I'm trying to reindex, the field has "1" and "0" (string).
How can I evaluate if field = "1" set true (same for 0, but false)?
I have read about runtime, but can't figure out how does it work.
my mapping
{
mappings:{
"OPTIONS": {
"type": "nested",
"properties":{
"COMBINABLE": {
"type": "boolean"
}
}
}
}
}
and document
{
"options": [
{
"COMBINABLE": "0"
}
]
}
You might consider using pipeline ingestion to convert your number to a boolean value, you can do something like this:
POST _ingest/pipeline/_simulate
{
"pipeline": {
"description": "convert to boolean",
"processors": [
{
"script": {
"source": "def options = ctx.options;def pairs = new ArrayList();for (def pair : options) {def k = false;if (pair[\"COMBINABLE\"] == \"1\" || pair[\"COMBINABLE\"] == 1) {k = true;}pair[\"COMBINABLE\"] = k;}ctx.options = options;"
}
}
]
},
"docs": [
{
"_source": {
"options": [
{
"COMBINABLE": 1
}
]
}
}
]
}
The painless script above is pretty simple:
def options = ctx.options;
def pairs = new ArrayList();
for (def pair : options) {
def k = false;
if (pair["COMBINABLE"] == "1" || pair["COMBINABLE"] == 1) {
k = true;
}
pair["COMBINABLE"] = k;
}
ctx.options = options;
It simply loop through all your option under options, then if the COMBINABLE is 1 or "1", it will convert to true, otherwise, it will be false. You can set the pipeline as your default ingestion, see here
I have this rake task which uses rest-client to fetch some messy JSON from this API, and then uses hashie to make the code prettier.
Unfortunately I'm unable to fetch one of the deeply nested values, productGroup. If working correctly, it should output :category => "Jeans" or similar. Please see the JSON at the bottom.
This did not work:
mash.deep_fetch(:fields, 0).deep_locate(-> (key, value, object) { value.include?("product_group") }) { "ERROR: category" }
Example output:
% rake get_products
{:category=>nil, :name=>"Luxurous Jumpsuit", :image=>"http://nlyscandinavia.scene7.com/is/image/nlyscandinavia/productLarge/129579-0012.jpg", :price=>"599", :description=>"Lorem ipsum dolor"}
Example mash:
#<Hashie::Mash brand="Dr Denim" categories=[#<Hashie::Mash name="Kvinne > KLÆR > Jeans > Slim">] description="Lorem ipsum dolor." fields=[#<Hashie::Mash name="sale" value="false">, #<Hashie::Mash name="product_id_original" value="226693-7698">, #<Hashie::Mash name="gender" value="Kvinne">, #<Hashie::Mash name="artNumber" value="226693-7698">, #<Hashie::Mash name="productGroup" value="Jeans">, #<Hashie::Mash name="productStyle" value="Slim">, #<Hashie::Mash name="extraImageProductSmall" value="http://nlyscandinavia.scene7.com/is/image/nlyscandinavia/cart_thumb/226693-7698.jpg">, #<Hashie::Mash name="productClass" value="Klær">, #<Hashie::Mash name="extraImageProductLarge" value="http://nlyscandinavia.scene7.com/is/image/nlyscandinavia/productLarge/226693-7698.jpg">, #<Hashie::Mash name="sizes" value="W24/L32,W25/L32,W26/L32,W27/L32,W28/L32,W29/L32,W30/L32,W31/L32,W32/L32,W26/L30,W27/L30,W28/L30,W29/L30,W24/L30,W25/L30,W32/L30,W31/L30,W30/L30">, #<Hashie::Mash name="color" value="Mid Blue">] identifiers=#<Hashie::Mash sku="226693-7698"> language="no" name="Regina Jeans" offers=[#<Hashie::Mash feed_id=10086 id="2820760a-c5b2-494a-b5dd-ab713f796cb9" in_stock=1 modified=1474947357838 price_history=[#<Hashie::Mash date=1474949513421 price=#<Hashie::Mash currency="NOK" value="599">>] product_url="http://pdt.tradedoubler.com/click?a1234" program_logo="http://hst.tradedoubler.com/file/17833/2014-logos/200X200.png" program_name="Nelly NO" source_product_id="226693-7698">] product_image=#<Hashie::Mash url="http://nlyscandinavia.scene7.com/is/image/nlyscandinavia/productLarge/226693-7698.jpg">>
get_products.rake:
# encoding: utf-8
# Gets messy JSON from other store via REST client and cleans it up with Hashie
require "rest_client"
require "hashie"
Product = Struct.new(:category, :name, :image, :price, :description)
module ProductsFromOtherStore
CATEGORIES = [
"festkjoler",
"jakker",
"jeans",
"jumpsuit",
"vesker"
]
def self.fetch
CATEGORIES.map do |category|
Tradedoubler.fetch category
end
end
# Prettify, ie. `fooBar` => `foo_bar`
def self.prettify(x)
case x
when Hash
x.map { |key, value| [key.underscore, prettify(value)] }.to_h
when Array
x.map { |value| prettify(value) }
else
x
end
end
end
class ProductsFromOtherStore::Tradedoubler
KEY = "FE34B1309AB749F1578AEE87D9D74535513F6B54"
# Products to fetch from API
LIMIT = 2
def self.fetch category
new(category).filtered_products.take(LIMIT)
rescue RestClient::RequestTimeout => e
Array.new
end
def initialize category
#category = category
# API doesn't support gender or category searches, so do some filtering based on available JSON fields
#filters = Array.new
define_filter { |mash|
mash.fields.any? { |field|
field.name == "gender" && field.value.downcase == "kvinne"
}
}
define_filter { |mash|
mash.categories.any? { |category|
category.name.underscore.include? #category
}
}
end
def define_filter(&filter)
#filters << filter
end
def filtered_products
filtered_mashes.map { |mash|
# puts mash
Product.new(
# mash.deep_fetch(:fields, 0).find { |field| field[:name] == "product_group" }[:value],
mash.deep_fetch(:fields, 0).deep_locate(-> (key, value, object) { value.include?("product_group") }) { "ERROR: category" },
mash.deep_fetch(:name) { "ERROR: name" },
mash.deep_fetch(:product_image, :url) { "ERROR: image URL" },
mash.deep_fetch(:offers, 0, :price_history, 0, :price, :value) { "ERROR: price" },
mash.deep_fetch(:description) { "ERROR: description" }
)
}
end
private
def request
response = RestClient::Request.execute(
:method => :get,
:url => "http://api.tradedoubler.com/1.0/products.json;q=#{ URI.encode(#category) };limit=#{ LIMIT }?token=#{ KEY }",
:timeout => 0.4
)
end
def hashes
ProductsFromOtherStore.prettify(JSON.parse(request)["products"])
end
def mashes
hashes.map { |hash| Hashie::Mash.new(hash) }.each do |mash|
mash.extend Hashie::Extensions::DeepFetch
mash.extend Hashie::Extensions::DeepLocate
end
end
def filtered_mashes
mashes.select { |mash| mash_matches_filter? mash }
end
def mash_matches_filter? mash
# `.all?` requires all filters to match, `.any?` requires only one
#filters.all? { |filter| filter.call mash }
end
end
# All that for this
task :get_products => :environment do
#all_products_from_all_categories = ProductsFromOtherStore.fetch
#all_products_from_all_categories.each do |products|
products.each do |product|
puts product.to_h
end
end
end
The messy JSON we got via rest-client:
{
"productHeader": {
"totalHits": 367
},
"products": [{
"name": "501 CT Jeans For Women",
"productImage": {
"url": "http://nlyscandinavia.scene7.com/is/image/nlyscandinavia/productLarge/441576-1056.jpg"
},
"language": "no",
"description": "Jeans fra Levi's. Noe kortere nederst, fem lommer. Normal høyde på midjen, med hemper i linningen og knappegylfen. Dekorative slitte partier foran og nederst på benet.<br />Laget av 100% bomull.",
"brand": "Levis",
"identifiers": {
"sku": "441576-1056"
},
"fields": [{
"name": "sale",
"value": "false"
}, {
"name": "sizes",
"value": "W24/L32,W25/L32,W26/L32,W27/L32,W28/L32,W29/L32,W30/L32,W31/L32,W25/L34,W26/L34,W27/L34,W28/L34,W29/L34,W30/L34"
}, {
"name": "productStyle",
"value": "Straight"
}, {
"name": "gender",
"value": "Kvinne"
}, {
"name": "product_id_original",
"value": "441576-1056"
}, {
"name": "productGroup",
"value": "Jeans"
}, {
"name": "extraImageProductLarge",
"value": "http://nlyscandinavia.scene7.com/is/image/nlyscandinavia/productLarge/441576-1056.jpg"
}, {
"name": "extraImageProductSmall",
"value": "http://nlyscandinavia.scene7.com/is/image/nlyscandinavia/cart_thumb/441576-1056.jpg"
}, {
"name": "artNumber",
"value": "441576-1056"
}, {
"name": "productClass",
"value": "Klær"
}, {
"name": "color",
"value": "Indigo"
}],
"offers": [{
"feedId": 10086,
"productUrl": "http://pdt.tradedoubler.com/click?a(2402331)p(80279)product(57d37b9ce4b085c06c38c96b)ttid(3)url(http%3A%2F%2Fnelly.com%2Fno%2Fkl%C3%A6r-til-kvinner%2Fkl%C3%A6r%2Fjeans%2Flevis-441%2F501-ct-jeans-for-women-441576-1056%2F)",
"priceHistory": [{
"price": {
"value": "1195",
"currency": "NOK"
},
"date": 1473477532181
}],
"modified": 1473477532181,
"inStock": 1,
"sourceProductId": "441576-1056",
"programLogo": "http://hst.tradedoubler.com/file/17833/2014-logos/200X200.png",
"programName": "Nelly NO",
"id": "57d37b9ce4b085c06c38c96b"
}],
"categories": [{
"name": "Kvinne > KLÆR > Jeans > Straight"
}]
}, {
"name": "501 CT Jeans For Women",
"productImage": {
"url": "http://nlyscandinavia.scene7.com/is/image/nlyscandinavia/productLarge/441576-6581.jpg"
},
"language": "no",
"description": "Jeans fra Levi's. Noe kortere nederst, fem lommer. Normal høyde på midjen, med hemper i linningen og knappegylfen. Dekorative slitte partier foran og nederst på benet.<br />Laget av 100% bomull.",
"brand": "Levis",
"identifiers": {
"sku": "441576-6581"
},
"fields": [{
"name": "sale",
"value": "false"
}, {
"name": "artNumber",
"value": "441576-6581"
}, {
"name": "productStyle",
"value": "Straight"
}, {
"name": "gender",
"value": "Kvinne"
}, {
"name": "extraImageProductLarge",
"value": "http://nlyscandinavia.scene7.com/is/image/nlyscandinavia/productLarge/441576-6581.jpg"
}, {
"name": "extraImageProductSmall",
"value": "http://nlyscandinavia.scene7.com/is/image/nlyscandinavia/cart_thumb/441576-6581.jpg"
}, {
"name": "productGroup",
"value": "Jeans"
}, {
"name": "product_id_original",
"value": "441576-6581"
}, {
"name": "productClass",
"value": "Klær"
}, {
"name": "color",
"value": "Desert"
}, {
"name": "sizes",
"value": "W24/L32,W25/L32,W26/L32,W27/L32,W28/L32,W29/L32,W30/L32,W31/L32,W25/L34,W26/L34,W27/L34,W28/L34,W29/L34,W30/L34,W31/L34"
}],
"offers": [{
"feedId": 10086,
"productUrl": "http://pdt.tradedoubler.com/click?a(2402331)p(80279)product(57b3cafbe4b06cf59bc254bf)ttid(3)url(http%3A%2F%2Fnelly.com%2Fno%2Fkl%C3%A6r-til-kvinner%2Fkl%C3%A6r%2Fjeans%2Flevis-441%2F501-ct-jeans-for-women-441576-6581%2F)",
"priceHistory": [{
"price": {
"value": "1195",
"currency": "NOK"
},
"date": 1471400699283
}],
"modified": 1471400699283,
"inStock": 1,
"sourceProductId": "441576-6581",
"programLogo": "http://hst.tradedoubler.com/file/17833/2014-logos/200X200.png",
"programName": "Nelly NO",
"id": "57b3cafbe4b06cf59bc254bf"
}],
"categories": [{
"name": "Kvinne > KLÆR > Jeans > Straight"
}]
}]
}
There is a lot of things going on in your code sample. I tried to split in parts and restructure it. It does not do the same as your code but I think it should get you started and perhaps you can come back when you have a more specific question.
Note that I did not use hashie, I think that accessing some deeply nested hash structures in a few places does not justify adding a new library to a project.
Questions/Ideas/Hints:
are prices Integers or Floats?
Is the JSON consistent (all elements present all the time?)
Are you using Ruby 2.3? Then look into Hash#dig
Why did you prettify the JSON keys? Does not make sense to me as you build Product objects to work with anyway?
Unless there are performance issues i would convert all products to Ruby objects first and filter then. Just easier and more readable.
Code
Product (same as yours)
Product = Struct.new(:category, :name, :image, :price, :description)
JsonProductBuilder converts the parsed JSON to Product Objects.
class JsonProductBuilder
def initialize(json)
#json = json
end
def call
json.fetch('products', []).map do |item|
Product.new(
extract_category(item),
item['name'],
item.fetch('productImage', {})['url'],
extract_price(item),
item['description']
)
end
end
private
attr_reader :json
def extract_category(item)
field = item['fields'].find do |field|
field['name'] == 'productGroup'
end
field['value'] if field
end
def extract_price(item)
offer = item['offers'].first
history = offer['priceHistory'].first
value = history['price']['value']
Integer(value) # Or use Float?
end
end
CategoryFilter returns a limited subset of the products. You can easily add other filters and combine them. Perhaps you might want to look into lazy for performance improvements.
class CategoryFilter
def initialize(products, *categories)
#products = products
#categories = categories
end
def call
products.select do |product|
categories.include?(product.category)
end
end
private
attr_reader :products, :categories
end
Use it like this:
limit = 10
categories = ['laptop', 'something']
params = {
q: categories.join(','),
limit: limit,
}
paramsString = params.map do |key, value|
"#{key}=#{value}"
end.join(';')
response = RestClient.get(
"http://api.tradedoubler.com/1.0/products.json;#{paramsString}?token=#{token}"
)
json = JSON.parse(response)
products = JsonProductBuilder.new(json).call
puts products.size
products = CategoryFilter.new(products, 'Klær', 'Sko', 'Jeans').call
puts products.size
products.each do |product|
puts product.to_h
end
How can i change my tree made of Array of hashes into another structure such as:
My data looks like :
{
"A": [
{ "A1": [] },
{ "A2": [] },
{
"A3": [
{
"A31": [
{ "A311": [] },
{ "A312": [] }
]
}
]
}
]
}
into something like :
{
"name": "A",
"children": [
{ "name": "A1" },
{ "name": "A2" },
{
"name": "A3",
"children": [
{
"name": "A31",
"children": [
{ "name": "A311" },
{ "name": "A312" }
]
}
]
}
]
}
I tried a few things but nothing worked as I hoped.
This is how i move into my tree
def recursive(data)
return if data.is_a?(String)
data.each do |d|
keys = d.keys
keys.each do |k|
recursive(d[k])
end
end
return data
end
I tried my best to follow how to ask so to clarify :
The tree can have a unlimited deeph
Names are more complexe than A1, A2 ...
λ = ->(h) { [h[:name], h[:children] ? h[:children].map(&λ).to_h : []] }
[λ.(inp)].to_h
#⇒ {
# "A" => {
# "A1" => [],
# "A2" => [],
# "A3" => {
# "A31" => {
# "A311" => [],
# "A312" => []
# }
# }
# }
# }
This solution returns hashes that are not wrapped in arrays inside. If you really want to wrap nested hashes with arrays, map them in λ.
When you don't know how to implement something, always think the simplest case first.
Step 1: Convert {"A1" => []} to{"name" => "A1", "children" => []}
This is simple
def convert(hash)
pair = hash.each_pair.first
["name", "children"].zip(pair).to_h
end
Step2: Recursively convert all hashes in children
def convert(hash)
pair = hash.each_pair.first
pair[1] = pair[1].map{|child| convert(child)}
["name", "children"].zip(pair).to_h
end
Step 3: Handle corner cases
If children is empty then omit it.
def convert(hash)
pair = hash.each_pair.first
pair[1] = pair[1].map{|child| convert(child)}
result = {"name" => pair[0]}
result.merge!("children" => pair[1]) if pair[1].any?
result
end
Im trying to iterate through a returned response from the Facebook Graph api
def get_feed
uri = URI(FACEBOOK_URL)
response = HTTParty.get(uri)
results = JSON.parse(response.body)['data']
puts formatted_data(results)
end
def formatted_data(results)
return unless results
formatted = results['data'].each do |d|
unless d.nil?
{
message: d['message'],
}
end
formatted.delete_if {|x| x.nil?}
end
end
The response is very large so here is a snippet if it helps
{
"data": [
{
"id": "197958940234297_827831980580320",
"from": {
"category": "Amateur sports team",
"category_list": [
{
"id": "189018581118681",
"name": "Sports Club"
},
{
"id": "139721016091877",
"name": "Outdoor Recreation"
},
{
"id": "109615542448700",
"name": "Physical Fitness"
}
],
"name": "Varsity Vandals",
"id": "197958940234297"
},
"to": {
"data": [
{
"id": "668983363",
"name": "Heather Walker"
},
{
"id": "638195502",
"name": "Emma Williams"
},
{
"id": "1286337937",
"name": "Becky Williams"
}
]
},
"with_tags": {
"data": [
{
"id": "668983363",
"name": "Heather Walker"
},
{
"id": "638195502",
"name": "Emma Williams"
},
{
"id": "1286337937",
"name": "Becky Williams"
}
]
},
"message": "Great turnout for the women's intro session today. Cool to have a women's game and a men's game running side by side. Touch is for all.",
"picture": "https://fbcdn-photos-f-a.akamaihd.net/hphotos-ak-prn2/t1.0-0/1507550_827829843913867_410211203232735862_s.jpg",
"link": "https://www.facebook.com/photo.php?fbid=827829843913867&set=pcb.827831980580320&type=1&relevant_count=2",
"icon": "https://fbstatic-a.akamaihd.net/rsrc.php/v2/yz/r/StEh3RhPvjk.gif",
"actions": [
{
"name": "Comment",
"link": "https://www.facebook.com/197958940234297/posts/827831980580320"
},
{
"name": "Like",
"link": "https://www.facebook.com/197958940234297/posts/827831980580320"
}
],
"privacy": {
"value": ""
},
I am getting an error
TypeError: no implicit conversion of String into Integer
At the moment i would just like to pull out all the Messages from the JSON object...Am i handling the extraction correctly
Any help appreciated
Thanks
I tried you code, I change you require is move formatted.delete_if {|x| x.nil?} out of loop, like following, as formatted will be nil inside the loop.
def formatted_data(results)
return unless results
formatted = results['data'].each do |d|
unless d.nil?
{
message: d['message'],
}
end
end
formatted.delete_if {|x| x.nil?}
end
are you sure your not using the data key twice?
results = JSON.parse(response.body)['data'] in main method and formatted = results['data'].each in your formatted_data method?
Thinking maybe?
def def formatted_data(results)
return unless results
results['data'].map {|m| {message: m['message']} }.compact
end
I'd do this:
def get_feed
uri = URI(FACEBOOK_URL)
response = HTTParty.get(uri)
messages = format_data(response)
for message in messages do
puts message
end
end
def format_data(response, new_data = [])
if response.present?
results = JSON.parse(response)
for result in results do
new_data << result[:data][:message] if result[:data][:message].present?
end
return new_data #-> array of messages
end
end
I am trying to move data between services and need to remove a reoccurring hash from a large record that contains both hashes and arrays.
The hash to remove from every section of the record is
{
"description": "simple identifier",
"name": "id",
"type": "id"
},
Heres example data :
{"stuff": { "defs": [
{
"description": "simple identifiery",
"name": "id",
"type": "id"
},
{
"name": "aDate",
"type": "date"
},
{
"defs": [
{
"description": "simple identifier",
"name": "id",
"type": "id"
},
{
"case-sensitive": true,
"length": null,
"name": "Id",
"type": "string"
},
{
"name": "anotherDate",
"type": "dateTime"
}
],
},
{
"defs": [
{
"description": "simple identifier",
"name": "id",
"type": "id"
},
...lots more....
I created a couple recursive function to remove the element(s) but I'm left with an empty hash '{}'. I also tried to remove the parent but found that I removed the hashes parent and not the hash itself.
I'm pretty sure I could create a new hash and populate it with the data I want but there must be a way to do this.
I am not working in rails and would like to avoid using rails gems.
I figured this out by looking at the data structure closer. The elements that need to be removed are always in an array so before recursing check if the hash key/value exists and delete if so. I'm sure this could be coded better so let me know what you think.
def recursive_delete!(node, key, value)
if node.is_a?(Array)
node.delete_if { |elm| elm[key] == value }
node.each do |elm|
recursive_delete!(elm, key, value)
end
elsif node.is_a?(Hash)
node.each_value do |v|
recursive_delete!(v, key, value)
end
end
end
If you are looking for the way to delete the same hash as you have inside complex Array/Hash data structure, it's easy:
def remove_hash_from(source, hsh)
return unless source.is_a?(Hash) || source.is_a?(Array)
source.each do |*args|
if args.last == hsh
source.delete(args.first)
elsif args.last.is_a?(Hash) || args.last.is_a?(Array)
remove_hash_from(args.last, hsh)
end
end
source
end
data = [
{h: 'v',
j: [{h: 'v'},
{a: 'c'},
8,
'asdf']
},
asdf: {h: 'v', j: 'c'}
]
remove_hash_from(data, {h: 'v'})
# => [{:h=>"v", :j=>[{:a=>"c"}, 8, "asdf"]}, {:asdf=>{:h=>"v", :j=>"c"}}]
Possibly, you will need to adjust method above for your needs. But common idea is clear, I hope.