I'm using Ruby 2.1.1 When I run this code:
<CSV.foreach("public/data/original/example_data.csv",headers: true, converters: :numeric) do |info|
I get an error:
No such file or directory # rb_sysopen
It works if I place example_data.csv in the same directory as shown below, but my boss said it can't be that way he wants all *.csv files in a different directory:
<CSV.foreach("example_data.csv",headers: true, converters: :numeric) do |info|
I had to use a workaround that bypassed File Utilities. Using thoughtbot/paperclip generated a directory called csvcontroller. I placed the csv file in that directory folder.
class Uploader < ActiveRecord::Base
attr_accessible :purchase_name, :item_description, :item_price, :purchase_count,
:merchant_address, :merchant_name, :csvdata
has_attached_file :csvdata, :url => "/csvcontroller/:basename.:extension",
:path => ":rails_root/csvcontroller/:basename.:extension"
#:default_url => "/controllers/original/example_data.csv"
validates_attachment_content_type :csvdata, :content_type => ["text/csv"]
end
Then I placed my parser in that directory to avoid using FileUtils
require 'csv'
#total_cost = 0
#errors out FileUtils.move '/public/data/original/example_data.csv', '/controllers'
#errors out require File.expand_path('../app/public/data/original/', __FILE__)
# errors outCSV.foreach("any_path_name_outside_the_same_directory/example_data.csv",
#headers: true, converters: :numeric) do |info|
CSV.foreach("example_data.csv", headers: true, converters: :numeric) do |info|
a =(info["item price"]).to_f
b = (info["purchase count"]).to_i
#total_cost += a * b
#store = []
customer = []
customer << info["purchaser name"]
#store << info["item description"]
#store << (info["item price"]).to_f
#store << (info["purchase count"]).to_i
#store << info["merchant address"]
#store << info["merchant name"]
puts #customer
puts #store
puts #total_cost
end
It looks ugly but that is what it is.
I could not get the FileUtils:: class to work properly. This is a Ruby bug for 2.1.1
Related
I am newbie in ruby and working on making performance automation framework using using 'ruby-jmeter' gem (provided by flood.io)
I have made following files structure
Where payload.rb and request.rb contains my common utility methods . Which I am calling from test.rb
(test.rb would be going to be written by QA ppl)
Performance Automation Framework Structure
request.rb (lying under 'common' folder)
require 'ruby-jmeter' #(Consider any 3rd party gem )
require 'rubygems'
module RubyJmeter
class ExtendedDSL < DSL #'ruby-jmeter' class extending
def get_common_headers()
commonHeaderHashMap = {
'tns' => 'urn:',
'Content-Type' => 'text/xml; charset=utf-8',
'Accept-Language' => 'en-US,en;q=0.5',
'Accept-Encoding' => 'gzip, deflate'
}
return commonHeaderHashMap
end
end
end
class Request
def initialize(dsl)
#dsl = dsl
end
def soap_post_web_request(name,rawBody)
endPoint = '/SoapEndPoint'
post name: name, url: endPoint , raw_body: rawBody do
tempHeaderHashMap = get_common_headers.merge( {'SOAPAction' =>
'urn:'+name} )
finalHeaderArray = []
tempHeaderHashMap.each {|key, value|
localHashMap = Hash.new
localHashMap = {:name => key, :value => value}
finalHeaderArray << localHashMap
}
header finalHeaderArray
end # End of soapCall
end #end of soap_post_web_request
# Passes method calls through to the underlying DSL (ruby-jmeter).
def method_missing method, *args, &block
#dsl.__send__ method, *args, &block
end
end
wrappingclasses_under_single_module.rb (lying under 'common' folder)
require 'rubygems'
require 'ruby-jmeter'
require 'require_all'
require_all './'
module PerformanceAutomation
def self.MyRequest
Request.new(self)
end
end
test.rb (lying under 'testflow->simpleflow' folder)
require 'ruby-jmeter' #(Consider any 3rd party gem )
require 'rubygems'
require 'require_all' #(another 3rd party gem)
require_all '../../common/'
include PerformanceAutomation
test name:'JiraAnalyticsPerformanceFlow' do
threads name: 'NoOfUsers',scheduler: false,continue_forever: false,
count: 1 do
PerformanceAutomation.MyRequest.soap_post_web_request('soapRequestmethodName',rawdata)# End of Soap request 'soapRequestmethodName'
end # End of TestPlan
view_results_tree
puts "JMX FILE IS GONNA SAVED # "+Dir.pwd+"/CreatedJMeter_DB2Navigation.jmx"
end.jmx(file: Dir.pwd+"/CreatedJMeter_DB2Navigation.jmx")
When running test.rb , I am getting following error
`<top (required)>': uninitialized constant PerformanceAutomation (NameError)
Edit (It is working fine now)
Updated common utility files by using
request.rb
module RubyJmeter
class ExtendedDSL < DSL
def get_admin_common_headers()
commonHeaderHashMap = {
'X-XSRF-TOKEN' => '${COOKIE_XSRF-TOKEN}',
'Content-Type' => 'text/xml; charset=utf-8',
'Accept-Language' => 'en-US,en;q=0.5',
'Accept-Encoding' => 'gzip, deflate'
}
return commonHeaderHashMap
end
end
end
module API
class AdminWebService
def initialize(dsl)
#dsl = dsl
end
## Admin request for 'get_space_properties'
def get_space_properties(rawBody)
endPoint = "/AdminService.asmx"
post name: "admin_GetSpaceProperties", url: endPoint ,
raw_body:rawBody do
tempHeaderHashMap = get_admin_common_headers.merge( {'SOAPAction' =>
'http://example.com/GetSpaceProperties'} )
finalHeaderArray = []
tempHeaderHashMap.each {|key, value|
localHashMap = Hash.new
localHashMap = {:name => key, :value => value}
finalHeaderArray << localHashMap
}
header finalHeaderArray
end # End get_space_properties
end
test.rb
require 'rubygems'
require 'ruby-jmeter'
require 'require_all'
require_all '../../../common/'
defaults domain: 'example.com', protocol: 'http', connect_timeout:
'2000', response_timeout: '3000'
cookies policy: 'compatibility', clear_each_iteration: true
#'cookies' is method defined under 'ruby-jmeter'
cache clear_each_iteration: true # 'cache' is method defined under
'ruby-jmeter'
# starting testPlan 'JiraAnalyticsPerformanceFlow'
test name:"testFlowName" do
adminwebservice = API::AdminWebService.new(self)
adminwebservice.get_space_properties(Payload.get_local_payload("get_space_properties.xml"))
#Payload is another common utility class for fetching payload stuff
end
puts "JMX FILE IS GONNA SAVED #
"+Dir.pwd+"/CreatedJMeter_DB2Navigation.jmx"
end.jmx(file: Dir.pwd+"/CreatedJMeter_DB2Navigation.jmx")
The following models are linked via belongs_to:
require 'mongoid'
class Sensor
include Mongoid::Document
field :sensor_id, type: String
validates_uniqueness_of :sensor_id
end
...
require 'mongoid'
require_relative 'sensor.rb'
class SensorData
include Mongoid::Document
belongs_to :sensor
field :date, type: Date
field :ozonMax1h, type: Float
field :ozonMax8hMittel, type: Float
index({ date: 1, sensor_id: 1 }, { unique: true })
end
Here is a Sinatra app which provides a few API paths based on these models:
require 'sinatra'
require 'csv'
require_relative './models/sensor.rb'
require_relative './models/sensor_data.rb'
configure do
Mongoid.load!('./mongoid.yml')
end
def prepare_for_export(sensor_data)
converted_data = sensor_data.asc(:date).map do |e|
{
sensor_id: e.sensor.nil? ? :null : e.sensor.sensor_id,
date: e.date,
ozonMax1h: e.ozonMax1h,
ozonMax8hMittel: e.ozonMax8hMittel
}
end
converted_data
end
def convert_to_json(sensor_data)
prepare_for_export(sensor_data).to_json
end
def convert_to_csv(sensor_data)
data = prepare_for_export sensor_data
csv_string = CSV.generate do |csv|
csv << data.first.keys
data.each do |hash|
csv << hash.values
end
end
csv_string
end
def get_recent
max_date = SensorData.max(:date)
SensorData.where(date: max_date)
end
def get_for_year(year)
SensorData.where(:date.gte => Date.new(year, 1, 1)).where(:date.lte => Date.new(year, 12, 31))
end
def get_for_sensor(sensor)
foo = SensorData.where(sensor_id: sensor)
puts "hallo"
return foo
end
get '/api/v1/stations' do
content_type :json
Sensor.all.map { |e| {sensor_id: e.sensor_id} }.to_json
end
get '/api/v1/sensordata/:year' do
content_type :json
convert_to_json get_for_year(params[:year].to_i)
end
get '/api/v1/sensordata/:year/csv' do
convert_to_csv get_for_year(params[:year].to_i)
end
get '/api/v1/recent' do
content_type :json
convert_to_json get_recent
end
I would like to output the SensorData for a particular sensor such as here:
/api/v1/stations/:sensor_id/sensordata/:year/csv
I am not sure what you are trying to do or even if you are still looking for an answer but here it goes. Something seems wrong with the models in the example you have here. Sounds like part of what you are doing would work if Sensor knows about sensor_data. So might need to add this to Sensor class:
has_many :sensor_data
Though the singular of data is datum. The class would be expected to be SensorDatum. If you can't change it, you need to tell Mongoid the class_name to expect in the has_many is actuall SensorData.
You CAN specify foreign_key in Mongoid with belongs_to.
You CANNOT filter with the belongs_to like you can with ActiveRecord, but you can use scopes outside of the belongs_to to get the same effect. Exampe:
belongs_to :sensor
scope :for_year, -> (year) { where(:date.gte => Date.new(2015,1,1)).where(:date.lte => Date.new(2015, 12, 31))}
or
belongs_to :sensor
def self.for_year year
where(:date.gte => Date.new(year,1,1)).where(:date.lte => Date.new(year, 12, 31))
end
So your query would become something like this:
sensor = Sensor.find_by(sensor_id: params[:sensor_id])
sensor.sensor_data.for_year(2015)
I can't seem to figure this out. I have the following class:
class CSV_Email
attr_accessor :client_array, :email_array
def load(file)
#file = file
#Parse csv file into ruby arrays...
#Column Headers - Email, Client
#client_array = []
#email_array = []
CSV.foreach(file, :col_sep => ",", :headers => :first_row, :return_headers => false) do |column|
client_array << column[0]
email_array << column[1]
end
end
end
Now I need to access client_id_array and email_array. I tried this:
test = CSV_Email.new
test.load("Email_Test.csv")
puts client_array
But I receive a undefined local variable client_array error. How can I access that variable?
I am using ruby 1.9.3.
You need to use the object you created:
puts test.client_array
I wrote this ruby script to scrape product info from the manufacturer website. The scraping and storage of the product objects in an array works, but I can't figure out how to export the array data to a csv file. This error is being thrown:
scraper.rb:45: undefined method `send_data' for main:Object (NoMethodError)
I do not understand this piece of code. What's this doing and why isn't it working right?
send_data csv_data,
:type => 'text/csv; charset=iso-8859-1; header=present',
:disposition => "attachment; filename=products.csv"
Full code:
#!/usr/bin/ruby
require 'rubygems'
require 'anemone'
require 'fastercsv'
productsArray = Array.new
class Product
attr_accessor :name, :sku, :desc
end
# Scraper Code
Anemone.crawl("http://retail.pelicanbayltd.com/") do |anemone|
anemone.on_every_page do |page|
currentPage = Product.new
#Product info parsing
currentPage.name = page.doc.css(".page_headers").text
currentPage.sku = page.doc.css("tr:nth-child(2) strong").text
currentPage.desc = page.doc.css("tr:nth-child(4) .item").text
if currentPage.sku =~ /#\d\d\d\d/
currentPage.sku = currentPage.sku[1..-1]
productsArray.push(currentPage)
end
end
end
# CSV Export Code
products = productsArray.find(:all)
csv_data = FasterCSV.generate do |csv|
# header row
csv << ["sku", "name", "desc"]
# data rows
productsArray.each do |product|
csv << [product.sku, product.name, product.desc]
end
end
send_data csv_data,
:type => 'text/csv; charset=iso-8859-1; header=present',
:disposition => "attachment; filename=products.csv"
If you are new to Ruby, you should be using Ruby 1.9 or later, in which case you can use the built-in CSV output which builds in fast csv plus l18n support:
require 'csv'
CSV.open('filename.csv', 'w') do |csv|
csv << [sku, name, desc]
end
http://ruby-doc.org/stdlib-1.9.2/libdoc/csv/rdoc/CSV.html
File.open('filename.csv', 'w') do |f|
f.write(csv_data)
end
It probably makes more sense to do:
#csv = FasterCSV.open('filename.csv', 'w')
and then write to it as you go along:
#csv << [sku, name, desc]
that way if your script crashes halfway through you've at least got half of the data.
The simple_client.rb file below works perfectly fine against my emulation cas server; however, the casport.rb file (main file of oa-casport OmniAuth strategy) is not setting or passing the headers / format properly. It needs to be dynamically assigned to the class to allow initializer options to be able to create them, but I'm not sure how else to do it besides how I've attempted to do it here. I was fairly certain I had this working at some point, but I can't see any other explanation for why this wouldn't be working given the simplicity of the client file.
Any help is greatly appreciated on figuring out how to best set the format and headers settings for HTTParty within my Casport class dynamically. As it is it just keeps returning the HTML view for that particular user.
simple_client.rb:
### simple_client.rb - works properly w/ parsed XML response
### The cas.dev project is coming from this Github repo:
### https://github.com/stevenhaddox/oa-casport-server
require 'rubygems'
require 'httparty'
require 'awesome_print'
class Casport
include HTTParty
base_uri 'cas.dev/users'
format :xml
headers 'Accept' => 'application/xml'
def self.find_user(id)
get("/#{id}").parsed_response
end
end
user = Casport.find_user(1)
ap user
casport.rb:
# lib/omniauth/strategies/casport.rb
require 'omniauth/core'
require 'httparty'
require 'redis'
require 'uri'
module OmniAuth
module Strategies
#
# Authentication to CASPORT
#
# #example Basic Usage
#
# use OmniAuth::Strategies::Casport, {
# :setup => true
# }
# #example Full Options Usage
#
# use OmniAuth::Strategies::Casport, {
# :setup => true,
# :cas_server => 'http://cas.slkdemos.com/users/',
# :format => 'xml',
# :format_header => 'application/xml',
# :ssl_ca_file => 'path/to/ca_file.crt',
# :pem_cert => '/path/to/cert.pem',
# :pem_cert_pass => 'keep it secret, keep it safe.'
# }
class Casport
include OmniAuth::Strategy
include HTTParty
def initialize(app, options)
super(app, :casport)
#options = options
#options[:cas_server] ||= 'http://cas.dev/users'
#options[:format] ||= 'xml'
#options[:format_header] ||= 'application/xml'
end
def request_phase
Casport.setup_httparty(#options)
redirect(callback_path)
end
def callback_phase
begin
raise 'We seemed to have misplaced your credentials... O_o' if user.nil?
super
rescue => e
redirect(request_path)
# fail!(:invalid_credentials, e)
end
call_app!
end
def auth_hash
# store user in a local var to avoid new method calls for each attribute
# convert all Java camelCase keys to Ruby snake_case, it just feels right!
user_obj = user.inject({}){|memo, (k,v)| memo[k.gsub(/[A-Z]/){|c| '_'+c.downcase}] = v; memo}
begin
user_obj = user_obj['userinfo']
rescue => e
fail!(:invalid_user, e)
end
OmniAuth::Utils.deep_merge(super, {
'uid' => user_obj['uid'],
'user_info' => {
'name' => user_obj['full_name'],
'email' => user_obj['email']
},
'extra' => {'user_hash' => user_obj}
})
end
# Set HTTParty params that we need to set after initialize is called
# These params come from #options within initialize and include the following:
# :ssl_ca_file - SSL CA File for SSL connections
# :format - 'json', 'xml', 'html', etc. || Defaults to 'xml'
# :format_header - :format Header string || Defaults to 'application/xml'
# :pem_cert - /path/to/a/pem_formatted_certificate.pem for SSL connections
# :pem_cert_pass - plaintext password, not recommended!
def self.setup_httparty(opts)
format opts[:format].to_sym
headers 'Accept' => opts[:format_header]
if opts[:ssl_ca_file]
ssl_ca_file opts[:ssl_ca_file]
if opts[:pem_cert_pass]
pem File.read(opts[:pem_cert]), opts[:pem_cert_pass]
else
pem File.read(opts[:pem_cert])
end
end
end
def user
# Can't get user data without a UID from the application
begin
raise "No UID set in request.env['omniauth.strategy'].options[:uid]" if #options[:uid].nil?
#options[:uid] = #options[:uid].to_s
rescue => e
fail!(:uid_not_found, e)
end
url = URI.escape(#options[:cas_server] + '/' + #options[:uid])
# It appears the headers aren't going through properly to HTTParty...
# The URL + .xml works in the application & the url w/out .xml works in standalone file
# Which means somehow the setup with self.setup_httparty isn't kicking in properly :(
ap Casport.get(url+'.xml').parsed_response
begin
cache = #options[:redis_options].nil? ? Redis.new : Redis.new(#options[:redis_options])
unless #user = (cache.get #options[:uid])
# User is not in the cache
# Retrieving the user data from CASPORT
# {'userinfo' => {{'uid' => UID}, {'fullName' => NAME},...}},
#user = Casport.get(url).parsed_response
cache.set #options[:uid], #user
# CASPORT expiration time for user (24 hours => 1440 seconds)
cache.expire #options[:uid], 1440
end
# If we can't connect to Redis...
rescue Errno::ECONNREFUSED => e
#user ||= Casport.get(url).parsed_response
end
#user = nil if user_empty?
#user
end
# Investigate user_obj to see if it's empty (or anti-pattern data)
def user_empty?
is_empty = false
is_empty = true if #user.nil?
is_empty = true if #user.empty?
# If it isn't empty yet, let's convert it into a Hash object for easy parsing via eval
unless #user.class == Hash
is_empty = true
raise "String returned when a Hash was expected."
end
is_empty == true ? true : nil
end
end
end
end
This was apparently working properly, what I failed to do was to provide the header for Content-Type:
...
def self.setup_httparty(opts)
format opts[:format].to_sym
headers 'Accept' => opts[:format_header]
headers 'Content-Type' => opts[:format_header]
...
Once I added that additional line everything kicked in properly.