Limitation on number of result in search engine scraping - ruby

I am scraping bing search engine using Mechanize. But I get only max 200 results programmatically if I execute same search query on bing.com it returns 1400 results. What is gotcha here?
def generate_profiles_from_group(options={})
raise "TypeError", "Invalid Arguments" unless options.is_a? Hash
group = options[:group] if options.has_key? :group
query = build_query(options)
page = bing_search(query)
contacts_stack = extract_contacts_from_bing_page page: page
bing_links_stack = bing_links page
return contacts_stack, bing_links_stack
end
def extract_contacts_from_bing_page(options)
page = options[:page]
company = options[:company] || nil
title = options[:title] || nil
stack = []
while true
page.parser.search('h3 a').each do |cite|
text = cite.text
unless text == ""
name_array = text.split(' ')
if name_array.size >= 2
name = name_array[0]+' '+name_array[1]
unless name=~/[^a-zA-Z',\s]/i
stack << {name: name, company: company, title: title}
end
end
end
end
keyw = page.parser.xpath('//*[contains(concat( " ", #class, " " ), concat( " ", "sb_pagN", " " ))]').text
break if keyw == ""
page = #agent.click page.link_with(text: keyw )
end
stack
end
def bing_links page
stack = []
while true
page.parser.xpath('//cite').each do |cite|
stack << cite.text unless cite.text == ""
end
keyw = page.parser.xpath('//*[contains(concat( " ", #class, " " ), concat( " ", "sb_pagN", " " ))]').text
break if keyw == ""
sleep(10+rand(40))
page = #agent.click page.link_with(text: keyw )
end
stack
end
def build_query(options)
name = options[:name] if options.has_key? :name
title = options[:title] if options.has_key? :title
company = options[:company] if options.has_key? :company
group = options[:group] if options.has_key? :group
if name && company
return "site:linkedin.com \"#{name}\" \"at #{company}\""
elsif name && title
return "site:linkedin.com \"#{name}\" \"#{title}\""
elsif title && company
return "site:linkedin.com/ \"#{title}\" \"at #{company}\""
elsif group
return "site:linkedin.com \"groups and association\" + \"#{group}\""
end
end

Related

What condition do I check for to raise a custom error with Net::OpenTimeout execution expired error

Open-uri and nokogiri are slow to scrape the site I want hence the Net::OpenTimeout execution expired error. I attempted to code a custom error with rescue however I do not know what condition I can look for to raise that custom error.
I attempted to few if else statements however I really just guessed how to check if I was gonna get that error. I hard coded a condition that failed and thus rescued the error. I am very new to ruby and custom errors. In fact this is my first.
class Scrape
Base = 'http://www.wine.com'
##menu = []
##pages = []
def self.index
index_url = Base + "/list/wine/7155?sortBy=savings&pricemax=90"
#below is where I need to check for the condition to
raise the error
if doc = Nokogiri::HTML(open(index_url))
container = doc.css('.prodList')
wines = container.css('.prodItem')
wines.each do |wine|
##menu << {
:link => wine.css('.prodItemInfo_link').attribute('href').value,
:name => wine.css('.prodItemInfo_name').text,
:rating => (wine.css('.averageRating_average').text.to_i) > 0 ?
(wine.css('.averageRating_average').text) : 'no rating',
:price => wine.css('.productPrice_price-saleWhole').text.strip
}
end
##menu.each do |item|
Bottle.new.create(item)
end
else
begin
raise Custom_error
rescue Custom_error => error
puts error.message
end
end
end
def self.scrape_page(wine_obj)
wine_link = wine_obj.link
individual_page = Base + wine_link
docu = Nokogiri::HTML(open(individual_page))
y = docu.css('.viewMoreModule_text')
more = docu.css('.viewMoreModule_text')
##pages << {
:obj => wine_obj,
:name => docu.css('.pipName').text,
:alcohol_percent => y
x = docu.css('.mobileProdAttrs').css('.prodAlcoholPercent')
y = x.css('.prodAlcoholPercent_percent').text,
:price => docu.css('span.productPrice_price-saleWhole').text,
:origin => docu.css('span.prodItemInfo_originText a').text,
:winemaker_notes => docu.css('.viewMoreModule_text').first.text,
:more => y[2].text,
:rating => docu.css('span.averageRating_average').first.text
}
Page.create_find_by_name( ##pages.last )
end
def self.pages
##pages
end
end
class Cli
def run
puts 'loading from cyberspace'
Scrape.index
Bottle.make_list
controller
end
def controller
input = ''
response = ''
puts ' '
view
while input != 11
response = gets.chomp.to_i
input = "#{response}11".to_i
if input == 111
menu
elsif input == 11
exit
elsif input > 0 && input < 26
find_by_input(input)
elsif input != 0 && input != 111
error_1
end
end
end
def view
puts "welcome to the wine bar"
puts "================="
puts " W I N E "
puts " B A R "
puts "================="
puts " "
puts "type 1 for list of wine"
puts " "
puts "type 0 to exit "
end
def menu
wines = Bottle.list
second_input = ''
while second_input != 0
puts "<<<<<<<<<<<<<<<<<<<<<<<<<<>>>>>>>>>>>>>>>>>>>>>>>>>>"
puts "type the corresponding number to view more wine info"
puts
"__________________________________________________________________"
wines.each do |wine|
puts "#{wine.index}) #{wine.name} #{wine.price}"
end
second_input = gets.chomp.to_i
if second_input > 0 && second_input < 26
find_by_input(second_input)
elsif second_input == 0
exit
second_input = 0
elsif second_input > 25 && second_input != 101
error_1
end
end
puts <<-DOC
the the wine number again
for winemaker notes
DOC
end
def find_by_input(input)
while input != 0
selection = Bottle.find_by_input(input)
puts "NAME: #{selection.name}"
puts "PRICE: $#{selection.price}"
puts "RATING: #{selection.rating}"
puts "________________________________________"
puts " type #{input} again "
puts " for more info "
puts " provided by the winemaker "
# reseting input and extending user control functionality
third_input = ''
third_input = gets.chomp.to_i
if third_input == selection.index
response = Scrape.scrape_page(selection)
view_2(response, third_input)
elsif input == 0
exit
end
end
end
def view_2(response, old_input)
next_input = ''
while next_input != 0
puts "Alcohol Percent: #{response.alcohol_percent}"
puts "Winemaker Notes: #{response.winemaker_notes}"
puts " "
puts "Type #{old_input} again for more!!"
next_input = gets.chomp.to_i
if next_input == old_input
input = 0
next_input = 0
# refacort as it puts out 88 again and should not. Also 0 is not
exiting with correct behavior
# refactor so looking for "#{input}"1 to prevent the recall of
input
more(response)
end
end
end
def more(response)
puts response.more
puts menu
end
def error_1
puts " WHOA coder "
puts "type a better number!"
end
def exit
puts <<-DOC
well that was fun
Thank you for checking out
my first cli program
DOC
end
end ```
```class Page
attr_accessor :alcohol_percent, :price, :name, :origin, :winemaker_notes,
:rating, :more, :obj
##web_pages = []
def self.create_find_by_name(hash)
if answer = ##web_pages.find{ |obj| obj.name == hash[:name]}
answer
else
self.new.create(hash)
end
end
def create(hash)
hash.each do |key, value|
self.send(("#{key}="), value)
end
save
view_more
end
def view_more
##web_pages.last
end
def save
##web_pages << self
end
end
attr_accessor :link, :name, :price, :rating, :index
##bottles = []
def create(hash)
hash.each do |key, words|
self.send(("#{key}="), words )
end
save
end
def save
##bottles << self
end
def self.make_list
##numbered_list = ##bottles.sort{ |x,y| x.price <=>
y.price}.map.with_index(1) do
|w,i| w.index = i
w
end
end
def self.list
##numbered_list
end
def self.find_by_input(input)
a = ##numbered_list.find{ |wine| wine.index == input}
# puts "#{a.name} $#{a.price} rating: #{a.rating}"
# puts "type #{input} again for winemaker notes"
# more = ''
# while more != 0
# more = gets.chomp.to_i
# (input == more) ? (Scrape.scrape_page(a.link)) : (self.list)
# end
end
end
class Scrape
Base = 'http://www.wine.com'
##menu = []
##pages = []
def self.index
index_url = Base + "/list/wine/7155?sortBy=savings&pricemax=90"
if doc = Nokogiri::HTML(open(index_url))
container = doc.css('.prodList')
wines = container.css('.prodItem')
wines.each do |wine|
##menu << {
:link => wine.css('.prodItemInfo_link').attribute('href').value,
:name => wine.css('.prodItemInfo_name').text,
:rating => (wine.css('.averageRating_average').text.to_i) > 0 ?
(wine.css('.averageRating_average').text) : 'no rating',
:price => wine.css('.productPrice_price-saleWhole').text.strip
}
end
##menu.each do |item|
Bottle.new.create(item)
end
else
begin
raise Custom_error
rescue Custom_error => error
puts error.message
end
end
end
def self.scrape_page(wine_obj)
wine_link = wine_obj.link
individual_page = Base + wine_link
docu = Nokogiri::HTML(open(individual_page))
y = docu.css('.viewMoreModule_text')
more = docu.css('.viewMoreModule_text')
##pages << {
:obj => wine_obj,
:name => docu.css('.pipName').text,
alcholo = docu.css('.mobileProdAttrs').css('.prodAlcoholPercent'),
:alcohol_percent => alcholo.css('.prodAlcoholPercent_percent').text,
:price => docu.css('span.productPrice_price-saleWhole').text,
:origin => docu.css('span.prodItemInfo_originText a').text,
:winemaker_notes => docu.css('.viewMoreModule_text').first.text,
:more => y[2].text,
:rating => docu.css('span.averageRating_average').first.text
}
Page.create_find_by_name( ##pages.last )
end
def self.pages
##pages
end
end
When the internet connection is down/too slow the custom error is raised.
When an exception is thrown, the program stops its normal flow. You need to surround the part of the code that can throw an exception with a begin..rescue clause, and attempt to handle it, re-raise it, or raise another exception instead.
In your example, that would be:
begin
Nokogiri::HTML(open(url))
rescue Net::OpenTimeoutError => e
# log the error message if needed, raise your CustomError instead
raise CustomError, e.message
end
You can omit the begin, and put a rescue clause at the end of the method, ruby will interpret this as if the entire method body was wrapped in a begin..rescue block, something like this:
def open_page(url)
return Nokogiri::HTML(open(url))
rescue Net::OpenTimeoutError => e
raise CustomError, e.message
end

Category Hierarchy in Jekyll

I've been trying to use Jekyll categories in a hierarchical fashion, i.e.
A: ['class', 'topic', 'foo']
AA: ['class', 'topic', 'foo', 'bar']
AB: ['class', 'topic', 'foo', 'baz']
AAA: ['class', 'topic', 'foo', 'bar', 'qux']
I'm trying to create a listing of all immediate subdirectories programmatically. That is, on a page with categories (A), I wish to be able to list the posts with categories (AA) and (AB), but not (AAA). Is this possible with Jekyll's vanilla structure, or should I consider using a plugin?
You need to use a plugin.
I've managed to make most of what you describe happen, but I'm not using categories at all, just representing a directory tree.
require 'digest/md5'
require 'open-uri'
module Jekyll
# Add accessor for directory
class Page
attr_reader :dir
end
class NavTree < Liquid::Tag
def render(context)
site = context.registers[:site]
#page_url = context.environments.first["page"]["url"]
#folder_weights = site.data['folder_weights']
#folder_icons = site.data['folder_icons']["icons"]
#nodes = {}
tree = {}
sorted_tree = {}
site.pages.each do |page|
# exclude all pages that are hidden in front-matter
if page.data["navigation"]["show"] != false
path = page.url
path = path.index('/')==0 ? path[1..-1] : path
#nodes[path] = page.data
end
end
#let's sort the pages by weight
array = []
#nodes.each do |path, data|
array.push(:path => path, :weight => data["weight"], :title => data["title"])
end
sorted_nodes = array.sort_by {|h| [-(h[:weight]||0), h[:path] ]}
sorted_nodes.each do |node|
current = tree
node[:path].split("/").inject("") do |sub_path,dir|
sub_path = File.join(sub_path, dir)
current[sub_path] ||= {}
current = current[sub_path]
sub_path
end
end
tree.each do |base, subtree|
folder_weight = #folder_weights[base]? #folder_weights[base] : 0
tree[base] = {"weight" => folder_weight, "subtree" => subtree}
end
tree_array = []
tree.each do |key, value|
tree_array.push(:base => key, :weight => value["weight"], :subtree => value["subtree"])
end
sorted_tree = tree_array.sort_by {|node| [ -(node[:weight]), node[:base] ]}
puts "generating nav tree for #{#page_url}"
files_first_traverse "", sorted_tree, 0
end
def files_first_traverse(prefix, nodes = [], depth=0)
output = ""
if depth == 0
id = 'id="nav-menu"'
end
output += "#{prefix}<ul #{id} class=\"nav nav-list\">"
nodes.each do |node|
base = node[:base]
subtree = node[:subtree]
name = base[1..-1]
if name.index('.') != nil
icon_name = #nodes[name]["icon"]
name = #nodes[name]["title"]
end
li_class = ""
if base == #page_url
li_class = "active"
if icon_name
icon_name = icon_name + " icon-white"
end
end
icon_html = "<span class=\"#{icon_name}\"></span>" unless icon_name.nil?
output += "#{prefix}<li class=#{li_class}>#{icon_html}#{name}</li>" if subtree.empty?
end
nodes.each do |node|
base = node[:base]
subtree = node[:subtree]
next if subtree.empty?
href = base
name = base[1..-1]
if name.index('.') != nil
is_parent = false
name = #nodes[name]["title"]
else
is_parent = true
href = base + '/index.html'
if name.index('/')
name = name[name.rindex('/')+1..-1]
end
end
name.gsub!(/_/,' ')
li_class = ""
if #page_url.index(base)
list_class = "collapsibleListOpen"
else
list_class = "collapsibleListClosed"
end
if href == #page_url
li_class = "active"
end
if is_parent
id = Digest::MD5.hexdigest(base)
icon_name = #folder_icons[base]
icon_html = icon_name.nil? ? "" : "<span class=\"#{icon_name}\"></span>"
li = "<li id=\"node-#{id}\" class=\"parent #{list_class}\"><div class=\"subtree-name\">#{icon_html}#{name}</div>"
else
icon_name = #nodes[name]["icon"]
if icon_name && li_class=="active"
icon_name = icon_name + " icon-white"
end
icon_html = icon_name.nil? ? "<i class=\"#{icon_name}\"></i>" : ""
li = "<li class=\"#{li_class}\">#{icon_html}#{name}</li>"
end
output += "#{prefix} #{li}"
subtree_array = []
subtree.each do |base, subtree|
subtree_array.push(:base => base, :subtree => subtree)
end
depth = depth + 1
output += files_first_traverse(prefix + ' ', subtree_array, depth)
if is_parent
output+= "</li>"
end
end
output += "#{prefix} </ul>"
output
end
end
end
Liquid::Template.register_tag("navigation", Jekyll::NavTree)
It is ugly code, but it's a start. You can see it in action on https://sendgrid.com/docs

Ruby script error

I've got a ruby script
#!/usr/bin/ruby
require 'rubygems'
require 'mechanize'
require 'nokogiri'
require 'highline/import'
require 'stringio'
#Change based on Semester
$term = '09'
$year = '2012'
$frequency = 4 #Number of Seconds between check requests
$agent = Mechanize.new
$agent.redirect_ok = true
$agent.user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.11 Safari/535.19"
$agent.verify_mode = OpenSSL::SSL::VERIFY_NONE
#Uber simway to colorize outputin
class String
def color(c)
colors = {
:black => 30,
:red => 31,
:green => 32,
:yellow => 33,
:blue => 34,
:magenta => 35,
:cyan => 36,
:white => 37
}
return "\e[#{colors[c] || c}m#{self}\e[0m"
end
end
#Logins, Gets the Courses, Returns Courses Obj with Name/URL/Tools for each
def login(username, password)
#Login to the system!
page = $agent.get("https://auth.vt.edu/login?service=https://webapps.banner.vt.edu/banner-cas-prod/authorized/banner/SelfService")
login = page.forms.first
login.set_fields({
:username => username,
:password => password
})
if (login.submit().body.match(/Invalid username or password/)) then
return false
else
return true
end
end
#Gets Course Information
def getCourse(crn)
begin
courseDetails = Nokogiri::HTML( $agent.get(
"https://banweb.banner.vt.edu/ssb/prod/HZSKVTSC.P_ProcComments?CRN=#{crn}&TERM=#{$term}&YEAR=#{$year}"
).body)
rescue
return false #Failed to get course
end
#Flatten table to make it easier to work with
course = {}
dataSet = false
course[:title] = courseDetails.css('td.title').last.text.gsub(/-\ +/, '')
course[:crn] = crn
courseDetails.css('table table tr').each_with_index do |row|
#If we have a dataSet
case dataSet
when :rowA
[ :i, :days, :end, :begin, :end, :exam].each_with_index do |el, i|
if row.css('td')[i] then
course[el] = row.css('td')[i].text
end
end
when :rowB
[ :instructor, :type, :status, :seats, :capacity ].each_with_index do |el, i|
course[el] = row.css('td')[i].text
end
end
dataSet = false
#Is there a dataset?
row.css('td').each do |cell|
case cell.text
when "Days"
dataSet = :rowA
when "Instructor"
dataSet = :rowB
end
end
end
return course
end
#Registers you for the given CRN, returns true if successful, false if not
def registerCrn(crn)
#Follow Path
$agent.get("https://banweb.banner.vt.edu/ssb/prod/twbkwbis.P_GenMenu?name=bmenu.P_MainMnu")
reg = $agent.get("https://banweb.banner.vt.edu/ssb/prod/hzskstat.P_DispRegStatPage")
dropAdd = reg.link_with(:href => "/ssb/prod/bwskfreg.P_AddDropCrse?term_in=#{$year}#{$term}").click
#Fill in CRN Box and Submit
crnEntry = dropAdd.form_with(:action => '/ssb/prod/bwckcoms.P_Regs')
crnEntry.fields_with(:id => 'crn_id1').first.value = crn
crnEntry['CRN_IN'] = crn
add = crnEntry.submit(crnEntry.button_with(:value => 'Submit Changes')).body
if add =~ /#{crn}/ && !(add =~ /Registration Errors/) then
return true
else
return false
end
end
#Main loop that checks the availaibility of each courses and fires to registerCrn on availaibility
def checkCourses(courses)
requestCount = 0
startTime = Time.new
loop do
system("clear")
requestCount += 1
nowTime = Time.new
puts "Checking Availaibility of CRNs".color(:yellow)
puts "--------------------------------\n"
puts "Started:\t#{startTime.asctime}".color(:magenta)
puts "Now: \t#{nowTime.asctime}".color(:cyan)
puts "Request:\t#{requestCount} (Once every #{$frequency} seconds)".color(:green)
puts "--------------------------------\n\n"
courses.each_with_index do |c, i|
puts "#{c[:crn]} - #{c[:title]}".color(:blue)
course = getCourse(c[:crn])
next unless course #If throws error
puts "Availaibility: #{course[:seats]} / #{course[:capacity]}".color(:red)
if (course[:seats] =~ /Full/) then
else
if (registerCrn(c[:crn])) then
puts "CRN #{c[:crn]} Registration Sucessfull"
courses.slice!(i)
else
puts "Couldn't Register"
end
end
print "\n"
end
sleep $frequency
end
end
#Add courses to be checked
def addCourses
crns = []
loop do
system("clear")
puts "Your CRNs:".color(:red)
crns.each do |crn|
puts " -> #{crn[:title]} (CRN: #{crn[:crn]})".color(:magenta)
end
#Prompt for CRN
alt = (crns.length > 0) ? " (or just type 'start') " : " "
input = ask("\nEnter a CRN to add it#{alt}".color(:green) + ":: ") { |q| q.echo = true }
#Validate CRN to be 5 Digits
if (input =~ /^\d{5}$/) then
#Display CRN Info
c = getCourse(input.to_s)
puts "\nCourse: #{c[:title]} - #{c[:crn]}".color(:red)
puts "--> Time: #{c[:begin]}-#{c[:end]} on #{c[:days]}".color(:cyan)
puts "--> Teacher: #{c[:instructor]}".color(:cyan)
puts "--> Type: #{c[:type]} || Status: #{c[:status]}".color(:cyan)
puts "--> Availability: #{c[:seats]} / #{c[:capacity]}\n".color(:cyan)
#Add Class Prompt
add = ask("Add This Class? (yes/no)".color(:yellow) + ":: ") { |q| q.echo = true }
crns.push(c) if (add =~ /yes/)
elsif (input == "start") then
checkCourses(crns)
end
end
end
def main
system("clear")
puts "Welcome to CourseAdd by mil".color(:blue)
username = ask("PID ".color(:green) + ":: ") { |q| q.echo = true }
password = ask("Password ".color(:green) + ":: " ) { |q| q.echo = "*" }
system("clear")
if login(username, password) then
addCourses
else
puts "Invalid PID/Password"
exit
end
end
main
but when I run ruby Untitled.rb it give me this error.
/System/Library/Frameworks/Ruby.framework/Versions/1.8/usr/lib/ruby/1.8/rubygems/custom_require.rb:31:in `gem_original_require': no such file to load -- mechanize (LoadError)
from /System/Library/Frameworks/Ruby.framework/Versions/1.8/usr/lib/ruby/1.8/rubygems/custom_require.rb:31:in `require'
from /Users/user/Desktop/Untitled.rb:3
What does this mean and how can I fix it? I'm not sure if I need to be doing this through an IDE or if terminal works. I'm brand new to ruby so I honestly have not a clue what the issue could be.
You need to install mechanize. In your terminal, type:
gem install mechanize
Retry your script when it finishes installing. If you have other gems that are missing, you can use the same command to install them.
gem install <gem name>

Adding users from a text file into active directoy

I've been working on this code for ages and I can't get it working. It takes user information form a text file, creates user groups and puts the users in the groups. It makes the security group just fine, but it does not put the users in the groups. There is no error message but after going through the error checking is goes through the code that should add the user and puts the error at the bottom. Can anyone help please?
The usernames in the text file are setup like so:
fred,bush,1990-20-3,123456781,2008-20-3,D5,
xin,zhao,1990-20-2,123456782,2008-20-3,D5,
bobby,bob,1990-20-1,123456783,2008-20-3,D5,
john,lose,1990-20-4,123456784,2008-20-3,D5,
elly,moose,1990-20-5,123456785,2008-20-3,D5,
jackie,chan,1990-20-6,123456786,2008-20-3,D5,
katarina,lotus,1990-20-7,123456787,2008-20-3,D5,
kelly,nunu,1990-20-8,123456788,2008-20-3,D5,
lois,harris,1990-20-9,123456789,2008-20-3,D5,
gutwold,manly,1990-20-10,123456780,2008-20-3,D5,
griswold,womenly,1990-20-11,123456710,2008-20-3,D5,
bessy,horse,1990-20-12,123456711,2008-20-3,D5,
And the code is:
require 'date'
domain = "dc=TROPTRAIN,dc=net,dc=au"
ou = "ou=studentsOU"
filepath = 'C:\Documents and Settings\Administrator\My Documents\Luke Abbey Ruby Final\userfile\error_log.txt'
users = 'C:\Documents and Settings\Administrator\My Documents\Luke Abbey Ruby Final\userfile\newusers.txt'
#intro function for the program
def intro
system "cls"
puts ""
puts " Tropical Train - Adding users"
puts "====================================================================="
puts ""
end
#get user file
check = 0
while check == 0
intro
puts "Please enter the path to the user file,"
puts "or enter the the letter 's' to use the stored file location:"
puts ""
puts "#{users}"
userfile = gets.chomp.to_s
if ((userfile == 's') || (userfile == 'S'))
check = 1
userfile = users
#puts userfile
#system "pause"
elsif(test(?e,userfile))
check = 1
elsif userfile =~ (/[\x00\/\\:\*\?\"<>\|]/)
puts "The userfile contains invalid characters. Try again."
system "pause"
check = 0
elsif userfile == ""
puts "You entered nothing. Try again."
system "pause"
check = 0
elsif(!test(?e,userfile))
check = 0
puts "The file #{userfile} does not exist. Try again."
system "pause"
else
puts "Your entry is invalid. Try again."
system "pause"
check = 0
end
end
#create the security groups via DOS and check wheteher they already exist
grpC2 = 0
grpC3 = 0
grpC4 = 0
grpD5 = 0
expired = 0
students = 0
%x[dsquery group "dc=troptrain,dc=net,dc=au"].each do |line|
data = line
if line.include?("grpC2")
grpC2 = 1
end
if line.include?("grpC3")
grpC3 = 1
end
if line.include?("grpC4")
grpC4 = 1
end
if line.include?("grpD5")
grpD5 = 1
end
if line.include?("Expired")
expired = 1
end
if line.include?("Students")
students = 1
end
end
if grpC2 != 1
%x[dsadd group "cn=grpC2,ou=studentsOU,dc=troptrain,dc=net,dc=au"]
end
if grpC3 != 1
%x[dsadd group "cn=grpC3,ou=studentsOU,dc=troptrain,dc=net,dc=au"]
end
if grpC4 != 1
%x[dsadd group "cn=grpC4,ou=studentsOU,dc=troptrain,dc=net,dc=au"]
end
if grpD5 != 1
%x[dsadd group "cn=grpD5,ou=studentsOU,dc=troptrain,dc=net,dc=au"]
end
if expired != 1
%x[dsadd group "cn=Expired,ou=studentsOU,dc=troptrain,dc=net,dc=au"]
end
if students != 1
%x[dsadd group "cn=Students,ou=studentsOU,dc=troptrain,dc=net,dc=au"]
end
#open the file
f1 = File.open(userfile,"r")
#read the file line by line
count = 0
f1.each do |line|
data = line.split(',')
firstname = data[0].capitalize
surname = data[1].capitalize
dob = data[2].to_s
snumber = data[3].to_s
enddate1 = data [4].gsub!("-","/")
enddate = enddate1
area = data[5]
fullname = (firstname + (" ") + surname)
group = "cn=#{'grp'+data[5]}"
password1 = data[2].gsub("-", "")
password = password1
count = count + 1
#display values for establishing error checks
#puts ""
#puts "fullname: #{fullname}"
#puts "dob: #{dob}"
#puts "snumber: #{snumber}"
#puts "enddate: #{enddate}"
#puts "area: #{area}"
#puts "password: #{password}"
#puts "group: #{group}"
#puts "count: #{count}"
#puts ""
if ((data[2] =~ /[^0-9-]/) || (data[3] =~ /[^0-9-]/) || (data[4] =~ /[^0-9\/]/) || (data[5] =~ /[^A-Za-z0-9 ]/) || (data[0] =~ /[^A-Za-z ]/) || (data[1] =~ /[^A-Za-z ]/) || (data[0] == "") || (data[1] == "") || (data[2] == "") || (data[3] == "") || (data[4] == "") || (data[5] == "") || (data[6] == ""))
puts ""
puts "====================================================================="
puts "User account No #{count}: #{fullname} was not created."
puts ""
if (File.exist?(filepath))
file = File.open((filepath), "a+")
t = Time.now
file.puts "====================================================================="
file.puts "User account No #{count}: #{fullname} was not created."
file.puts t
file.puts ""
else
puts "Error log cannot be found!"
puts "Searched directory: #{filepath}"
end
#checking database entries for errors
if ((data[0] == "") || (data[0] =~ /[^A-Za-z ]/))
puts "No/Incorrect firstname on file."
file.puts "No/Incorrect firstname on file."
end
if ((data[1] == "") || (data[1] =~ /[^A-Za-z ]/))
puts "No/Incorrect surname on file."
file.puts "No/Incorrect surname on file."
end
if ((data[2] == "") || (data[2] =~ /[^0-9-]/))
puts "No/Incorrect birthdate on file."
file.puts "No/Incorrect birthdate on file."
end
if ((data[3] == "") || (data[3] =~ /[^0-9-]/))
puts "No/Incorrect student number on file."
file.puts "No/Incorrect student number on file."
end
if ((data[4] == "") || (data[4] =~ /[^0-9\/]/))
puts "No/Incorrect study end date on file."
file.puts "No/Incorrect study end date on file."
end
if ((data[5] == "") || (data[5] =~ /[^A-Za-z0-9 ]/))
puts "No/Incorrect study area on file."
file.puts "No/Incorrect study area on file."
end
file.close
else
begin
puts "working"
system 'pause'
ad = WIN32OLE.connect("LDAP://#{ou},#{domain}")
user = ad.create("user","cn=#{fullname}")
user.givenName = firstname
user.Sn = surname
user.SAMAccountname = "#{snumber}"
user.userPrincipalName = "#{snumber}#Troptrain.net.au"
user.displayName = fullname
user.profilePath = "c:\\profiles\\students\\logon"
user.setInfo
user_fqdn = "cn=#{fullname},#{ou},#{domain}"
user = WIN32OLE.connect("LDAP://#{user_fqdn}")
user.userPassword = Password01
user.accountDisabled = 0
user.accountExpirationDate = enddate
user.setInfo
studentgroup_fqdn = "#{group},#{ou},#{domain}"
grp = WIN32OLE.connect("LDAP://#{studentgroup_fqdn}")
grp.add("LDAP://#{user_fqdn}")
allgroup_fqdn = "cn=students,#{ou},#{domain}"
allgrp = WIN32OLE.connect("LDAP://#{allgroup_fqdn}")
allgrp.add("LDAP://#{user_fqdn}")
puts ""
puts "====================================================================="
puts "User account No #{count}: #{fullname} was created."
sleep 0.5
rescue
end
end
end
puts "====================================================================="
puts ""
puts "See Error log for details."
puts ""
puts "The program has finished creating users."
f1.close
Separate your concerns.
You need to parse a text file and retrieve users and groups from it in a structured way.
You need to import your groups and then import your users.
You then assign the users to the groups you created.
Each User and Group must be located in their own OUs to identify them.
Those OUs must be known.
Now work backward and factor out everything you need to continue with.
Find your host name, port and the credentials to bind to the LDAP server.
Find your Base DN. (DC=troptrain, DC=net, DC=au)
Find your User and Group prefixes. (the OU segments after the Base DN).
Import your data.
Verify. (This is also important!)
As LDAP can appear to be somewhat backward from the typical relational database, I recommend using the ActiveLDAP ruby gem with gem install active_ldap. It makes working with LDAP far easier.
#!/usr/bin/env ruby
require 'active_ldap'
class User < ActiveLdap::Base
ldap_mapping :dn_attribute => 'CN', :prefix => '<user-ou-prefix>',
has_many :groups, :class => 'Group', :wrap => 'memberOf', :primary_key => 'DN'
end
class Group < ActiveLdap::Base
ldap_mapping :dn_attribute => 'CN', :prefix => '<group-ou-prefix>',
has_many :members, :class => 'User', :wrap => 'member', :primary_key => 'DN'
end
ActiveLdap::Base.setup_connection(
:host => '<hostname>',
:base => '<base-dn>',
:bind_dn => '<bind-dn>',
:password => '<bind-password>',
:allow_anonymous => false,
:try_sasl => false,
:port => 389
)
From there, you should have the ability to create, read, update, and delete users using an ActiveRecord-like interface. the call to #setup_connection establishes what options are used to connect, and the has_many calls establish foreign-key-like relations between your classes.
# Find and display all users:
User.find(:all, '*') do |user|
puts user.cn
end
# Create a group with a CN of 'Anonymous'
group = Group.new('Anonymous')
group.description = "An anonymous group."
group.save
# Change the user with the CN 'Nobody' and change their displayName attribute.
user = User.find('Nobody')
user.displayName = "Nobody's Name"
user.save
# Delete a group with a CN of 'Foobar'.
group = Group.find('Foobar')
group.delete
ActiveLdap makes it extremely easy to manipulate LDAP from Ruby, I highly recommend it.

Ruby output formats differently on dynamic v. static input

FYI: Fist time I've written Ruby.
We have to write an Address Book program. I have many parts working at this point but have run into something I don't get.
I add contacts to the address book statically before the program runs. Then I try to have a user add a contact dynamically. When I print this address book the contacts added statically are printed according to formatting I expect, but the ones added dynamically are all over the place.
I've added all my code here. It's sort of a big chunk but not too bad...
# Address class
class Address
attr_accessor :street, :city, :state, :zip
# default constructor
def initialize(*args)
if (args.size == 0 )
#street = #city = #state = #zip = ""
elsif (args.size == 4)
#street = args[0]
#city = args[1]
#state = args[2]
#zip = args[3]
else
puts('Constructor takes 0 or 4 arguments. No address information has been set.')
end
end
# string representation of address
def to_s
" " + #street + "\n" + \
" " + #city + " " + #state + ", " + #zip
end
end
# Person class which holds full name, phone, and
# address as an object
class Person
attr_accessor :fname, :lname, :phone, :address
def initialize(*args)
if (args.size == 0 )
#fname = #lname = #phone = ""
#address = Address.new
elsif (args.size == 4)
#fname = args[0]
#lname = args[1]
#phone = args[2]
#address = args[3]
else
puts('Incorrect number of arguments.')
end
end
# returns full name
def full_name
#lname + ", " + #fname
end
def to_s
" " + full_name + "\n" + \
#address.to_s + "\n" + \
" " + #phone
end
end
# AddressBook class to hold addresses
class AddressBook
def initialize
# empty array
#persons = []
end
# adds a person to the address book
def add(person)
#persons += [person]
#persons = #persons.sort{|a,b| by_name(a,b)}
end
def by_name(a,b)
if a.lname == b.lname
a.fname <=> b.fname
else
a.lname <=> b.lname
end
end
# removes a person from the address book
def remove(person)
# Use Array#delete
#persons.delete(person)
end
def to_s
add_book = ""
for p in #persons do
add_book += p.to_s + "\n\n"
end
return add_book
end
end
#
# Here I add three contacts in two different ways
#
#
# FIRST ONE
#
sandy_addr = Address.new
sandy_addr.street = "324 Campus Dr."
sandy_addr.city = "College Park"
sandy_addr.state = "OH"
sandy_addr.zip = "55555"
sandy = Person.new
sandy.fname = "Sandy"
sandy.lname = "Koh"
sandy.phone = "651-442-5710"
sandy.address = sandy_addr
#
# SECOND ONE
#
bill_addr = Address.new('536 Green Rd.', "Saint Paul", "MN", "56545")
bill = Person.new('William', 'Perry', '675-778-6754', bill_addr)
#
# THIRD ONE
#
angela_addr = Address.new('3390 Crookston Rd.', "Miami", "FL", "78654")
angela = Person.new('Angela', 'Anderson', '345-748-1754', angela_addr)
# Contacts added to the Address Book
#addressBook = AddressBook.new
#addressBook.add(sandy)
#addressBook.add(angela)
#addressBook.add(bill)
# Main method loop that runs the program
# Allows you to enter a contact, print a list of
# contacts and exit the program
def loop(addBook)
selection = 0
until(selection == 5)
puts("Wlecome to the Address Book\n")
puts("1. Add a Contact\n")
puts("2. Delete a Contact\n")
puts("3. Retrieve a Contact\n")
puts("4. Print all Contacts\n")
puts("5. Exit Address Book\n")
selection = gets().to_i
if(selection == 1)
addContact(addBook)
elsif(selection == 2)
#delete_contact
elsif(selection == 3)
#retrieve_contact
elsif(selection == 4)
puts(addBook)
end
end
end
def addContact(addBook)
print("Enter First Name: ")
fname = gets()
print("Enter Last Name: ")
lname = gets()
print("Enter Phone Number: ")
phone = gets()
print("Enter Street Address: ")
street = gets()
print("Enter City: ")
city = gets()
print("Enter State: ")
state = gets()
print("Enter Zip Code ")
zip = gets()1
newAddress = Address.new(street, city, state, zip)
newPerson = Person.new(fname, lname, phone, newAddress)
addBook.add(newPerson)
out = %q/#{fname} #{lname} has been added to the Address Book./
puts(out)
end
loop(#addressBook)
The gets method includes the newline so your fname in addContact will be, for example, "Bob\n" rather than the "Bob" that you're expecting. Have a look at chomp.
You problem is probably caused because gets() returns the entire line including the newline character at the end. Try replacing every gets() with gets.chomp.

Resources