god doesn't stop unicorn - ruby

I have this file
rails_env = ENV['RAILS_ENV'] || 'development'
rails_root = ENV['RAILS_ROOT'] || "/home/luiz/rails_dev/api"
God.watch do |w|
w.name = "unicorn"
w.interval = 30.seconds # default
# unicorn needs to be run from the rails root
w.start = "cd #{rails_root} && unicorn_rails -c config/unicorn.rb -E #{rails_env}"
# QUIT gracefully shuts down workers
w.stop = "kill -QUIT `cat #{rails_root}/tmp/pids/unicorn.pid`"
# USR2 causes the master to re-create itself and spawn a new worker pool
w.restart = "kill -USR2 `cat #{rails_root}/tmp/pids/unicorn.pid`"
w.start_grace = 10.seconds
w.restart_grace = 10.seconds
w.pid_file = "#{rails_root}/tmp/pids/unicorn.pid"
w.behavior(:clean_pid_file)
w.start_if do |start|
start.condition(:process_running) do |c|
c.interval = 5.seconds
c.running = false
end
end
w.restart_if do |restart|
restart.condition(:memory_usage) do |c|
c.above = 300.megabytes
c.times = [3, 5] # 3 out of 5 intervals
end
restart.condition(:cpu_usage) do |c|
c.above = 50.percent
c.times = 5
end
end
# lifecycle
w.lifecycle do |on|
on.condition(:flapping) do |c|
c.to_state = [:start, :restart]
c.times = 5
c.within = 5.minute
c.transition = :unmonitored
c.retry_in = 10.minutes
c.retry_times = 5
c.retry_within = 2.hours
end
end
end
I start unicorn with god -c unicorn.god -D -p 8081 and my workers are setup fine. but, sometimes I need do stop unicorn (god stop unicorn -p 8081 in another console) but the server keep up and running.
what am I missing?
Edit
We're moving from unicorn to puma (not because this question, it's a performance thing), and not going to use god anymore...thanks everybody for your help

Now we are using foreman and puma right now...this is our initializer:
web: bundle exec puma -q -d -e production -b 'unix:///home/api/shared/web.socket' -S /home/api/shared/web.state --control 'unix:///home/api/shared/web.ctl'
and deploying with capistrano, so that we can stop and restart the server like this
config/deploy.rb
# Puma commands
_cset(:puma_cmd) { "#{fetch(:bundle_cmd, 'bundle')} exec puma" }
_cset(:pumactl_cmd) { "#{fetch(:bundle_cmd, 'bundle')} exec pumactl" }
_cset(:puma_state) { "#{shared_path}/puma.state" }
_cset(:puma_role) { :app }
# Puma
namespace :puma do
desc 'Start puma'
task :start do
run "cd #{current_path} ; bundle exec foreman start web"
end
desc 'Stop puma'
task :stop, :roles => lambda { fetch(:puma_role) }, :on_no_matching_servers => :continue do
run "cd #{current_path} && #{fetch(:pumactl_cmd)} -S #{fetch(:puma_state)} stop"
end
desc 'Restart puma'
task :restart, :roles => lambda { fetch(:puma_role) }, :on_no_matching_servers => :continue do
run "cd #{current_path} && #{fetch(:pumactl_cmd)} -S #{fetch(:puma_state)} restart"
end
end

Related

Unicorn is not part of the bundle while starting from God

I have a sinatra app that works great when i start it from the project directory using bundle exec unicorn -c config/unicorn.rb -D. However when i use god to start it, I get error stating unicorn is not part of the bundle. Add it to Gemfile. (Gem::LoadError). I do have unicorn in my gemfile. I have been struggling with it for some time now, without much help. Any clues?
The exact error I get is
/Users/pranav/.rvm/gems/ruby-2.1.5#global/gems/bundler-1.11.2/lib/bundler/rubygems_integration.rb:304:in `block in replace_gem': unicorn is not part of the bundle. Add it to Gemfile. (Gem::LoadError)
from /Users/pranav/.rvm/gems/ruby-2.1.5/bin/unicorn:22:in `<main>'
from /Users/pranav/.rvm/gems/ruby-2.1.5/bin/ruby_executable_hooks:15:in `eval'
from /Users/pranav/.rvm/gems/ruby-2.1.5/bin/ruby_executable_hooks:15:in `<main>'
Here is the god config I use
proj_dir = ENV['DIR_API_HANDLER']
script_name = 'api_handler'
pid_file = File.join(proj_dir, 'shared', 'pids', 'unicorn.pid')
log_file = File.join(proj_dir, 'shared', 'log', 'unicorn.stderr.log')
God.watch do |w|
w.name = script_name
w.group = 'apihandler'
w.log = log_file
w.dir = proj_dir
w.env = ENV
w.start = "cd #{ proj_dir } && bundle exec unicorn -c config/unicorn.rb -D"
# QUIT gracefully shuts down workers
w.stop = "cd #{ proj_dir} && kill -QUIT `cat #{ pid_file }`"
# USR2 causes the master to re-create itself and spawn a new worker pool
w.restart = "kill -USR2 `cat #{ pid_file }`"
w.start_grace = 10.seconds
w.restart_grace = 15.seconds
w.interval = 30.minutes
w.pid_file = pid_file
w.behavior(:clean_pid_file)
w.start_if do |start|
start.condition(:process_running) do |c|
c.interval = 5.seconds
c.running = false
end
end
w.restart_if do |restart|
restart.condition(:memory_usage) do |c|
c.above = 300.megabytes
c.times = [3, 5] # 3 out of 5 intervals
end
restart.condition(:cpu_usage) do |c|
c.above = 50.percent
c.times = 5
end
end
# lifecycle
w.lifecycle do |on|
on.condition(:flapping) do |c|
c.to_state = [:start, :restart]
c.times = 5
c.within = 5.minute
c.transition = :unmonitored
c.retry_in = 10.minutes
c.retry_times = 5
c.retry_within = 2.hours
end
end
end
My Gemfile looks like
source 'https://rubygems.org'
ruby '2.1.5'
gem 'sinatra'
gem 'sinatra-contrib'
gem 'json'
gem 'unicorn', '~>5.1.0'
gem 'ruby-kafka', '~>0.3.2'
gem 'dotenv'
gem 'curb'
The config/unicorn.rb looks like below
# set path to application
app_dir = File.expand_path("../..", __FILE__)
shared_dir = "#{app_dir}/shared"
working_directory app_dir
# Set unicorn options
worker_processes 2
preload_app false
timeout 30
listen 9001
# Set up socket location
#listen "#{shared_dir}/sockets/unicorn.sock", :backlog => 64
# Logging
stderr_path "#{shared_dir}/log/unicorn.stderr.log"
stdout_path "#{shared_dir}/log/unicorn.stdout.log"
# Set master PID location
pid "#{shared_dir}/pids/unicorn.pid"
The log from god -c config.god -D is below
I [2016-04-22 18:42:37] INFO: Loading octo.god
I [2016-04-22 18:42:37] INFO: Syslog enabled.
I [2016-04-22 18:42:37] INFO: Using pid file directory: /Users/pranav/.god/pids
I [2016-04-22 18:42:37] INFO: Started on drbunix:///tmp/god.17165.sock
I [2016-04-22 18:42:37] INFO: api_handler move 'unmonitored' to 'up'
I [2016-04-22 18:42:37] INFO: api_handler moved 'unmonitored' to 'up'
I [2016-04-22 18:42:37] INFO: api_handler [trigger] process is not running (ProcessRunning)
I [2016-04-22 18:42:37] INFO: api_handler move 'up' to 'start'
I [2016-04-22 18:42:37] INFO: api_handler before_start: no pid file to delete (CleanPidFile)
I [2016-04-22 18:42:37] INFO: api_handler start: cd /Users/pranav/workspace/apihandler && bundle exec unicorn -c config/unicorn.rb -D
W [2016-04-22 18:42:38] WARN: api_handler start command exited with non-zero code = 1
It looks like God is using default location for PID file, while the location is explicitly specified. Could it be because of this?

Mixlib::ShellOut - timeout

I am trying to use Mixlib::ShellOut to execute commands under ruby_block inside a chef recipe.
In Some situations, we cannot complete the task in 600 seconds, and I would like extend further. I have added command in below way,
ruby_block "#{host_short_name}_reg_chef_node" do
block do
puts "Registering Chef Node #{host_full_name}"
_command = "cd #{node['nodeManager']['app']['base_dir']}; #{node['nodeManager']['knife']['binary']} bootstrap --sudo #{host_full_name}"
_command += " --ssh-user #{node['nodeManager']['admin']['user']} --no-host-key-verify --identity-file #{node['nodeManager']['admin']['keyfile']}"
_command +=" --environment #{params[:environment]} --run-list 'role[#{params[:role_hash]['role']}]'"
puts _command
vsphere_output = Mixlib::ShellOut.new(_command, :timeout => 10000)
vsphere_output.run_command
puts "Output: #{vsphere_output.stdout}"
puts "Error : #{vsphere_output.stderr}"
end
action :nothing
end
and I suspect it is not respecting timeout value. Please advise.

Clockwork recipe for Capistrano 3

I'm trying to write a Clockwork recipe for Capistrano 3. After having a look at Sidekiq's recipe I've come up with this:
namespace :load do
task :defaults do
set :clockwork_default_hooks, -> { true }
set :clockwork_pid, -> { 'tmp/pids/clockwork.pid' }
set :clockwork_log, -> { "#{current_path}/log/clockwork.log" }
set :clockwork_roles, -> { :app }
set :clockwork_config, -> { 'clock.rb' }
end
end
namespace :deploy do
before :starting, :check_clockwork_hooks do
invoke 'clockwork:add_default_hooks' if fetch(:clockwork_default_hooks)
end
end
namespace :clockwork do
def clockwork_pid_full_path
if fetch(:clockwork_pid).start_with?('/')
fetch(:clockwork_pid)
else
"#{current_path}/#{fetch(:clockwork_pid)}"
end
end
task :add_default_hooks do
after 'deploy:updated', 'clockwork:stop'
after 'deploy:reverted', 'clockwork:stop'
after 'deploy:published', 'clockwork:start'
end
desc 'Stop clockwork'
task :stop do
on roles fetch(:clockwork_roles) do
if test "[ -f #{clockwork_pid_full_path} ]"
within current_path do
execute "kill -int $(cat #{clockwork_pid_full_path}) 2>/dev/null"
end
else
execute "echo 'clockwork was not running'"
end
end
end
desc 'Start clockwork'
task :start do
on roles fetch(:clockwork_roles) do
within current_path do
with rails_env: fetch(:rails_env, 'production') do
execute "export RAILS_ENV=$RAILS_ENV"
execute :bundle, :exec, :clockwork, "#{fetch(:clockwork_config)} >> #{fetch(:clockwork_log)} 2>&1 &"
end
execute "ps -eo pid,command | grep clockwork | grep -v grep | awk '{print $1}' > #{clockwork_pid_full_path}"
end
end
end
desc 'Restart clockwork'
task :restart do
invoke 'clockwork:stop'
invoke 'clockwork:start'
end
end
However, the clockwork:start task only works if I remove the 2>&1 & part at the end. If I try to start the process in the background, nothing seems to happen.
What am I doing wrong?
This is what I ended up with. I adapted it from a Gist I found.
namespace :clockwork do
desc "Stop clockwork"
task :stop do
on roles(:app) do
within release_path do
with rails_env: fetch(:rails_env) do
execute :bundle, :exec, :clockworkd, "-c clock.rb --pid-dir=#{cw_pid_dir} --log-dir=#{cw_log_dir} --log stop"
end
end
end
end
desc "Clockwork status"
task :status do
on roles(:app) do
within release_path do
with rails_env: fetch(:rails_env) do
execute :bundle, :exec, :clockworkd, "-c clock.rb --pid-dir=#{cw_pid_dir} --log-dir=#{cw_log_dir} --log status"
end
end
end
end
desc "Start clockwork"
task :start do
on roles(:app) do
within release_path do
with rails_env: fetch(:rails_env) do
execute :bundle, :exec, :clockworkd, "-c clock.rb --pid-dir=#{cw_pid_dir} --log-dir=#{cw_log_dir} --log start"
end
end
end
end
desc "Restart clockwork"
task :restart do
on roles(:app) do
within release_path do
with rails_env: fetch(:rails_env) do
execute :bundle, :exec, :clockworkd, "-c clock.rb --pid-dir=#{cw_pid_dir} --log-dir=#{cw_log_dir} --log restart"
end
end
end
end
def cw_log_dir
"#{shared_path}/log"
end
def cw_pid_dir
"#{shared_path}/tmp/pids"
end
def rails_env
fetch(:rails_env, false) ? "RAILS_ENV=#{fetch(:rails_env)}" : ''
end
end
Try this gem capistrano-clockwork.
It's the most easy way that worked very well to me. I tried several options of 'gists' but no success.
You just need update this files Gemfile, Capifile and deploy.rb.
Gemfile:
gem 'daemons'
gem 'capistrano-clockwork', group: :development
Capfile:
require 'capistrano/clockwork'
deploy.rb:
set :clockwork_file, "lib/name_of_your_clockwork_config.rb"
Remember to execute bundle install before deploy ;)

one god for different rubies with rvm

I have two apps on my machine.
Each app (server) has it's own gemset and works on a different ruby version.
I will manage those apps with god which is installed in it's own gemset.
My god config file config.god looks like this:
God.watch do |w|
current_path = "/home/vagrant/server-1"
w.name = "server 1"
w.start = "ruby #{current_path}/simple-server.rb"
w.keepalive
end
God.watch do |w|
current_path = "/home/vagrant/server-2"
w.name = "server 2"
w.start = "ruby #{current_path}/simple-server.rb"
w.keepalive
end
My servers are simply writing the ruby version to a file (/home/vagrant/server-2/simple-server.rb):
require "date"
loop do
# simple console output
puts "Hello on #{RUBY_VERSION}, #{RUBY_PATCHLEVEL}, #{RUBY_PLATFORM}, #{RUBY_RELEASE_DATE}"
# Specify the name of the log file
log_file = File.join File.expand_path( File.dirname(__FILE__) ), "testfile.txt"
# Write the log into the file
File.open( log_file, 'a') do |f|
date = DateTime.now
date = date.strftime("%H:%M:%S")
f.puts "#{date} on #{RUBY_VERSION}, #{RUBY_PATCHLEVEL}, #{RUBY_PLATFORM}, #{RUBY_RELEASE_DATE}"
end
sleep 2
end
I run god with god -c config.god.
The problem is that my apps are not running with the ruby versions which is specified in the .rvmrc.
I have also tried:
~/.rvm/bin/wrapped_god -d config.god -D
rvmsudo ~/.rvm/bin/wrapped_god -d config.god -D
rvmsudo god -d config.god -D
Is there a solution for this case?
EDIT 2012.08.27:
I have changed my god config as follows:
w.start="~/.rvm/bin/rvm in #{current_path} do ruby simple-server.rb"
And it worked.
try:
start="~/.rvm/bin/rvm in #{current_path} do ruby simple-server.rb"

How to configure god/redis for two environments?

I want to use god to monitor my redis server. The problem is that they are in different locations locally and on my server. I tried doing something like this but this causes god to fail.
%w{6379}.each do |port|
God.watch do |w|
w.name = "redis"
w.interval = 30.seconds
if RAILS_ENV == 'development'
w.start = "/usr/local/bin/redis-server /usr/local/bin"
w.stop = "/usr/local/bin/redis-cli -p 6379 shutdown"
else # in production
w.start = "/usr/local/etc/redis-server /usr/local/etc"
w.stop = "/usr/local/etc/redis-cli -p 6379 shutdown"
end
end
end
Use
if ENV["RAILS_ENV"] == 'development'
And then call your script using
RAILS_ENV=development sudo god ...
in your dev environment.
and in production:
sudo god ...

Resources