diff --git a/cookbooks/sidekiq/attributes/default.rb b/cookbooks/sidekiq/attributes/default.rb index 082f9d34..668ebd2a 100644 --- a/cookbooks/sidekiq/attributes/default.rb +++ b/cookbooks/sidekiq/attributes/default.rb @@ -9,13 +9,13 @@ # only be installed on to a utility instance that matches # the name :utility_name => 'sidekiq', - + # Number of workers (not threads) :workers => 1, - + # Concurrency :concurrency => 25, - + # Queues :queues => { # :queue_name => priority @@ -30,5 +30,17 @@ # Timeout (in seconds) to use when terminating a bloated process # this is passed as a parameter to sidekiqctl, invoked inside /engineyard/bin/sidekiq - :timeout => 115 + :timeout => 115, + + # Setting this to true installs a cron job that + # regularly terminates sidekiq workers that aren't being monitored by monit, + # and terminates those workers + # + # default: false + :orphan_monitor_enabled => false, + + # sidekiq_orphan_monitor cron schedule + # + # default: every 5 minutes + :orphan_monitor_cron_schedule => "*/5 * * * *" } diff --git a/cookbooks/sidekiq/files/default/sidekiq_orphan_monitor b/cookbooks/sidekiq/files/default/sidekiq_orphan_monitor new file mode 100644 index 00000000..13402608 --- /dev/null +++ b/cookbooks/sidekiq/files/default/sidekiq_orphan_monitor @@ -0,0 +1,30 @@ +containsElement () { + local e match="$1" + shift + for e; do [[ "$e" == "$match" ]] && return 0; done + return 1 +} + +app_name=$1 +sidekiq_pids=`cat /var/run/engineyard/sidekiq/$app_name/sidekiq_*.pid` + +sidekiq_parent_pids=() +for sidekiq_pid in $sidekiq_pids; do + parent_pid="$(ps -o ppid= $sidekiq_pid | tr -d '[:space:]')" + sidekiq_parent_pids+="$parent_pid " +done + +running_sidekiqs="$(ps -ef | grep [s]idekiq | awk '{print $2}')" +echo "Running sidekiqs: $running_sidekiqs" +echo "Monit PIDs: $sidekiq_pids" +echo "Parent PIDs: $sidekiq_parent_pids" +for sidekiq in $running_sidekiqs; do + if (containsElement $sidekiq $sidekiq_pids); then + echo "Not terminating $sidekiq - it is being monitored by monit" + elif (containsElement $sidekiq $sidekiq_parent_pids); then + echo "Not terminating $sidekiq - it is a parent of a monit sidekiq process" + else + echo "Terminating $sidekiq" + sudo kill $sidekiq + fi +done diff --git a/cookbooks/sidekiq/recipes/setup.rb b/cookbooks/sidekiq/recipes/setup.rb index b3777591..a249c9b3 100644 --- a/cookbooks/sidekiq/recipes/setup.rb +++ b/cookbooks/sidekiq/recipes/setup.rb @@ -54,5 +54,28 @@ variables(node[:sidekiq]) end end + + if node[:sidekiq][:orphan_monitor_enabled] + cookbook_file '/engineyard/bin/sidekiq_orphan_monitor' do + source 'sidekiq_orphan_monitor' + owner node[:owner_name] + group node[:owner_name] + mode 0755 + backup false + action :create + end + + cron 'sidekiq_orphan_monitor' do + user node[:owner_name] + action :create + minute node[:sidekiq][:orphan_monitor_cron_schedule].split[0] + hour node[:sidekiq][:orphan_monitor_cron_schedule].split[1] + day node[:sidekiq][:orphan_monitor_cron_schedule].split[2] + month node[:sidekiq][:orphan_monitor_cron_schedule].split[3] + weekday node[:sidekiq][:orphan_monitor_cron_schedule].split[4] + command "/engineyard/bin/sidekiq_orphan_monitor #{app_name}" + end + end + end end