From 9a528a77e4dc929cee45d1479767f05e60e5ba2d Mon Sep 17 00:00:00 2001
From: Radamanthus Batnag <radamanthus@gmail.com>
Date: Fri, 25 Aug 2017 10:02:16 +0800
Subject: [PATCH] Sidekiq: adds a script to terminate orphan workers

---
 cookbooks/sidekiq/attributes/default.rb       | 20 ++++++++++---
 .../files/default/sidekiq_orphan_monitor      | 30 +++++++++++++++++++
 cookbooks/sidekiq/recipes/setup.rb            | 24 ++++++++++++++-
 3 files changed, 69 insertions(+), 5 deletions(-)
 create mode 100644 cookbooks/sidekiq/files/default/sidekiq_orphan_monitor

diff --git a/cookbooks/sidekiq/attributes/default.rb b/cookbooks/sidekiq/attributes/default.rb
index 082f9d34..668ebd2a 100644
--- a/cookbooks/sidekiq/attributes/default.rb
+++ b/cookbooks/sidekiq/attributes/default.rb
@@ -9,13 +9,13 @@
   # only be installed on to a utility instance that matches
   # the name
   :utility_name => 'sidekiq',
-  
+
   # Number of workers (not threads)
   :workers => 1,
-  
+
   # Concurrency
   :concurrency => 25,
-  
+
   # Queues
   :queues => {
     # :queue_name => priority
@@ -30,5 +30,17 @@
 
   # Timeout (in seconds) to use when terminating a bloated process
   # this is passed as a parameter to sidekiqctl, invoked inside /engineyard/bin/sidekiq
-  :timeout => 115
+  :timeout => 115,
+
+  # Setting this to true installs a cron job that
+  # regularly terminates sidekiq workers that aren't being monitored by monit,
+  # and terminates those workers
+  #
+  # default: false
+  :orphan_monitor_enabled => false,
+
+  # sidekiq_orphan_monitor cron schedule
+  #
+  # default: every 5 minutes
+  :orphan_monitor_cron_schedule => "*/5 * * * *"
 }
diff --git a/cookbooks/sidekiq/files/default/sidekiq_orphan_monitor b/cookbooks/sidekiq/files/default/sidekiq_orphan_monitor
new file mode 100644
index 00000000..13402608
--- /dev/null
+++ b/cookbooks/sidekiq/files/default/sidekiq_orphan_monitor
@@ -0,0 +1,30 @@
+containsElement () {
+  local e match="$1"
+  shift
+  for e; do [[ "$e" == "$match" ]] && return 0; done
+  return 1
+}
+
+app_name=$1
+sidekiq_pids=`cat /var/run/engineyard/sidekiq/$app_name/sidekiq_*.pid`
+
+sidekiq_parent_pids=()
+for sidekiq_pid in $sidekiq_pids; do
+  parent_pid="$(ps -o ppid= $sidekiq_pid | tr -d '[:space:]')"
+  sidekiq_parent_pids+="$parent_pid "
+done
+
+running_sidekiqs="$(ps -ef | grep [s]idekiq | awk '{print $2}')"
+echo "Running sidekiqs: $running_sidekiqs"
+echo "Monit PIDs: $sidekiq_pids"
+echo "Parent PIDs: $sidekiq_parent_pids"
+for sidekiq in $running_sidekiqs; do
+  if (containsElement $sidekiq $sidekiq_pids); then
+    echo "Not terminating $sidekiq - it is being monitored by monit"
+  elif (containsElement $sidekiq $sidekiq_parent_pids); then
+    echo "Not terminating $sidekiq - it is a parent of a monit sidekiq process"
+  else
+    echo "Terminating $sidekiq"
+    sudo kill $sidekiq
+  fi
+done
diff --git a/cookbooks/sidekiq/recipes/setup.rb b/cookbooks/sidekiq/recipes/setup.rb
index d1f5ec74..cd85403a 100644
--- a/cookbooks/sidekiq/recipes/setup.rb
+++ b/cookbooks/sidekiq/recipes/setup.rb
@@ -62,5 +62,27 @@
         notifies :run, resources(:execute => "restart-sidekiq-for-#{app_name}")
       end
     end
-  end 
+
+    if node[:sidekiq][:orphan_monitor_enabled]
+      cookbook_file '/engineyard/bin/sidekiq_orphan_monitor' do
+        source 'sidekiq_orphan_monitor'
+        owner node[:owner_name]
+        group node[:owner_name]
+        mode 0755
+        backup false
+        action :create
+      end
+
+      cron 'sidekiq_orphan_monitor' do
+        user    node[:owner_name]
+        action  :create
+        minute  node[:sidekiq][:orphan_monitor_cron_schedule].split[0]
+        hour    node[:sidekiq][:orphan_monitor_cron_schedule].split[1]
+        day     node[:sidekiq][:orphan_monitor_cron_schedule].split[2]
+        month   node[:sidekiq][:orphan_monitor_cron_schedule].split[3]
+        weekday node[:sidekiq][:orphan_monitor_cron_schedule].split[4]
+        command "/engineyard/bin/sidekiq_orphan_monitor #{app_name}"
+      end
+    end
+  end
 end