Gitlab Sidekiq Memory Killer

kill-sidekiq-if-huge.py

#! /usr/bin/env python3

import psutil
import signal
from time import sleep
from subprocess import Popen, PIPE

import logging
logger = logging.getLogger("sidekiq-killer")
#logging.basicConfig(level=logging.INFO)

for proc in psutil.process_iter():
   cmdline = proc.cmdline()

   if cmdline and "sidekiq" in cmdline[0] and "gitlab" in cmdline[0]:
       rss = proc.memory_info().rss
       if rss >= 500_000_000:
           logger.info(f"shutting down sidekiq with pid {proc.pid} after using {rss/1024/1024} MiB")
           proc.send_signal(signal.SIGTERM)
           logger.info(f"waiting...")

           increment = 10
           timeout = 240
           while timeout and proc.is_running():
               sleep(increment)
               timeout = max(0, timeout-increment)

           if proc.is_running():
               logger.info(f"killing")
               proc.kill()
           else:
               logger.info(f"process shut down on its own")

           logger.info(f"restarting")
           p = Popen(["/etc/init.d/gitlab", "start"], stdout=PIPE, stderr=PIPE)
           stdout, stderr = p.communicate()

If you (like me) run a version of Gitlab installed from source, you too will have experienced the joy that is the ballooning memory requirements of the Sidekiq instances run by Gitlab. Their omnibus package contains a sidekiq memory killer, but the from-source installation does not have this feature. Fear not! Stick the script above in a cron job, and enjoy all that memory that you didn't think you had available!