Memory Watchdog Service for GNU Social daemons + RabbitMQ

memory_watcher.php

#!/usr/bin/php
<?php
if(php_sapi_name() !== 'cli'){
	die();
}
//Tiny config options
$max = 95; //Max percentage the script should detect (in RAM Usage)
$restartScript = "/root/restartqueue.sh"; //Location of the restart script for the daemons.
/* restartqueue.sh "/root/restartqueue.sh" - also see: https://gist.github.com/mitchellurgero/e914c6109d0ad31fd85bf6d4dbb7e1f1
#!/bin/bash
systemctl stop gnusocial.service
systemctl stop mysql.service
systemctl restart rabbitmq-server.service
systemctl restart apache2.service
systemctl start mysql.service
systemctl start gnusocial.service
*/
echo "Watching RAM usage...\r\n";
while(true){
	//Test to see if current RAM usage is greater (or equal) to configured MAX limit.
	if(getRam() >= $max){
		echo "RAM USAGE IS OVER $max%!\r\nDetecting which program is taking up that amount of RAM.\r\n";
		//Now we want to detect if RMQ and queuedaemon.php processes are using more than 80% of that.
		scanRMQ();
	}
	sleep(4);
}
//Get current ram usage in percentage
function getRam(){
	$total = exec("grep MemTotal /proc/meminfo | awk '{print $2}'");
	$free = exec("grep MemFree /proc/meminfo | awk '{print $2}'");
	$cached = shell_exec("grep Cached /proc/meminfo | awk '{print $2}'");
	$used = $total - $free - $cached;
	$free = $free + $cached;
	$total = round($total / 1024, 2);
	$free = round($free / 1024, 2);
	$used = round($used / 1024, 2);
	$percent = (($free / $total) * 100);
	return round($percent);
}
//Get total USED RAM in MB
function getUsed(){
	$total = exec("grep MemTotal /proc/meminfo | awk '{print $2}'");
	$free = exec("grep MemFree /proc/meminfo | awk '{print $2}'");
	$cached = shell_exec("grep Cached /proc/meminfo | awk '{print $2}'");
	$used = $total - $free - $cached;
	$free = $free + $cached;
	$total = round($total / 1024, 2);
	$free = round($free / 1024, 2);
	$used = round($used / 1024, 2);
	return round($used);
}
//Get total RAM in MB
function getTotal(){
	$total = exec("grep MemTotal /proc/meminfo | awk '{print $2}'");
	$total = round($total / 1024, 2);
	return round($total);
}
//We will specifically scan both RMQ PID's and queuedaemon.php PID's for their TOTAL RAM usage in MB & percentage
function scanRMQ(){
	global $restartScript;
	//Get PID's
	exec('ps aux | grep -i rabbitmq | awk {\'print $2\'}', $rmq);
	exec('ps aux | grep -i queuedaemon.php | awk {\'print $2\'}', $qd);
	//Remove first item in array because that will be our grep command.
	array_shift($rmq);
	array_shift($qd);
	$trmq = 0;
	//for each PID RMQ has, get it's current RAM usage in MB.
	foreach($rmq as $rp){
		$pid = trim($rp);
		//Get ram usage of RMQ process(s):
		$pmap = exec('pmap -x '.$pid.' | grep "total"');
		preg_match_all('!\d+!', $pmap, $matches);
		if(count($matches[0]) != 0){
			//We only want to match PID's that are using RAM, not cached processes.
			$trmq = $trmq + (round($matches[0][0] / 1024, 0));
		}
	}
	//For each PID queuedaemon.php has, get it's current RAM usage in MB
	foreach($qd as $rp){
		$pid = trim($rp);
		//Get ram usage of queuedaemon.php process(s):
		$pmap = shell_exec('pmap -x '.$pid.' | grep "total"');
		preg_match_all('!\d+!', $pmap, $matches);
		if(count($matches[0]) != 0){
			//We only want to match PID's that are using RAM, not cached processes.
			$trmq = $trmq + (round($matches[0][0] / 1024, 0));
		}
	}
	//Now we grab current percentage to compare to the total amount of RAM in system (We only want to restart these daemons if THEY are the ones using all the RAM.)
	$prmq = round(($trmq / getUsed()) * 100);
	echo "RabbitMQ & queuedaemon.php Memory Usage: $trmq MB($prmq% usage of currently USED RAM.)\r\n";
	//80% seems like a good number, but maybe needs to be a 'little' higher?
	if($prmq >= 80){
		//Run restart of daemons because they are using more than 80% of the currently used RAM...
		echo "RabbitMQ & queuedaemon.php is running with more than 80% of the currently used RAM! Restarting them now... \r\n";
		echo shell_exec("bash $restartScript");
	} else {
		echo "RabbitMQ is NOT using all the RAM - Do NOT restart...\r\n";
	}
}
?>