nsm_sensor_purge.sh

#!/bin/bash
#
# Copyright (C) 2011 Doug Burks and Security Onion
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License Version 2 as
# published by the Free Software Foundation.  You may not use, modify or
# distribute this program under any other version of the GNU General
# Public License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#

# Version:
# 20111229

# Changelog:
# 20111214 - Initial version
# 20111229 - Add date to output
# 20160401 - Dustin corrupted on April Fools
# 20160412 - Adjusted to reflect server_clear and added ELSA purge
#
# INCLUDES
#
INC="/etc/nsm/administration.conf"
. $INC

. $NSM_LIB_DIR/lib-console-utils
. $NSM_LIB_DIR/lib-nsm-common-utils
. $NSM_LIB_DIR/lib-nsm-sensor-utils
. $NSM_LIB_DIR/lib-nsm-server-utils

##############################################
# Site specific variables!
##############################################

# Server or Sensor (change to 0 if sensor or NOT standalone)
SERVER=1

# Snorby email and password information
SNORBY_EMAIL="[email protected]"
SNORBY_PASS="soadmin"

# Sguil/Squert/Elsa user and password information
SGUIL_USER="soadmin"
SGUIL_PASS="soadmin"

##############################################
# Default site variables
##############################################

# Securityonion.conf location for user specific configs
SO_CONF="/etc/nsm/securityonion.conf"

# Default server name
SERVER_NAME="securityonion"

# Snorby table information
TABLES="settings lookups sensor"
DATE=`date +%Y%m%d%H%m%S`
BACKUP="/root/snorby-backup-$DATE"

# Sguil database information
SERVER_DB_NAME="securityonion_db"
SERVER_DB_USER="sguil"
SERVER_DB_PASS="password"

# Logging dir
LOG=`mktemp /tmp/sensorpurge.log.XXXXXXXXXX`

# Calculate half of available disk space for Elsa logs (default)
DISK_SIZE_K=`df /nsm |grep -v "^Filesystem" | awk '{print $2}'`
let DISK_SIZE=DISK_SIZE_K*1000
let LOG_SIZE_LIMIT=DISK_SIZE/2

##############################################
# Usage
##############################################

print_usage()
{
	echo 
	echo "The NSMnow Administration scripts come with ABSOLUTELY NO WARRANTY."
	echo 
	echo "Usage: $0 [options]"
	echo
	echo "Options:"
	echo "    -d         Use dialog mode"
	echo "    -y         Force yes"
	echo "    -V         Show version information"
	echo "    -?         Show usage information"
	echo 
	echo "Long Options:"
	echo
	echo "    --dialog              Same as -d"
	echo "    --force-yes           Same as -y"
	echo
	echo "    --version             Same as -V"
	echo "    --help                Same as -?"
	echo 
}

##############################################
# Version
##############################################

print_version()
{ 
	echo "V 1.1"
}

# script specific variables
PROMPT_SCRIPT="Purge Sensor"
PROMPT_MODE=cli
FORCE_YES=""

# sensor specific variables
SENSOR_NAME=""

# extract necessary pre-check arguments from the commandline
while [ "$#" -gt 0 ]
do
	case $1 in
		"-d" | "--dialog")
			PROMPT_MODE=dialog
			;;
		"-y" | "--force-yes")
			FORCE_YES=yes
			;;
		"-?" | "--help")
			SHOW_HELP_ONLY=yes
			;;
		"-V" | "--version")
			SHOW_VERSION_ONLY=yes
			;;
		--server*)
			# any server directive is clearly meant for the server
			exit 0
			;;
        *)
			echo_error_msg 0 "OOPS: Unknown option \"${1}\" found!"
			print_usage
			exit 1
            ;;
	esac
	shift
done

# check for help or version requests
if [ -n "$SHOW_HELP_ONLY" ]
then
	print_usage
	exit 0
elif [ -n "$SHOW_VERSION_ONLY" ]
then
	print_version
	exit 0
fi

# ensure we are root user before continuing any further
is_root
if [ "$?" -ne 0 ]
then
	echo_error_msg 0 "OOPS: Must be root to run this script!"
	exit 1;
fi
	

echo
echo "===================================================================="
echo "This script will delete all logs located in:"
echo "- /nsm/bro/logs/*"
echo "- /nsm/bro/extracted/*"
echo "- /nsm/sensor_data/SENSORDIR/dailylogs/*"
echo "- /nsm/sensor_data/SENSORDIR/argus/*"
echo "- /nsm/sensor_data/SENSORDIR/snort-*/*"
echo 
echo "It will delete all alerts in the Snorby database."
echo "It will also attempt to do the following in Snorby:"
echo "- backup the following tables to $BACKUP"
echo "	$TABLES"
echo "- initialize a new Snorby database"
echo "- restore the Snorby backup from $BACKUP"
echo
echo "In addition, it will delete and recreate the Sguil and ELSA"
echo "databases while purging Sphinx data."
echo 
echo "Finally, a rule-update will be performed prior to restarting"
echo "sensor and server services."
echo 
echo "If you have other sensors reporting to this server, you should"
echo "manually stop their services before continuing."
echo "You can do so by running the following command on each sensor:"
echo "sudo service nsm stop"
echo "===================================================================="

##############################################
# Collect Input
##############################################

#
# We clean all sensors at the same time anyway, so just grab the first sensor
SENSOR_NAME=`grep -v "^#" /etc/nsm/sensortab |awk '{print $1}' |head -1`

# check that the sensor DOES exists via it's config
if [ ! -f "/etc/nsm/${SENSOR_NAME}/sensor.conf" ]
then
	echo_error_msg 0 "OOPS: The sensor \"${SENSOR_NAME}\" does not exist!"
	exit 1
else
	# load existing variables for the sensor
	. "/etc/nsm/${SENSOR_NAME}/sensor.conf"
fi

# check that the sensor exists
if [ ! -d "/nsm/sensor_data/${SENSOR_NAME}" ]
then
	echo_error_msg 0 "OOPS: Collected data for sensor \"${SENSOR_NAME}\" does not exist!"
	exit 1
fi

if [ "$FORCE_YES" == "" ]
then
	# prompt to clean the sensor
	prompt_user_yesno "Do you want to continue?\n" "N"
	[ "$?" -ne 0 ] && exit 1
	if [ "$PROMPT_RET" != "Y" -a "$PROMPT_RET" != "y" ]
	then
		exit 1
	fi
fi

##############################################
# Beginning the purge 
##############################################

date
echo_msg 0 "\n\nBeginning to remove data from sensors...."

# Stop sensor and server services
echo_msg 0 "\n\nStopping services...."
service nsm stop

# Remove the Bro log files as appropriate
if [ ! -d "/nsm/bro/logs" ]
then
        echo_error_msg 0 "Bro does not seem to be installed or have logs on disk"
        continue
else
	echo_msg 0 "\n\nDeleting Bro logs...."
	rm -rf /nsm/bro/logs/*
fi

# Delete any extracted files from default Bro directory
if [ ! -d "/nsm/bro/extracted" ]
then
        echo_error_msg 0 "Bro does not seem to have any extracted files on disk"
        continue
else
        echo_msg 0 "\n\nDeleting Bro extracted files...."
        rm -rf /nsm/bro/extracted/*
fi

# Deleting GeoIP info if this script is ran while old .dats exist in /tmp
if [ -f "/tmp/GeoIP.dat" -o -f "/tmp/GeoLiteCity.dat" ]
then
	echo_msg 0 "\n\nRemoving leftover GeoIP data...."
	rm -rf /tmp/Geo*.dat
fi

# Remove pcap data from sensor
echo_msg 0 "\n\nDeleting dailylogs...."
for f in /nsm/sensor_data/*-*/dailylogs/*; do
    [ -e "$f" ]  && rm -rf /nsm/sensor_data/*-*/dailylogs/* 
done

# Remove Snort alert data from sensor
echo_msg 0 "\n\nDeleting unified2 alerts...."
for f in /nsm/sensor_data/*-*/snort-*/*; do
    [ -e "$f" ] && rm -rf /nsm/sensor_data/*-*/snort-*/* 
done

# Remove argus flow data from sensor
echo_msg 0 "\n\nDeleting Argus data...."
for f in /nsm/sensor_data/*-*/argus/*; do
    [ -e "$f" ] && rm -rf /nsm/sensor_data/*-*/argus/* 
done

##############################################
# Snorby Purge
##############################################

# Notification message 
echo_msg 0 "\n\nWiping and recreating Snorby data...."

# shut down NSM services and Snorby
pkill delayed_job

# backup snorby's config-related db tables
mysqldump snorby $TABLES > $BACKUP

# Delete any existing Snorby data.
if [ -d /var/lib/mysql/snorby ]; then
   mysql -e "drop database snorby"
fi

# Set  email and password to facilitate initialization 
cp /opt/snorby/db/seeds.rb.securityonion /opt/snorby/db/seeds.rb
sed -i "s|ReplaceWithDesiredEmail|$SNORBY_EMAIL|g" /opt/snorby/db/seeds.rb
sed -i "s|ReplaceWithDesiredPassword|$SNORBY_PASS|g" /opt/snorby/db/seeds.rb

# Set FPC options
IP=`ifconfig |grep "inet addr" | awk '{print $2}' |cut -d\: -f2 |grep -v "127.0.0.1" |head -1`
sed -i "s|packet_capture_url, nil|packet_capture_url, 'https://$IP/capme/'|g" /opt/snorby/db/seeds.rb
sed -i "s|packet_capture, nil|packet_capture, 1|g" /opt/snorby/db/seeds.rb
sed -i "s|packet_capture_auto_auth, 1|packet_capture_auto_auth, nil|g" /opt/snorby/db/seeds.rb

# Initialize Snorby DB - will take while
su www-data -c "cd /opt/snorby; bundle exec rake snorby:setup RAILS_ENV=production"

# restore config tables
mysql snorby < $BACKUP

# Shred the Snorby password
shred -u /opt/snorby/db/seeds.rb

echo
echo "===================================================================="
echo "Snorby database backup can be found at $BACKUP."
echo "If you're able to login to Snorby and everything works properly,"
echo "then you'll probably want to shred this file:"
echo "sudo shred -u $BACKUP"
echo "===================================================================="

##############################################
# Sguil Purge
##############################################

echo_msg 0 "\n\nPurging Sguil/Squert database...."

# clear the files as appropriate
if [ -d "/nsm/server_data/${SERVER_NAME}" ]
then
	echo_msg_begin 1 "Removing collected data files."
	find "/nsm/server_data/${SERVER_NAME}/archive" "/nsm/server_data/${SERVER_NAME}/load" -type f | while read FILE
	do
		rm -f "${FILE}"
	done
	echo_msg_end "$?"
fi

# remove the database entry
echo_msg_begin 1 "Clearing database."
server_sguil_database_clear "${SERVER_DB_NAME}" "${SERVER_DB_USER}" "${SERVER_DB_PASS}"
echo_msg_end "$?"

# Add Sguil/Squert/Elsa user again
echo_msg 0 "\n\nAdding Sguil/Squert/Elsa user...."
sguild-add-user $SGUIL_USER $SGUIL_PASS

#########################################
# ELSA Purge
#########################################

# update LOG_SIZE_LIMIT
sed -i "s|\"log_size_limit\" :.*$|\"log_size_limit\" : $LOG_SIZE_LIMIT,|g" /opt/elsa/contrib/securityonion/contrib/securityonion-elsa-node.conf

# update query_timeout
#sed -i "s|\"query_timeout\":.*$|\"query_timeout\": 10000,|g" /opt/elsa/contrib/securityonion/contrib/securityonion-elsa-web.conf
#sed -i "s|\"query_timeout\":.*$|\"query_timeout\": 10000,|g" /opt/elsa/contrib/securityonion/contrib/securityonion-elsa-api.conf

# check if there is an ELSA directive already
ELSA_USE=$(grep "ELSA=" $SO_CONF)
if [ ! $? -eq 0 ]; then
	echo "" >> $SO_CONF
	echo "# ELSA" >> $SO_CONF
        echo "ELSA=YES" >> $SO_CONF
else
        sed -i 's,ELSA=NO,ELSA=YES,' $SO_CONF
fi

# Delete any existing ELSA databases.
echo_msg 0 "\n\nDeleting ELSA databases...."
if [ -d /var/lib/mysql/elsa_web/ ]; then
	mysql -e "drop database elsa_web" >> $LOG 2>&1
fi
if [ -d /var/lib/mysql/syslog/ ]; then
    	mysql -e "drop database syslog" >> $LOG 2>&1
fi
if [ -d /var/lib/mysql/syslog_data/ ]; then
    	mysql -e "drop database syslog_data" >> $LOG 2>&1
fi

# Delete sphinxsearch binlog files
echo_msg 0 "\n\nDeleting Sphinx binlog files...."
rm -f /var/lib/sphinxsearch/data/binlog.*

# Configure all Log Node and Web Node functionality if we are a server
echo_msg 0 "\n\nConfiguring Elsa Node functionality along with GeoIP data...."
if [ $SERVER -eq 1 ]; then
	/usr/bin/securityonion-elsa-config.sh -t WEB >> $LOG 2>&1
	# restart apache to update ELSA APIKEY
	service apache2 restart >> $LOG 2>&1
else
        /usr/bin/securityonion-elsa-config.sh -t LOG >> $LOG 2>&1
        # Update the securityonion group <= Come back to this later
        #echo "usermod -a -G securityonion $SSH_USERNAME" >> $SOSETUPSCP
fi

# update Snorby's reference tables
echo_msg 0 "\n\nUpdating ruleset...."
rule-update

# Restart sensor after cleanup
echo_msg 0 "\n\nStarting sensor and server...."
nsm_sensor_ps-restart --skip-barnyard2 && nsm_server_ps-restart

echo_msg 0 "\n\nComplete.\nPlease remember to delete the ${BACKUP} file if everything checks out okay.\nHave fun!\n\n"
exit 0