Free up disk space on Ubuntu - clean log, cache, archive packages/apt archives, orphaned packages, old kernel and remove the trash

clean.sh

#!/bin/sh
# source of the script can be found at: https://gist.github.com/ad3bbd9d5cafea47bfd66367a0a8cdb6.git

#Check the space on the computer before clean up
df -h

#Check the Drive Space Used by Cached Files
du -sh /var/cache/apt/archives

#Clean all the log file
#for logs in `find /var/log -type f`;  do > $logs; done
du -h --max-depth=1 /var/log

logs=`find /var/log -type f`
for i in $logs
do
#> $i
  truncate $i --size 0
done

#Getting rid of partial packages
apt-get clean && apt-get autoclean
apt-get remove --purge -y software-properties-common

#Getting rid of no longer required packages
apt-get autoremove -y


#Getting rid of orphaned packages
deborphan | xargs sudo apt-get -y remove --purge

#Free up space by clean out the cached packages
apt-get clean

# Remove the Trash
rm -rf /home/*/.local/share/Trash/*/**
rm -rf /root/.local/share/Trash/*/**

# Remove Man
rm -rf /usr/share/man/?? 
rm -rf /usr/share/man/??_*

#Delete all .gz and rotated file
find /var/log -type f -regex ".*\.gz$" | xargs rm -Rf
find /var/log -type f -regex ".*\.[0-9]$" | xargs rm -Rf

#Cleaning the old kernels
dpkg-query -l|grep linux-im*
#dpkg-query -l |grep linux-im*|awk '{print $2}'
apt-get purge $(dpkg -l 'linux-*' | sed '/^ii/!d;/'"$(uname -r | sed "s/\(.*\)-\([^0-9]\+\)/\1/")"'/d;s/^[^ ]* [^ ]* \([^ ]*\).*/\1/;/[0-9]/!d' | head -n -1) --assume-yes
apt-get install linux-headers-`uname -r|cut -d'-' -f3`-`uname -r|cut -d'-' -f4`

#Once the old kernel is cleaned, there are other no longer required packages
apt-get autoremove -y

#Cleaning is completed
echo "Cleaning is completed"

#Check the space on the computer after clean up
df -h

clean_old_backups.sh

#!/bin/bash
##		    -i for interactive to allow pass the shell aliases to the bash subproces.
# Script origin @ https://gist.github.com/delphym/ad3bbd9d5cafea47bfd66367a0a8cdb6#file-clean_old_backups-sh
# Cleaning the old backups made by Simple Backup Suite
# There is a bug in the app #1770528, so it can't purge old backup snapshots
# https://bugs.launchpad.net/ubuntu/+source/sbackup/+bug/1770528
#set -x

#Default values
dryRun="Yes"		#"Yes|No"
simpleOption="No"	#"Yes|No"
saferOption="Yes"	#"Yes|No"
purge=7				#erase all backups older than X days (counting from 24 hrs ago)

LOG_LOCATION=/var/log/sbackup/
LOG_FILE=`basename "$0"`
LOG_FILE=`echo ${LOG_FILE%%.*}`
LOG_NAME=${LOG_LOCATION}${LOG_FILE}.log

SBACKUP_BASE=/mnt/backups_and_builds/git_backup
export LOG_NAME
echo2 () {
	echo -e $@ | awk '{$1=$1;print}' | tee -a ${LOG_NAME}
}

srgx="2[0-9]\{3\}-.*"
rgx="2[0-9]\{3\}\(-[0-9]\{2\}\)\{2\}_[0-9]\{2\}\..*\."

if [ ! -d $LOG_LOCATION ]; then
	mkdir -p $LOG_LOCATION
fi

#Prints acceptable parameters 
helpme() {
        echo
        echo "`basename $0` help page"
        echo "Parameters might be:...and their default values are:"
        echo "----------------------------------------------------"
        echo "\$dryRun"			"Yes"
        echo "\$simpleOption"	"No"
        echo "\$saferOption"	"Yes"
        echo "\$purge"			"7"
        echo
        exit 1
}

#Parse various arguments 
parse_args() {
  while [ $# -gt 0 ] ; do
    echo "PROCESSING ARGs ...: '$1', '$2'"
	if ! [[ "$2" = "Yes" || "$2" = "No" || (-n "$2" && -z "${2//[0-9]}") ]] ; then
		echo "INCORECT values of ARGS" | tee -a ${LOG_NAME}
		helpme		
	fi
    case "$1" in
        -dryRun)   shift; if [ $# -gt 0 ] ; then dryRun=$1; fi;;
        -simpleOption)   shift; if [ $# -gt 0 ] ; then simpleOption=$1; fi;;
        -saferOption)   shift; if [ $# -gt 0 ] ; then saferOption=$1; fi;;
        -purge)   shift; if [ $# -gt 0 ] ; then purge=$1; fi;;
        -h|--help|-?|/?|/h)  echo "DONE.";helpme;
    esac
    shift
  done
  if [[ -n "$purge" && -z "${purge//[0-9]}" ]] ; then
	echo "DONE."
  else 
	echo "INCORECT values of ARGS" | tee -a ${LOG_NAME}
	helpme
  fi
} # parse_args() 

# Process arguments given on the command line. 
parse_args "$@" 

if [ ! -d $LOG_LOCATION ]; then
	mkdir -p $LOG_LOCATION
fi
if [ ! -d $SBACKUP_BASE ]; then
	echo "THERE IS NO BACKUP FOUND"
	exit 1
fi

getBackupInfo() {
	BACKUPS=`ls -r1 $SBACKUP_BASE | grep -o -e "$rgx\(ful\|inc\|corrupt\)$"`
	NO_ALL_BACKUPS=`echo $BACKUPS | wc -w` 
	NO_COR_BACKUPS=`echo $BACKUPS| grep -o -e "\(corrupt\)" | wc -w`
	NO_FUL_BACKUPS=`echo $BACKUPS | grep -o -e "\(ful\)" | wc -w`
	NO_INC_BACKUPS=`expr $NO_ALL_BACKUPS - $NO_COR_BACKUPS - $NO_FUL_BACKUPS`
}

printFooter() {
if [ $dryRun = "No" ] ; then
	getBackupInfo
	echo2 "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
	echo2 "Backups take after purge:" `du -hs $SBACKUP_BASE`", Free space remaining:" `df -h | grep /mnt/backups_ | awk '{print $4}'`
	echo2 -e "#of ALL backups:"$NO_ALL_BACKUPS "#of CORRUPTED:"$NO_COR_BACKUPS "#of FULL:"$NO_FUL_BACKUPS "#of INC:"${NO_INC_BACKUPS}"\n"
fi
}

echo2 "####################################################"
echo2 "This is the backup script logger for"
echo2 "Backup script: $0"
echo  "Logging into: $LOG_NAME" | tee -a $LOG_NAME
echo  "To watch the log live, type this command in other terminal:"
echo  "         tail -f ${LOG_NAME}"
echo2 "Start time:" `date`
echo2 "Executed by $SUDO_USER as $(whoami)"
echo2 "Its PID: $$"
echo2 "===================================================="

getBackupInfo

foundFull=false
countFull=0
counter=0

echo2 -e "\nEntire backup takes:" `du -hs $SBACKUP_BASE`", Free space remaining:" `df -h | grep /mnt/backups_ | awk '{print $4}'`
echo2 "#of ALL backups:"$NO_ALL_BACKUPS "#of CORRUPTED:"$NO_COR_BACKUPS "#of FULL:"$NO_FUL_BACKUPS "#of INC:"$NO_INC_BACKUPS
echo2 "Following is the list of all backups found:"
du -h --max-depth=1 $SBACKUP_BASE >> ${LOG_NAME}
echo2 -e "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n"


## Simple, but dangerous solution
## This will do the job, but if there aren't new backups, we will lost them this way!!!
echo dryRun = $dryRun
if [ $dryRun = "No" ] && [ $simpleOption = "Yes" ] && [ $saferOption = "No" ] ; then
	echo2 "Following backups were detected as older than" $purge "days and shall be deleted with no mercy:"	
	find $SBACKUP_BASE/ -mindepth 1 -prune -regextype posix-basic -regex "$SBACKUP_BASE/$srgx\(ful\|inc\|corrupt\)$" -mtime +$purge -exec bash -c 'echo -n "gained:" `du -hs {} | cut -f1` "by " >> ${LOG_NAME}; rm -vr {} >> ${LOG_NAME}' \;
	echo2 "PURGE has been DONE."
	printFooter
	exit 0
elif [ $dryRun = "Yes" ] && [ $simpleOption = "Yes" ] ; then
	echo2 "Following backups were detected as older than" $purge "days and can be deleted with no mercy:"
	find $SBACKUP_BASE/ -mindepth 1 -prune -regextype posix-basic -regex "$SBACKUP_BASE/$srgx\(ful\|inc\|corrupt\)$" -mtime +$purge -exec bash -c 'echo "run this command to gain `du -hs {} | cut -f1` of space: rm -r {} " >> ${LOG_NAME}' \;
	echo2 -e "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n"
elif [ $simpleOption = "Yes" ] && [ $saferOption = "Yes" ] ; then
	echo "Your arguments are clashing." | tee -a ${LOG_NAME}
	helpme
elif [ $dryRun = "No" ] && [ $simpleOption = "No" ] && [ $saferOption = "No" ] ; then
	echo "Your arguments are clashing 2." | tee -a ${LOG_NAME}
	helpme
fi

## Safer, but a bit more complex solution of deleting old backups. Potential for failures.
if [ $dryRun = "Yes" ] && [ $simpleOption = "No" ] && [ $saferOption = "No" ] ; then
	echo "Nothing to do." | tee -a ${LOG_NAME}
	helpme
fi

for backup in $BACKUPS; do
	# Is it FULL backup?
	echo $backup | grep ful$ > /dev/null 2>&1
	[ "$?" = "0" ] && foundFull=true && countFull=`expr $countFull + 1`

	if [ ! "$foundFull" = true ] ; then
		if ! [[ $dryRun = "Yes" &&  $simpleOption = "Yes" && $saferOption = "No" ]] ; then
			echo2 "+++ Keeping  " `du -hs $SBACKUP_BASE/$backup`" +++"
		fi
	else
		if [ "$countFull" -gt "$NO_FUL_BACKUPS" ] ; then
			echo2 "Sorry, unexpected amount of FULL backups found."
			exit 1
		fi
		if [ "$counter" -ge "1" ] && [ "$countFull" -ge "1" ] ; then
			if [ $dryRun = "Yes" ] && [ $saferOption = "Yes" ] ; then
				echo2 "--- Can delete" `du -hs $SBACKUP_BASE/$backup`" ---"
			elif [ $dryRun = "No" ] && [ $saferOption = "Yes" ] && [ $simpleOption = "No" ] ; then
				echo2 "--- Deleting" `du -hs $SBACKUP_BASE/$backup`" ---"
				rm -rf $SBACKUP_BASE/$backup
			fi
		else
			if [[ ! ( $dryRun = "Yes" && $simpleOption = "Yes" && $saferOption = "No" ) ]] ; then
				echo2 "+++ Keeping  " `du -hs $SBACKUP_BASE/$backup`" +++"
			fi
		fi
		counter=`expr $counter + 1`
	fi
done

printFooter

clean_old_builds.sh

#!/bin/bash
##		    -i for interactive to allow pass the shell aliases to the bash subproces.
# Script origin @ https://gist.github.com/delphym/ad3bbd9d5cafea47bfd66367a0a8cdb6#file-clean_old_builds-sh
# Cleaning the old builds built by Jenkins http://mcs-ubuntu.am.trimblecorp.net:8081
# There is a bug in the app #1770528, so it can't purge old backup snapshots
# https://bugs.launchpad.net/ubuntu/+source/sbackup/+bug/1770528
#set -x

#Default values
dryRun="Yes"		#"Yes|No"
nightlyBuilds="No"	#"Yes|No"
ciBuilds="Yes"		#"Yes|No"
allBuilds="No"		#Yes?|No"
purge=7				#erase all builds older than X days (counting from 24 hrs ago)

BUILD_TYPE="CI"
LOG_LOCATION=/var/log/sbackup/
LOG_FILE=`basename "$0"`
LOG_FILE=`echo ${LOG_FILE%%.*}`
LOG_NAME=${LOG_LOCATION}${LOG_FILE}.log

BUILD_BASE=/mnt/backups_and_builds/builds/Otira
export LOG_NAME
echo2 () {
	echo -e $@ | awk '{$1=$1;print}' | tee -a ${LOG_NAME}
}

NIGHTLY_rgx=".*NIGHTLY.*[0-9]\{3\}.*"
CI_rgx=".*CI.*[0-9]\{3\}.*"
BOTH_rgx=".*\(NIGHTLY\|CI\).*[0-9]\{3\}.*"
ALL_rgx=".*[0-9]\{3\}.*"

if [ ! -d $LOG_LOCATION ] ; then
	mkdir -p $LOG_LOCATION
fi

#Prints acceptable parameters 
helpme() {
        echo
        echo "`basename $0` help page"
        echo "Parameters might be:...and their default values are:"
        echo "----------------------------------------------------"
        echo "\$dryRun"			"Yes"
        echo "\$nightlyBuilds"	"No"
        echo "\$ciBuilds"		"Yes"
        echo "\$allBuilds"		"No"
        echo "\$purge"			"7"
        echo
        exit 1
}

#Parse various arguments 
parse_args() {
  while [[ $# -gt 0 ]] ; do
    echo "PROCESSING ARGs out of $#...: '$1', '$2'"
	KEY=$1; VAL=$2
	
    case "$KEY" in
        -dryRun)   		dryRun=$VAL;;
        -nightlyBuilds)	nightlyBuilds=$VAL;;
        -ciBuilds)   	ciBuilds=$VAL;;
        -allBuilds)   	allBuilds=$VAL;;
        -purge)   		purge=$VAL;;
        -h|--help|-?|/?|/h)  echo "DONE.";helpme;;
		*)	echo "ERROR: unknown parameter \"$KEY\"" | tee -a ${LOG_NAME};helpme;;
    esac

	if ! [[ "$VAL" = "Yes" || "$VAL" = "No" || (-n "$VAL" && -z "${VAL//[0-9]}") ]] ; then
		if [[ "${1}" != @(-h|--help|-?|/?|/h) ]] ; then
			echo "INCORECT values of ARGS: '$KEY' x '$VAL'" | tee -a ${LOG_NAME}
		fi
		helpme		
	fi

    shift; shift
  done
  echo "DONE."
} # parse_args() 

# Process arguments given on the command line. 
parse_args "$@" 

if [ ! -d $LOG_LOCATION ] ; then
	mkdir -p $LOG_LOCATION
fi
if [ ! -d $BUILD_BASE ] ; then
	echo "THERE IS NO build FOUND"
	exit 1
fi

getBuildInfo() {
	BUILDS=`find $BUILD_BASE -mindepth 1 -type d -regextype posix-basic -regex "$BUILD_BASE/.*[0-9]\{3\}.*"`
	NO_ALL_BUILDS=`echo $BUILDS | wc -w` 
	NO_CI_BUILDS=`echo $BUILDS | grep -o -e "CI" | wc -w`
	NO_NIGHTLY_BUILDS=`echo $BUILDS | grep -o -e "NIGHTLY" | wc -w`
	NO_UNK_BUILDS=`expr $NO_ALL_BUILDS - $NO_NIGHTLY_BUILDS - $NO_CI_BUILDS` 
}

printFooter() {
	getBuildInfo
	echo2 "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
	echo2 "Builds occupy after purge:" `du -hs $BUILD_BASE`", Free space remaining:" `df -h | grep /mnt/backups_ | awk '{print $4}'`
	echo2 -e "#of ALL builds:$NO_ALL_BUILDS #of CI:$NO_CI_BUILDS #of NIGHTLY:$NO_NIGHTLY_BUILDS #of OLD:${NO_UNK_BUILDS}\n"
}

echo2 "####################################################"
echo2 "This is the build clean up script logger for"
echo2 "build clean script: $0"
echo  "Logging into: $LOG_NAME" | tee -a $LOG_NAME
echo  "To watch the log live, type this command in other terminal:"
echo  "         tail -f ${LOG_NAME}"
echo2 "Start time:" `date`
echo2 "Executed by $SUDO_USER as $(whoami)"
echo2 "Its PID: $$"
echo2 "===================================================="

getBuildInfo

echo2 -e "\nAll archived builds occupy" `du -hs $BUILD_BASE`", Free space remaining:" `df -h | grep /mnt/backups_ | awk '{print $4}'`
echo2 -e "#of ALL builds:$NO_ALL_BUILDS #of CI:$NO_CI_BUILDS #of NIGHTLY:$NO_NIGHTLY_BUILDS #of OLD:$NO_UNK_BUILDS\n"
echo2    "Following is the list of all builds found:"
du -h --max-depth=2 $BUILD_BASE >> ${LOG_NAME}
echo2 -e "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n"

# Deciding what type of builds to delete
if [ $allBuilds = "Yes" ] ; then
	BUILD_TYPE="ALL"; rgx=$BUILD_BASE/$ALL_rgx
elif [ $nightlyBuilds = "Yes" ] && [ $ciBuilds = "No" ] ; then
	BUILD_TYPE="NIGHTLY"; rgx=$BUILD_BASE/$NIGHTLY_rgx
elif [ $nightlyBuilds = "No" ] && [ $ciBuilds = "Yes" ] ; then
	BUILD_TYPE="CI"; rgx=$BUILD_BASE/$CI_rgx
elif [ $nightlyBuilds = "Yes" ] && [ $ciBuilds = "Yes" ] ; then
	BUILD_TYPE="CI & NIGHTLY"; rgx=$BUILD_BASE/$BOTH_rgx
elif [ $nightlyBuilds = "No" ] && [ $ciBuilds = "No" ] && [ $allBuilds = "No" ] ; then
	echo "Your arguments make not sense." | tee -a ${LOG_NAME}
	helpme
fi

echo -e "dryRun = $dryRun\nrgx = $rgx"
## This will only tell you what can build artefacts can be removed
if [ $dryRun = "Yes" ] ; then
	echo2 "Following $BUILD_TYPE builds were detected as older than" $purge "days and can be deleted with no mercy:"
	find $BUILD_BASE/ -mindepth 1 -type d -regextype posix-basic -regex "$rgx" -mtime +$purge -exec bash -c 'echo "run this command to gain `du -hs {} | cut -f1` of space: rm -r {} " >> ${LOG_NAME}' \;
	echo2 -e "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n"
fi

## This will remove build artefacts
if [ $dryRun = "No" ] ; then
	echo2 "Following $BUILD_TYPE builds were detected as older than" $purge "days and shall be deleted with no mercy:"	
	find $BUILD_BASE/ -mindepth 1 -type d -regextype posix-basic -regex "$rgx" -mtime +$purge -exec bash -c 'echo "gained:" `du -hs {} | cut -f1` "by removing ‘`echo {}`’." >> ${LOG_NAME}; rm -vr {}' \;
	echo2 "PURGE has been DONE."
	printFooter
	exit 0
fi