gk_clean.sh

# =============================================================================
# ON THE SERVER...

# Check the amount of disk space available on root (/) partition
df -h

# See where space is being consumed
cd /; sudo du -h --max-depth=1

# In this case, most was in /emergence; further digging and du-ing
# revealed three locations:
# - /emergence/services/data/mysql
# - /emergence/sites/gatekeeper/logs
# - /emergence/sql-backups/gatekeeper

# Add ubuntu to the mysql group so that it can read and download what
# it needs to
sudo usermod -a -G mysql ubuntu



# =============================================================================
# ON A LOCAL MACHINE...

# Create a folder for the old mysql data files
mkdir -p emergence/services/data/mysql

# Copy the mysqld data from the server. Oldest (000001) is from Sep 21 2015;
# newest (000015) is from May 15 2015. Leaving newest on the server as is.
# Each file is ~1G.
scp -i ~/.ssh/phl-gatekeeper.pem \
ubuntu@api.phila.cov:/emergence/services/data/mysql/mysqld-bin.000001 \
ubuntu@api.phila.cov:/emergence/services/data/mysql/mysqld-bin.000002 \
ubuntu@api.phila.cov:/emergence/services/data/mysql/mysqld-bin.000003 \
ubuntu@api.phila.cov:/emergence/services/data/mysql/mysqld-bin.000004 \
ubuntu@api.phila.cov:/emergence/services/data/mysql/mysqld-bin.000005 \
ubuntu@api.phila.cov:/emergence/services/data/mysql/mysqld-bin.000006 \
ubuntu@api.phila.cov:/emergence/services/data/mysql/mysqld-bin.000007 \
ubuntu@api.phila.cov:/emergence/services/data/mysql/mysqld-bin.000008 \
ubuntu@api.phila.cov:/emergence/services/data/mysql/mysqld-bin.000009 \
ubuntu@api.phila.cov:/emergence/services/data/mysql/mysqld-bin.000010 \
ubuntu@api.phila.cov:/emergence/services/data/mysql/mysqld-bin.000011 \
ubuntu@api.phila.cov:/emergence/services/data/mysql/mysqld-bin.000012 \
ubuntu@api.phila.cov:/emergence/services/data/mysql/mysqld-bin.000013 \
ubuntu@api.phila.cov:/emergence/services/data/mysql/mysqld-bin.000014 \
emergence/services/data/mysql/

# Remove the data files -- about 13G
ssh -i ~/.ssh/phl-gatekeeper.pem ubuntu@api.phila.cov sudo rm \
/emergence/services/data/mysql/mysqld-bin.000001 \
/emergence/services/data/mysql/mysqld-bin.000002 \
/emergence/services/data/mysql/mysqld-bin.000003 \
/emergence/services/data/mysql/mysqld-bin.000004 \
/emergence/services/data/mysql/mysqld-bin.000005 \
/emergence/services/data/mysql/mysqld-bin.000006 \
/emergence/services/data/mysql/mysqld-bin.000007 \
/emergence/services/data/mysql/mysqld-bin.000008 \
/emergence/services/data/mysql/mysqld-bin.000009 \
/emergence/services/data/mysql/mysqld-bin.000010 \
/emergence/services/data/mysql/mysqld-bin.000011 \
/emergence/services/data/mysql/mysqld-bin.000012 \
/emergence/services/data/mysql/mysqld-bin.000013 \
/emergence/services/data/mysql/mysqld-bin.000014 \

# Update the index
ssh -i ~/.ssh/phl-gatekeeper.pem ubuntu@api.phila.cov "echo './mysqld-bin.000015' | tee /emergence/services/data/mysql/mysqld-bin.index"



# Create a folder for the GateKeeper access logs
mkdir -p emergence/sites/gatekeeper/logs/

# Download the gatekeeper access log file -- about 6.7G. There's only one of
# these. Should be rotated better.
scp -i ~/.ssh/phl-gatekeeper.pem ubuntu@api.phila.cov:/emergence/sites/gatekeeper/logs/access.log emergence/sites/gatekeeper/logs/

# Keep the last 1000 lines of the access log
ssh -i ~/.ssh/phl-gatekeeper.pem ubuntu@api.phila.cov "sudo tail -n 1000 /emergence/sites/gatekeeper/logs/access.log | tee /emergence/sites/gatekeeper/logs/access.log > /dev/null"




# Create a folder for SQL backups
mkdir -p emergence/sql-backups/gatekeeper

# Copy all but the most recent SQL backups. These have been daily since
# March 31 2016, monthly before that. Did something change?
scp -i ~/.ssh/phl-gatekeeper.pem \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2014-11-01.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2014-12-01.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2015-01-01.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2015-02-01.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2015-03-01.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2015-04-01.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2015-04-29-premigrate.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2015-05-01.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2015-06-01.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2015-07-01.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2015-08-01.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2015-09-01.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2015-10-01.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2015-11-01.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2015-12-01.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2016-01-01.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2016-02-01.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2016-03-31.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2016-04-01.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2016-04-20.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2016-04-24.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2016-04-25.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2016-04-26.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2016-04-27.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2016-04-28.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2016-04-29.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2016-04-30.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2016-05-01.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2016-05-02.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2016-05-03.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2016-05-04.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2016-05-05.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2016-05-06.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2016-05-07.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2016-05-08.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2016-05-09.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2016-05-10.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2016-05-11.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2016-05-12.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2016-05-13.sql.bz2 \
ubuntu@api.phila.cov:/emergence/sql-backups/gatekeeper/gatekeeper.2016-05-14.sql.bz2 \
emergence/sql-backups/gatekeeper

# Remove all but the most recent SQL backups -- ~19G
ssh -i ~/.ssh/phl-gatekeeper.pem ubuntu@api.phila.cov sudo rm \
/emergence/sql-backups/gatekeeper/gatekeeper.2014-11-01.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2014-12-01.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2015-01-01.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2015-02-01.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2015-03-01.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2015-04-01.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2015-04-29-premigrate.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2015-05-01.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2015-06-01.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2015-07-01.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2015-08-01.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2015-09-01.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2015-10-01.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2015-11-01.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2015-12-01.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2016-01-01.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2016-02-01.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2016-03-31.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2016-04-01.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2016-04-20.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2016-04-24.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2016-04-25.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2016-04-26.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2016-04-27.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2016-04-28.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2016-04-29.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2016-04-30.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2016-05-01.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2016-05-02.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2016-05-03.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2016-05-04.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2016-05-05.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2016-05-06.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2016-05-07.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2016-05-08.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2016-05-09.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2016-05-10.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2016-05-11.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2016-05-12.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2016-05-13.sql.bz2 \
/emergence/sql-backups/gatekeeper/gatekeeper.2016-05-14.sql.bz2 \