Google Cloud Storage Setup

gsutil_backup_setup.sh

#!/bin/bash

#this setup is based on https://scotthelme.co.uk/easy-cheap-and-secure-backup-with-google-cloud-platform

LOG_DIR=~/logs
LOCK_FILE=~/gsutil_backup_cron.lock

#everything under /data (excluding data) will be put into the target gs bucket
SOURCE_DIR=/data

TARGET_BUCKET=gs://your.google_storage.bucket/path

#add -x "exclude_pattern" to EXTRA_RSYNC_OPTIONS to exclude certain patterns
#add -d to EXTRA_RSYNC_OPTIONS to delete or in other words keep source and target in sync
EXTRA_RSYNC_OPTIONS=
#EXTRA_RSYNC_OPTIONS="-d -x \"abcd\""

export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)"

echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | sudo tee /etc/apt/sources.list.d/google-cloud-sdk.list

curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -

sudo apt-get update && sudo apt-get install google-cloud-sdk

gcloud init

mkdir $LOG_DIR

#remove -m for home usage with limited network bandwidth
(crontab -l 2>/dev/null; echo "* * * * * /usr/bin/flock -w 0 $LOCK_FILE trickle -d 100 -u 500 /usr/bin/gsutil -m rsync $EXTRA_RSYNC_OPTIONS -r $SOURCE_DIR $TARGET_BUCKET >> $LOG_DIR 2>&1") | crontab -