google cloud snapshot to borg backup migration script

borgify.sh

#!/bin/bash
###############################################################################
# borgify.sh - google cloud snapshot to borg backup migration script          #
###############################################################################
#
# for every snapshot of SOURCE_DISK,
# 1. create a disk from it
# 2. mount it on a server
# 3. back up TARGET_DIR to RSYNC_LOGIN/~/REPO
# 4. unmount and delete the disk
# rinse, repeat
# optionally delete the snapshots
# to use this script, you must have gcloud installed and configured.
#
# ###################
# #  CONFIGURATION  #
# ###################
#
# .env must assign the following variables or they must be set manually
#
# RSYNC_LOGIN       # the user@xx-s###.rsync.net
#                   # (or other ssh/borg compatible server)
# REPO              # the name of the borg repo to create/target
# KEYFILE           # keyfile to use for encryption.  create new if empty
# SOURCE_DISK       # the name of the source disk of the snapshots
# MAKE_MACHINE      # (optional) create the GCE instance first
# SNAP_PREFIX       # (optional) any prefix to be stripped from snapshot names
#                   # note: snapshots must alphanumerically sort to the
#                   # chronological order, e.g. my-disk-YYYYMMDDHHMMSS
# TARGET_DIR        # (optional) subdirector(y/ies) of the snapshot to back up
# DELETE_SNAPSHOTS  # (optional) DANGER! delete snapshots after migration
# TEST1             # (optional) just do the first snapshot
#
###############################################################################

reset_verbosity () {
    if [[ -z $GCLOUD_VERBOSITY ]]; then
        gcloud config unset core/verbosity
    else
        gcloud config set core/verbosity $GCLOUD_VERBOSITY
    fi
}

exec 1> >(logger -s -t $(basename $0)) 2>&1
trap '{
code=$?
echo "caught error code $code"
echo -n "deleting instance borgus..."
gcloud -q compute instances delete borgus
echo "done."
echo "deleting remote repo $REPO"
ssh $RSYNC_LOGIN "rm -rf $REPO"
echo -n "deleting temporary disk..."
gcloud -q compute disks delete borgus-snapshot-temp
echo "done."
reset_verbosity
exit $code
}' ERR

if [[ -f .env ]]; then source ".env"; fi

# danger!
DELETE_SNAPSHOTS=${DELETE_SNAPSHOTS:-"false"}

# some default values
TARGET_DIR=${TARGET_DIR:-'.'}
KEYFILE=${KEYFILE:-"$(basename $REPO).key"}

# lower gcloud's default verbosity for this session
GCLOUD_VERBOSITY=$(gcloud config get-value core/verbosity)
gcloud config set core/verbosity error

if [[ $MAKE_MACHINE = true ]]; then
    echo "creating instance borgus..."
    gcloud compute instances create \
        --zone=europe-west1-b \
        --image-project ubuntu-os-cloud \
        --image-family ubuntu-1804-lts \
        borgus
    addr=$(gcloud compute instances describe borgus \
        --format 'value(networkInterfaces[0].accessConfigs[0].natIP)')
    echo "waiting for ssh..."
    until nc -z $addr 22; do sleep 5; done
    echo -n "setting up borgus: directories, "
    gcloud -q compute ssh ubuntu@borgus --command "mkdir data"
    echo -n "packages, "
    gcloud -q compute ssh ubuntu@borgus \
        --command "until sudo apt-get -q update && \
        sudo apt-get -q install -y borgbackup; do sleep 5; done 2>&1 > /dev/null"
    echo -n "ssh, "
    gcloud -q compute ssh ubuntu@borgus --command \
        "ssh-keygen -N '' -f .ssh/id_rsa && cat .ssh/id_rsa.pub" | \
        ssh -q -oStrictHostKeyChecking=no $RSYNC_LOGIN \
        'dd of=.ssh/authorized_keys oflag=append conv=notrunc'
    gcloud -q compute ssh ubuntu@borgus --command \
        "ssh -q -oStrictHostKeyChecking=no $RSYNC_LOGIN touch ."
    echo "borg repo, "
    if [[ -f $KEYFILE ]]; then
        gcloud compute scp $KEYFILE ubuntu@borgus:.
    fi
    gcloud -q compute ssh ubuntu@borgus \
        --command "BORG_PASSPHRASE= BORG_KEY_FILE=$KEYFILE \
        borg --remote-path=borg1 init -e keyfile \
        ssh://$RSYNC_LOGIN/~/$REPO"
    if [[ ! -f $KEYFILE ]]; then
        gcloud -q compute ssh ubuntu@borgus --command \
            "keyfile=$(tempfile)
        borg --remote-path=borg1 key export \
            ssh://$RSYNC_LOGIN/~/$REPO $keyfile && \
            cat $keyfile && \
            rm $keyfile" > $REPO.key
    fi
    echo "done"
else
    gcloud compute instances start borgus
fi

echo -n "getting snapshots..."
snaps=$(gcloud compute snapshots list \
    --filter "sourceDisk = '$SOURCE_DISK'" \
    --format '[no-heading](name)' | sort)
echo "got $(wc -w <<< $snaps) snapshots"

for snap in $snaps; do
    size=$(gcloud compute snapshots describe $snap \
        --format 'value[no-heading](diskSizeGb)')
    echo "creating disk from snapshot $snap..."
    disk_name="borgus-snapshot-temp"
    time gcloud compute disks create $disk_name \
        --size $size \
        --source-snapshot $snap \
        --type pd-ssd
    echo "attaching disk to borgus..."
    gcloud compute instances attach-disk borgus \
        --disk $disk_name \
        --device-name $disk_name
    echo "mounting disk on ~/data..."
    gcloud compute ssh ubuntu@borgus \
        --command "sudo mount /dev/disk/by-id/google-$disk_name-part1 data"
    archive=$(echo $snap | sed -e "s/^$SNAP_PREFIX//")
    echo "backing up $TARGET_DIR to borg" \
        "($RSYNC_LOGIN/~/$REPO::$archive)..."
    gcloud compute ssh ubuntu@borgus --command "
    cd data
    BORG_KEY_FILE=/home/ubuntu/$KEYFILE borg --remote-path borg1 create -s \
        ssh://$RSYNC_LOGIN/~/samuel_hall::$archive $TARGET_DIR
    "
    echo "done."
    echo "unmounting disk..."
    gcloud compute ssh ubuntu@borgus --command "sudo umount data"
    echo "detaching disk..."
    gcloud compute instances detach-disk borgus --disk borgus-snapshot-temp
    echo "deleting disk..."
    gcloud --quiet compute disks delete borgus-snapshot-temp
    [[ $TEST1 = true ]] && break
done

gcloud compute instances stop borgus

if [[ $DELETE_SNAPSHOTS = "true" ]]; then
    read -p "are you sure you want to delete all snapshots [No]? " confirm
    if [[ $confirm != "yes" ]]; then exit 1; fi
    for snap in $(IFS=" " sort -r <<< $snaps); do
        gcloud --quiet compute snapshots delete $snap
    done
fi

reset_verbosity