borg wrapper + systemd definition

.gitignore

/borg_config.py

borg.service

[Unit]
Description=borg backup

Wants=network-online.target
Wants=local-fs.target
After=network-online.target
After=local-fs.target

[Service]
User=borg
AmbientCapabilities=CAP_DAC_READ_SEARCH
ExecStart=/home/borg/borg-wrapper/borg_wrapper.py
Restart=always
RestartSec=10m

[Install]
WantedBy=multi-user.target

borg_wrapper.py

#!/usr/bin/env python3

import collections
import datetime
import email.utils
import os
import platform
import subprocess
import sys
import time


# You *need* to override these...
HOSTS = ('mynas.lan', 'mynas.external.tld')
REPO = '/srv/borg/myhost'
BORG_PASSPHRASE = 'A real good one, some 42 chars or whatever'

# Modify these at will...
PING_LIMIT = 30
SHARES = ('/etc', '/home')
EXCLUDES = (
    '-e', '**/nobackup',
    '-e', '/home/*/.cache',
    '-e', '/home/*/.npm',
    '-e', '/home/*/.mozilla',
    '-e', '/home/*/.steam',
    '-e', '/home/*/Downloads',
)

# Remember to make borg_config.py with HOSTS, REPO and BORG_PASSPHRASE set.
from borg_config import *


# A bit of a hack. CLOCK_BOOTTIME exists as of Linux 2.6.39, so the system call
# will succeed even though the symbolic value isn't defined until python >=3.7
timer_id = getattr(time, 'CLOCK_BOOTTIME', 7)


def setup_env():
    os.environ['BORG_PASSPHRASE'] = BORG_PASSPHRASE
    os.environ['BORG_RELOCATED_REPO_ACCESS_IS_OK'] = 'yes'

_log_lines = collections.deque(maxlen=10)

def log(*args, **kwargs):
    _log_lines.append(' '.join(args))
    kwargs.setdefault('file', sys.stderr)
    kwargs.setdefault('flush', True)
    print(*args, **kwargs)

def run(*cmd):
    return subprocess.check_call(cmd)

def sleep_for_mins(n_mins):
    now = time.clock_gettime(timer_id)
    then = now + (n_mins * 60)
    then_text = (datetime.datetime.now() + datetime.timedelta(minutes=n_mins)
            ).isoformat()
    log('Waiting for %d minutes (until >= ~%s)' % (n_mins, then_text))
    while now < then:
        time.sleep(60)
        now = time.clock_gettime(timer_id)

def backup(host):
    run('borg', 'create',
            '--stats',
            *EXCLUDES,
            '--exclude-caches',
            '--exclude-if-present', '.nobackup',
            '--one-file-system',
            '--checkpoint-interval', '300',
            '--compression', 'zstd,1',
            '%s:%s::{now}' % (host, REPO),
            *SHARES)

def prune(host):
    run('borg', 'prune',
            '--stats',
            '-d', '7',
            '-w', '4',
            '-m', '8',
            '-y', '3',
            '%s:%s' % (host, REPO))

def check(host):
    run('borg', 'check', '%s:%s' % (host, REPO))

def get_host():
    for host in HOSTS:
        log('Pinging ' + host)
        ping = trace_ping(host)
        if ping is None:
            log('No ping result for backup host ' + host)
        elif ping > PING_LIMIT:
            log('Ping %.1f for %s exceeds limit of %d' % (ping, host, PING_LIMIT))
        else:
            log('Successful ping at %.1f ms against %s' % (ping, host))
            return host

def try_op(op):
    host = get_host()
    while not host:
        sleep_for_mins(5)
        host = get_host()

    log('Performing %s against %s' % (op.__name__, host))
    try:
        op(host)
    except subprocess.CalledProcessError as err:
        log('Error(s) during %s, exit code %d' % (op.__name__, err.returncode))
        sendmail = subprocess.Popen(['/usr/sbin/sendmail', '-ti'], stdin=subprocess.PIPE)
        mail = '''To: root
From: borg
Subject: Error during borg %(op)s on %(hostname)s
Date: %(date)s

One or more error occurred running %(op)s against host %(host)s, check journal on %(hostname)s.

Last 10 log entries below.

%(log)s''' % {
    'op': op.__name__,
    'hostname': platform.node(),
    'date': email.utils.formatdate(localtime=True),
    'host': host,
    'log': '\n'.join(_log_lines),
}
        sendmail.communicate(mail.encode('utf-8'))
        sendmail.wait()
        sleep_for_mins(5)
    else:
        log('Successful ' + op.__name__)
        # 12 hours
        if op == backup:
            sleep_for_mins(12 * 60)
        else:
            sleep_for_mins(5)

def trace_ping(host):
    try:
        out = subprocess.check_output(['/usr/bin/mtr', '-rn', host])
    except subprocess.CalledProcessError:
        return None

    for line in reversed(out.splitlines()):
        parts = line.split()
        if parts[1] != b'???':
            return float(parts[5])
    return None

def main():
    log('Starting backup runner')
    counter = 0
    while True:
        # At startup, and every 11 runs
        if (counter % 11) == 0:
            try_op(check)

        # After first backup, then every 5 runs
        if (counter % 5) == 1:
            try_op(prune)

        try_op(backup)

        counter += 1

if __name__ == '__main__':
    setup_env()
    if len(sys.argv) > 1:
        op = sys.argv[1]
        if not op in ('backup', 'check', 'prune') or len(sys.argv) != 3:
            print('Usage: %s [{backup,check,prune} host]' % sys.argv[0])
        else:
            locals()[op](sys.argv[2])
    else:
        main()

check-is-current.sh

#!/bin/bash

remote=(`git ls-remote -h origin master 2>/dev/null`)
[[ $? -ne 0 ]] && exit 0
head=`git rev-parse HEAD`
[[ "${remote[0]}" = "$head" ]] || echo "HEAD ($head) != origin/master (${remote[*]})"

install-borg.sh

#!/bin/bash

set -e
# NB: If "-e" is removed, all errors are ignored and execution continues.

# Usage: Download and run as root. Url:
# https://gist.githubusercontent.com/xim/91c18bebafb9db5787a1b3b554986706/raw/install-borg.sh

BORG_HOME="/home/borg"

log() {
	echo "$@"
}

fail() {
	echo "$@" >&2
	return 1
}

prompt_data() {
	echo "Please provide a passphrase (make it, like, >= 42 ascii chars)"
	read -p "> " line
	export BORG_PASSPHRASE="$line"
	echo "Prioritized list of server hostnames (space separated)"
	read -p "> " line
	BORG_SERVERS=($line)
	echo "Repo path on server (e.g. /srv/borg/$HOSTNAME)"
	read -p "> " line
	BORG_REPO="$line"
}

log "Checking environment"

[[ `id -u` -eq 0 ]] || fail "Run this script as root"
for pkg in borgbackup exim4-daemon-light git mtr ; do
	# TODO: Consider making non-debian-specific tests...
	dpkg -l "$pkg" >/dev/null
done
getent passwd borg 2>/dev/null && fail "user borg exists"
for file in /etc/systemd/system/borg.service "$BORG_HOME" /etc/cron.hourly/borg-wrapper-check ;do
	test -e "$file" && fail "$file already exists"
done

log "Adding user"
adduser --system --home "$BORG_HOME" --shell /bin/bash --gecos 'Borg backup,,,' --disabled-password borg
chmod 700 "$BORG_HOME"

cd "$BORG_HOME"

log "Cloning repo"
git clone 'https://gist.github.com/91c18bebafb9db5787a1b3b554986706.git' borg-wrapper

log "Installing systemd service"
cp borg-wrapper/borg.service /etc/systemd/system/

log "Installing cron update check"

cat >/etc/cron.hourly/borg-wrapper-check <<EOF
#!/bin/bash -e

cd "$BORG_HOME/borg-wrapper"

exec ./check-is-current.sh
EOF
chmod +x /etc/cron.hourly/borg-wrapper-check

log "To complete, you need to answer some questions..."
prompt_data

log "Creating SSH key"
sudo -u borg ssh-keygen -t ed25519 -N '' -f "$BORG_HOME/.ssh/id_ed25519"
echo "Please add this line in ~borg/.ssh/authorized_keys on $BORG_SERVERS:"
echo -n "command=\"/usr/bin/borg serve --restrict-to-path $BORG_REPO\",restrict "
cat "$BORG_HOME/.ssh/id_ed25519.pub"
read -p '-- Press enter when you have done so --' line

log "Creating repo..."
while ! sudo -E -H -u borg borg init -e repokey-blake2 "$BORG_SERVERS:$BORG_REPO" ;do
	echo "Couldn't create repo. Probably messed up the ssh key stuff, fix it plx"
	read -p '-- Press enter to retry --' line
done

log "Verifying ssh login for all hostnames..."
for server in "${BORG_SERVERS[@]}" ;do
	while ! sudo -u borg borg with-lock "$server:$BORG_REPO" true ;do
		echo "Couldn't verify keyless ssh to '$server' -- Please correct and verify the situation..."
		read -p 'Retry? [Yn] ' line
		[[ "$line" =~ ^n ]] && break
	done
done

{
	echo "HOSTS = ("
	for server in "${BORG_SERVERS[@]}" ; do
		echo "    '$server',"
	done
	echo ")"
	echo "REPO = '$BORG_REPO'"
	echo "BORG_PASSPHRASE = '''$BORG_PASSPHRASE'''"
} > "$BORG_HOME/borg-wrapper/borg_config.py"

echo "Okay, we're pretty much done now. You can edit your config at $BORG_HOME/borg-wrapper/borg_config.py"
echo "Look over the default paths and excludes in particular, and see if you want to extend that."
read -p "Do vim \"$BORG_HOME\"/borg-wrapper/borg_{wrapper,config}.py now? [Yn] " line
[[ "$line" =~ ^n ]] || vim "$BORG_HOME"/borg-wrapper/borg_{wrapper,config}.py

systemctl enable borg
systemctl start borg

log "All bases loaded, you are good to go!"
log "Start a journalctl now to let you see it everything is running as expected:"

echo -e "\n  exec journalctl -xfu borg"