restic.sh - restic wrapper script with 1password support (and fallback on files if not available). Support for password/exclude and env.

restic.sh

#!/usr/bin/env bash

# The 1password entries are expected like this: (imagine a plex server from which we want to backup /etc - repository "etc" - and /var/lib/plexmediaserver - repository "library"):
# [plex]            <- 1password section, indicating the host/device
# password          <- host global password entry (used for all repo's on this host unless overwritten)
# backup            <- arguments when the backup subcommand is used, for all repositories under this host (for example "--host plex")
# env               <- env for the host (defined before the repo specific one). KEY=VALUE
# forget            <- arguments when the forget subcommand is used, for all repositories under this host (eg. "--prune --keep-daily=30")
# etc/password      <- dedicated password for "etc" repository
# etc/exclude       <- text field that contains the contents for the exclude-file when backing up
# etc/env           <- repo specific env settings (eg BACKUP_DIRECTORIES="/etc")
# etc/backup        <- etc-specific extra options for the backup subcommand (for example "--tag etc")
# etc/forget        <- etc-specific extra options for the forget subcommand
# library/password  <- password for the library repository
# library/env       <- env settings for the library repository (eg BACKUP_DIRECTORIES="/var/lib/plexmediaserver")

show_help() {
  cat <<EOHELP
Usage: ${0##*/} [--script-help] [--no-op] [--debug] [cron|do_backup] [options to pass to restic...]

Requirements
  This script requires "op" (1password CLI - https://developer.1password.com/docs/cli/get-started/#install) - but it does have a "--no-op"-on-disk-fallback
  and  "jq" (JSON processor). In synology it can be installed via entware.

Restic Repositories
  Every repository is identified by a "device" (host, technology, ...) and within the device an identifier.
  For example device = my-desktop, identifier = etc (for the /etc on your deskop) or device = backblaze, identifier = photos 
  (for a restic repository containing photo's on backblaze). Note that you would have to set the repository manually for this one 
  as is would be - for example - b2:photos for the bucket called photos.
  
  Disk repositories will end up in: <this scripts dir>/../../BACKUP_DEVICE_TYPE/BACKUP_DEVICE/BACKUP_IDENTIFIER unless
  RESTIC_REPOSITORY is explicitly specified.

.env support :
  If the folder where this script resides contains a .env file, it will be loaded. (handy for the globals)
  If the working folder contains a .env script, it will be loaded as well
  If you don't want to use .envs, you can just set the variables in your own shell script and then source this one.

Variables
  Globally:
    OP_SERVICE_ACCOUNT_TOKEN  The service account token (to be created in your 1password web interface)
    ONEPASSWORD_VAULT         The vault to use to obtain the restic 1password item
    ONEPASSWORD_ITEM_ID       The 1password restic item ID

  Per host/device and type:
    RESTIC_PASSWORD     You can set this directly, but why are you using this script then? :-)
    RESTIC_REPOSITORY   If set, this is used as the target repository. BACKUP_DEVICE_TYPE is ignored as it is for a physical path only 
    BACKUP_DEVICE_TYPE  assumed "host" if empty (mainly to determine where the repository is stored)
    BACKUP_DEVICE       hostname, device name, technology (basically the first level name) for the target repostory
    BACKUP_IDENTIFIER   within the device, the sub-directory (called identifier, can be like "etc" or "home", "home/steve" or whatever you want)
    BACKUP_DIRECTORY    Single directory to back up, script will cd to the directory and use "." as backup directory
    BACKUP_DIRECTORIES  Multiple directories to include (or a single but don't mess with the cwd and the specified directories)
                        If neither is specified, the current working directory will be backed up

1Password
  The item is expected to have a BACKUP_DEVICE-section with a "BACKUP_IDENTIFIER/password" entry 
  (or "password" to use the same for all the repositories within this device)
  It can also contain a "BACKUP_IDENTIFIER/exclude"-text entry that is added as --exclude-file during backups
  Lastly it can have a "BACKUP_IDENTIFIER/env" (or "env") text entry that contains KEY=VALUE lines to include in the environment prior to action.
  If 1password cli is not available it will look for a BACKUP_DEVICE_TYPE/BACKUP_DEVICE directory under the script location and use that (same sub-paths as the items in 1password)
  Finally, extra arguments for backup and prune can be specified the same way by means of a "BACKUP_IDENTIFIER/command" or "command" entry (command = backup|forget)

Exit codes:
  0: all ok
  1: Restic fatal error
  3: Restic could not read some files (snapshot may be incomplete)
  50: Problem with 1password CLI
  51: Missing required configuration value
  52: Unable to find expected value in the vault
  
The script outputs all its blabbing to STDERR as to not get in the way of any parsing attempts.
EOHELP
}

SCRIPT_ARGUMENTS=()
LOG_VERBOSE=1
OP_CACHED_ITEM=""
RESTIC_COMMAND=${RESTIC_COMMAND:-}
RESTIC_GLOBAL_OPTIONS=${RESTIC_GLOBAL_OPTIONS:---verbose}
RESTIC_COMMAND_OPTIONS=${RESTIC_COMMAND_OPTIONS:-}

# Where are we? 
# We use this as the base instead of a fixed NAS location, so it can be included on every device
# SCRIPT_DIR="$(dirname \"$(readlink -f \"$0\")\")"
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )

log() {
    local log_level="$1" # Log level (INFO, WARN, ERROR)
    local log_message="$2" # Log message

    # Check verbosity before logging
    if [[ "$LOG_VERBOSE" -eq 1 || "$log_level" == "ERROR" ]]; then
        # Write to stderr with timestamp and level
        echo -e "$(date +'%Y-%m-%d %H:%M:%S') [$log_level] $log_message" >&2
    fi
}

dotenv="$SCRIPT_DIR/.env"
if [[ -f "$dotenv" ]]; then
    log "INFO" "Loading \"$dotenv\"..."
    set -a
    source "$SCRIPT_DIR/.env"
    set +a
fi

local_dotenv=$(pwd)/.env
if [[ "$dotenv" != "$local_dotenv" ]] &&  [[ -f "$local_dotenv" ]]; then
    log "INFO" "Loading \"$local_dotenv\"..."
    set -a
    source "$local_dotenv"
    set +a
fi

main() {
  # If we have a backup environment file specified
  if [ ! -z "${BACKUP_ENV}" ]; then
    log "INFO" "Loading \$BACKUP_ENV..."
    # Source it. Might contain setup.
    set -a
    source "${BACKUP_ENV}"
    set +a
  fi  

  # By default the "root" backup location is "host". 
  # 99.99% of the backups we make are from this type, but allow override nonetheless.
  device_type="${BACKUP_DEVICE_TYPE:-host}"

  # We do need a device identifier to determine where to store the backup
  if [ -z "${BACKUP_DEVICE}" ]; then
    log "ERROR" "Please set BACKUP_DEVICE."
    exit 51
  fi

  # Within the device, we need a backup identifier
  if [ -z "${BACKUP_IDENTIFIER}" ]; then
    log "ERROR" "Please set BACKUP_IDENTIFIER."
    exit 51
  fi

  # The script is in the restice meta directory
  restic_script_dir=${SCRIPT_DIR}
  # The actual .meta directory is one level higher
  meta_dir="$(dirname "$restic_script_dir")"
  # The repositories-root is one higher still.
  repositories_root="$(dirname "${meta_dir}")"
  # Where do we create/find the repository (if on disk)?
  repository_dir=${repositories_root}/${device_type}/${BACKUP_DEVICE}/${BACKUP_IDENTIFIER}
  

  if [ -z "$RESTIC_PASSWORD" ]; then
    # Does the device/identifier have its own password?
    restic_password=$(get_value ${BACKUP_DEVICE} ${BACKUP_IDENTIFIER}/password ${device_type})
    if [[ $? -ne 0 ]]; then
      # No password, perhaps there is a "device global" password entry?
      restic_password=$(get_value ${BACKUP_DEVICE} password ${device_type})
      if [[ $? -ne 0 ]]; then
        log "ERROR" "No password found. Expected a section \"${BACKUP_DEVICE}\" in 1Password with either a \"${BACKUP_IDENTIFIER}/password\" or just \"password\" entry."
        log "ERROR" "echo When using files, \"${SCRIPT_DIR}/${device_type}/${BACKUP_DEVICE}\" with either a \"password\" or \"${BACKUP_IDENTIFIER}/password\" file."
        exit 52
      fi
    fi

    # Help out restic a bit
    export RESTIC_PASSWORD=${restic_password}
  fi

  # Environment set "device-globally"?
  restic_environment=$(get_value ${BACKUP_DEVICE} env ${device_type})
  if [ -n "$restic_environment" ]; then
    # Set ENV
    log "INFO" "Extra environment settings found for device \"${BACKUP_DEVICE}\". Exporting."
    set -a
    source /dev/stdin <<< "$restic_environment"
    set +a
  fi

  # Environment set "repository centric"
  restic_environment=$(get_value ${BACKUP_DEVICE} ${BACKUP_IDENTIFIER}/env ${device_type})
  if [ -n "$restic_environment" ]; then
    # Set ENV
    log "INFO" "Extra environment settings found for repository \"${BACKUP_IDENTIFIER}\". Exporting."
    set -a
    source /dev/stdin <<< "$restic_environment"
    set +a
  fi

  if [ -z "$RESTIC_REPOSITORY" ]; then
    export RESTIC_REPOSITORY=${repository_dir}
  fi

  if [[ -n "${RESTIC_COMMAND}" ]]; then
    # A command was specified, give the user chances to add command specific options for the device/identifier
    restic_command_options=$(get_value ${BACKUP_DEVICE} ${RESTIC_COMMAND} ${device_type})
    if [[ $? -ne 0 ]]; then
      restic_command_options=""
    fi

    restic_command_options_local=$(get_value ${BACKUP_DEVICE} ${BACKUP_IDENTIFIER}/$RESTIC_COMMAND ${device_type})
    if [[ $? -eq 0 ]]; then
      restic_command_options+=" ${restic_command_options_local}"
    fi

    if [[ -n "${restic_command_options}" ]]; then
      log "INFO" "Picked up options for \"${RESTIC_COMMAND}\": \"${restic_command_options}\""
      RESTIC_COMMAND_OPTIONS+=" ${restic_command_options}"
    fi
  fi

  if [[ "${RESTIC_COMMAND}" == "backup" ]]; then
    # The command was to back up, so prepare a bit extra
    RESTIC_COMMAND_OPTIONS+=" --exclude-caches"
    # Check if the repository already exists or if we have to create it
    restic snapshots >/dev/null 2>&1
    if [[ $? -ne 0 ]]; then
      log "INFO" "No repository at \"${RESTIC_REPOSITORY}\", initialising"
      restic init --repo ${RESTIC_REPOSITORY}
    fi

    exclude=$(get_value ${BACKUP_DEVICE} ${BACKUP_IDENTIFIER}/exclude ${device_type})
    if [[ $? -eq 0 ]]; then
      log "INFO" "Excluding: $exclude"
      temp_file=$(mktemp) || { echo "Failed to create temp file"; exit 1; }
      trap 'rm -f "$temp_file"' EXIT
      echo "$exclude" > "$temp_file"
      
      RESTIC_COMMAND_OPTIONS+=" --exclude-file=$temp_file"
    fi

    if [ -n "${BACKUP_DIRECTORY}" ]; then
        log "INFO" "Changing to directory ${BACKUP_DIRECTORY}"
        cd "${BACKUP_DIRECTORY}"
        BACKUP_DIRECTORIES="."
    elif [ -z "$BACKUP_DIRECTORIES" ]; then
        # Neither is specified, back up the cwd.
        BACKUP_DIRECTORIES="."
    fi
    RESTIC_COMMAND_OPTIONS+=" ${BACKUP_DIRECTORIES}"
  fi

  set -x
  restic $RESTIC_GLOBAL_OPTIONS $RESTIC_COMMAND "${SCRIPT_ARGUMENTS[@]}" $RESTIC_COMMAND_OPTIONS
  result=$?
  set +x

  exit $?
}

get_value() {
  local section_label="$1"
  local field_label="$2"
  local type="$3"
  local value

  if [[ -n "$OP_CACHED_ITEM" ]]; then
    value=$(echo "$OP_CACHED_ITEM" | jq -r -e --arg section "$section_label" --arg field "$field_label" '
        .fields[]
        | select(.section.label == $section and .label == $field)
        | .value
    ' 2>/dev/null)

    if [[ $? -ne 0 || "$value" == "null" ]]; then
      return 1
    fi

    echo "$value"
  else
    # File based approach
    file="${SCRIPT_DIR}/${type}/${section_label}/${field_label}"
    if [[ -f "$file" ]]; then
      IFS= read -r -d '' contents < "$file"
      echo "$contents"
    else
      return 1
    fi
  fi
    
  return 0
}

skip_op=0

while [[ $# -gt 0 ]]; do
  case $1 in
    --script-help)
      show_help
      exit 0 
      ;;    
    --debug)
      set -x
      ;;
    --no-op)
      skip_op=1
      ;;
   --ssht)
      LOG_VERBOSE=0
      ;;
   --type=*)
      BACKUP_DEVICE_TYPE="${1#*=}"
      ;;
    --device=*)
      BACKUP_DEVICE="${1#*=}"
      ;;
    --identifier=*)
      BACKUP_IDENTIFIER="${1#*=}"
      ;;
    --directory=*)
      BACKUP_DIRECTORY="${1#*=}"
      ;;
    --directories=*)
      BACKUP_DIRECTORIES="${1#*=}"
      ;;
    # If the command sets a $RESTIC_COMMAND then extra arguments in 1password are supported for it. Feel free to add others.
    cron)
      RESTIC_COMMAND_OPTIONS+=" --tag scheduled"
      RESTIC_COMMAND='backup'
      ;;
    backup|do_backup)
      RESTIC_COMMAND="backup"
      ;;
    forget)
      RESTIC_COMMAND=$1
      ;;
    *)
      SCRIPT_ARGUMENTS+=( "$1" )
  esac
  shift
done

if [[ $skip_op -eq 0 ]]; then
  _=`type -p op`
  if [[ $? -ne 0 ]]; then
    log "WARN" "\"op\" could not be found. The script will fall back to files (\"${SCRIPT_DIR}/BACKUP_DEVICE/BACKUP_IDENTIFIER\")"
    skip_op=1
  fi

  _=`type -p jq`
  if [[ $? -ne 0 ]]; then
    log "WARN" "\"jq\" could not be found. The script will fall back to files (\"${SCRIPT_DIR}/BACKUP_DEVICE/BACKUP_IDENTIFIER\")"
    skip_op=1
  fi
fi

if [[ $skip_op -eq 0 ]]; then
  # Prefill the cache so we can do error checking/logging here
  OP_CACHED_ITEM=$(op item get "${ONEPASSWORD_ITEM_ID}" --vault "${ONEPASSWORD_VAULT}" --format json 2>/dev/null)
  if [[ $? -ne 0 || -z "$OP_CACHED_ITEM" ]]; then
    log "ERROR" "Error: Failed to retrieve item from vault."
    op item get "${ONEPASSWORD_ITEM_ID}" --vault "${ONEPASSWORD_VAULT}" >&2
    exit 50
  fi
fi  


main "${SCRIPT_ARGUMENTS[@]}"