libvirt virtual machine automatic backup script to external location

libvirt-backups.sh

#!/bin/bash

# MIT License
#
# Copyright (c) 2024 Axel Segebrecht (bebraver.online)
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR the use or other DEALINGS IN THE
# SOFTWARE.

# README: Virtual Machine Backup Script

# This script automates the backup process for all virtual machines managed by libvirt.
# It performs the following actions for each VM:

# 1.  Shuts down the VM (if it's running).
# 2.  Creates a disk-only snapshot using virsh.
# 3.  Generates a standalone disk image from the snapshot using virsh blockpull.
# 4.  Compresses the standalone image with bzip2 to save storage space.
# 5.  Verifies the integrity of the compressed image using qemu-img check.
# 6.  Deletes the libvirt snapshot.
# 7.  Restarts the VM (if it was running before the backup).
# 8.  Checks dependencies (virsh, qemu-img, bzip2).
# 9.  Calculates and checks available disk space before starting backups.
# 10. Provides verbose output to the console for all steps if run with --verbose or -v.
# 11. Logs errors and continues to the next VM if a backup fails.
# 12. Adds specific error codes to logs for better troubleshooting.
# 13. Implements log rotation to keep logs within a user-defined frequency.

# Configuration:

#   - SNAPSHOT_DIR: The directory where libvirt stores disk images and snapshots.
#   - ARCHIVE_DIR: The directory where the backup archives will be stored.
#   - LOG_ERROR: The path to the error log file.
#   - LOG_SUCCESS: The path to the success log file.
#   - DATE_FORMAT: The date format used for the backup file names.
#   - LOG_RETENTION_DAYS: Number of days to keep log files.

# Important Notes:

#   - This script requires root privileges to run.
#   - Ensure that the ARCHIVE_DIR has sufficient storage space.
#   - The script will shut down running VMs temporarily during the backup process.
#   - Error and success messages are logged to the specified log files.
#   - The script will attempt to continue processing other VMs even if one fails.
#   - Ensure libvirt, qemu-img, and bzip2 are installed.
#   - Ensure the user running the script has permissions to write to the archive and log directories.
#   - It is recommended to test the script thoroughly in a non-production environment before relying on it for critical backups.
#   - Run the script with --verbose or -v for console output of all steps.
#   - Before running the script, ensure that the dependencies are met.
#   - Ensure that there is enough disk space in both the snapshot and archive directories.
#   - The script calculates the space needed, and will fail if the space is not available.
#   - If a virtual machine backup fails, the script will log the error and continue to the next machine.
#   - Log rotation is implemented to keep only the last LOG_RETENTION_DAYS of logs.

# How to run this script:

# 1. Directly in the console:
#    - Make the script executable: chmod +x your_script_name.sh
#    - Run the script: sudo ./your_script_name.sh
#    - For verbose output: sudo ./your_script_name.sh --verbose or sudo ./your_script_name.sh -v

# 2. Via a cron job (e.g., daily at 1 AM):
#    - Open the crontab editor: sudo crontab -e
#    - Add the following line to the crontab:
#      0 1 * * * /path/to/your_script_name.sh
#    - Save and exit the crontab editor.
#    - To get verbose output from cron, redirect the output to a file:
#      0 1 * * * /path/to/your_script_name.sh --verbose >> /path/to/cron_output.log 2>&1

# Changelog
# rev 6 - increased delay, retries and added domain check 
# rev 5 - added delay after a vm is shutdown to ensure the next step can continue without error 
# as the virtual machine may still be in the process of shutting down

SNAPSHOT_DIR="/var/lib/libvirt/images"
ARCHIVE_DIR="/big-tank/archives/virtual-machines"
LOG_ERROR="/var/log/vmbackup-error.log"
LOG_SUCCESS="/var/log/vmbackup-success.log"
DATE_FORMAT="%Y-%m-%d-%H%M"
DATE=$(date +"$DATE_FORMAT")
LOG_RETENTION_DAYS=7

VERBOSE=false
while [[ $# -gt 0 ]]; do
  case "$1" in
    -v|--verbose)
      VERBOSE=true
      shift
      ;;
    *)
      break
      ;;
  esac
done

mkdir -p "$ARCHIVE_DIR"
mkdir -p "$(dirname "$LOG_ERROR")"
touch "$LOG_ERROR"
touch "$LOG_SUCCESS"

log_error() {
  ERROR_CODE=$2
  ERROR_MESSAGE=$3
  echo "$(date): ERROR [$ERROR_CODE]: $ERROR_MESSAGE" >> "$LOG_ERROR"
  echo "$(date): ERROR [$ERROR_CODE]: $ERROR_MESSAGE"
  if [[ -n "$4" ]]; then
    echo "$(date): ERROR DETAILS: $4" >> "$LOG_ERROR"
    echo "$(date): ERROR DETAILS: $4"
  fi
}

log_success() {
  echo "$(date): SUCCESS: $1" >> "$LOG_SUCCESS"
  if $VERBOSE; then
    echo "$(date): SUCCESS: $1"
  fi
}

verbose_echo() {
  if $VERBOSE; then
    echo "$1"
  fi
}

if ! command -v virsh &> /dev/null; then
  echo "ERROR: virsh is not installed. Please install libvirt."
  exit 1
fi

if ! command -v qemu-img &> /dev/null; then
  echo "ERROR: qemu-img is not installed. Please install qemu-utils."
  exit 1
fi

if ! command -v bzip2 &> /dev/null; then
  echo "ERROR: bzip2 is not installed. Please install bzip2."
  exit 1
fi

if ! command -v jq &> /dev/null; then
    echo "ERROR: jq is not installed. Please install jq."
    exit 1
fi

VM_LIST=$(virsh list --name)

TOTAL_DISK_SPACE=0
for VM_NAME in $VM_LIST; do
    DISK_DEVICE=$(virsh domblklist "$VM_NAME" | awk '/vda/{print $2}')
    if [[ -n "$DISK_DEVICE" ]]; then
      DISK_SIZE=$(qemu-img info --output json "$DISK_DEVICE" 2>/dev/null | jq -r '.virtual-size')
      if [[ -n "$DISK_SIZE" && "$DISK_SIZE" -gt 0 ]]; then
        TOTAL_DISK_SPACE=$((TOTAL_DISK_SPACE + DISK_SIZE))
      else
        log_error "Failed to retrieve disk size for $VM_NAME. Skipping." "201" "Failed to retrieve virtual size for disk: $DISK_DEVICE"
      fi
    fi
done

ARCHIVE_FREE=$(df -P "$ARCHIVE_DIR" | tail -n 1 | awk '{print $4}')
SNAPSHOT_FREE=$(df -P "$SNAPSHOT_DIR" | tail -n 1 | awk '{print $4}')

if [[ "$ARCHIVE_FREE" -lt "$TOTAL_DISK_SPACE" ]]; then
  log_error "Insufficient disk space in $ARCHIVE_DIR" "101" "Insufficient disk space in $ARCHIVE_DIR. Required: $TOTAL_DISK_SPACE, Available: $ARCHIVE_FREE. Please free up space."
  exit 1
fi

if [[ "$SNAPSHOT_FREE" -lt "$TOTAL_DISK_SPACE" ]]; then
  log_error "Insufficient disk space in $SNAPSHOT_DIR" "102" "Insufficient disk space in $SNAPSHOT_DIR. Required: $TOTAL_DISK_SPACE, Available: $SNAPSHOT_FREE. Please free up space."
  exit 1
fi

for VM_NAME in $VM_LIST; do
  SNAPSHOT_NAME="daily-backup-$DATE"
  PULL_NAME="$VM_NAME-$DATE.qcow2"

  verbose_echo "$(date): Processing VM: $VM_NAME"

  VM_STATE=$(virsh domstate "$VM_NAME")

  if [[ "$VM_STATE" == "running" ]]; then
    verbose_echo "$(date): Shutting down $VM_NAME"
    virsh shutdown "$VM_NAME"
    sleep 10
    # Wait until domain is shut off.
    for i in {1..10}; do
        if [[ $(virsh domstate "$VM_NAME") == "shut off" ]]; then
          break;
        fi
        sleep 5;
    done
    if [[ $(virsh domstate "$VM_NAME") != "shut off" ]]; then
      log_error "Failed to shutdown $VM_NAME" "301" "$(virsh domstate \"$VM_NAME\")"
      continue
    fi
  fi

  verbose_echo "$(date): Creating snapshot for $VM_NAME"
  virsh snapshot-create-as "$VM_NAME" "$SNAPSHOT_NAME" --disk-only --no-metadata --quiesce || {
    log_error "Failed to create snapshot for $VM_NAME" "401" "$(virsh snapshot-create-as \"$VM_NAME\" \"$SNAPSHOT_NAME\" --disk-only --no-metadata --quiesce)"
    continue
  }

  # Retry to get the disk device after a longer delay and more attempts
  for i in {1..5}; do
    DISK_DEVICE=$(virsh domblklist "$VM_NAME" | awk '/vda/{print $2}')
    if [[ -n "$DISK_DEVICE" ]]; then
      break
    fi
    sleep 5 # Wait for 5 seconds before retrying
  done

  if [[ -z "$DISK_DEVICE" ]]; then
    log_error "Failed to find disk device for $VM_NAME" "501" "$(virsh domblklist \"$VM_NAME\")"
    continue
  fi

  verbose_echo "$(date): Performing blockpull for $VM_NAME"
  if virsh blockpull --wait "$VM_NAME" "$DISK_DEVICE" --base "$SNAPSHOT_NAME" "$ARCHIVE_DIR/$PULL_NAME"; then
    log_success "Blockpull successful for $VM_NAME"
  else
    log_error "Blockpull failed for $VM_NAME" "601" "$(virsh blockpull --wait \"$VM_NAME\" \"$DISK_DEVICE\" --base \"$SNAPSHOT_NAME\" \"$ARCHIVE_DIR/$PULL_NAME\")"
    continue
  fi

  verbose_echo "$(date): Compressing $ARCHIVE_DIR/$PULL_NAME"
  if ! bzip2 "$ARCHIVE_DIR/$PULL_NAME"; then
    log_error "Failed to compress $ARCHIVE_DIR/$PULL_NAME" "701" "$(bzip2 \"$ARCHIVE_DIR/$PULL_NAME\")"
    continue
  fi

  QEMU_CHECK_OUTPUT=$(qemu-img check "$ARCHIVE_DIR/$PULL_NAME.bz2" 2>&1)
  if [[ $? -ne 0 ]]; then
    log_error "Snapshot verification failed for $VM_NAME" "801" "$QEMU_CHECK_OUTPUT"
    continue
  fi
  log_success "Snapshot verification successful for $VM_NAME"

  if ! virsh snapshot-delete "$VM_NAME" "$SNAPSHOT_NAME"; then
    log_error "Failed to delete snapshot $SNAPSHOT_NAME" "901" "$(virsh snapshot-delete \"$VM_NAME\" \"$SNAPSHOT_NAME\")"
  fi

  if [[ "$VM_STATE" == "running" ]]; then
    if ! virsh start "$VM_NAME"; then
      log_error "Failed to start $VM_NAME" "1001" "$(virsh start \"$VM_NAME\")"
    fi
  fi
done

find "$LOG_ERROR" "$LOG_SUCCESS" -mtime +$LOG_RETENTION_DAYS -delete

# And so, the great quest for backups came to an end.

# For the BENEFIT of SEVERAL VIEWERS Mr. Segebrecht's much admir'd attempt
# to write shell scripts libvirt-backups.sh was performed by Google's Gemini
# 2.0 Flash model. To conclude with Rule Bash in full chorus 
# NO MONEY RETURN'D (C) be braver online MCMLXXXVII