ZFS Deep Archive Backup

zfs_full_backup_send.sh

#!/bin/bash

# Take a snapshot of a given pool/dataset and send a replication stream to S3 Glacier Deep Archive
# AWS cli must be configured with crendentials granting access to the specified bucket
# NOTES: 
# - Will only work for datasets that fit within the current S3 object size limit,
#   currently 5TB.
# - Deep Archive requires objects be stored for 180 days. Recommend setting a lifecyle
#   policy on the bucket to expire objects older than.
# - Recommend running script as a cronjob with appropriate interval, at least monthly.

# AWS destination bucket name
BUCKET="my-bucket"

# ZFS pool / dataset
POOL="my-pool/dataset"
echo "Using pool/datset: ${POOL}"

# Get a timestamp for the snapshot
SNAP_NAME="backup_script_$(date +%Y-%m-%d_%H-%M-%S)"

# Take a snapshot of the pool with the timestamp in the name
echo "Taking snapshot ${POOL}@${SNAP_NAME}"
zfs snapshot "${POOL}@${SNAP_NAME}"

# Get size for cp
SIZE=$(zfs list -Hpo used ${POOL})

# Fix pathname by converting / to _
POOL_UNPATH=${POOL//\//_}

# Create a compressed archive of the snapshot using zfs send and zstd compression
echo "Sending and uploading ${POOL_UNPATH}_${SNAP_NAME}.zst to ${BUCKET}"
zfs send -Rwc "${POOL}@${SNAP_NAME}" \
| aws s3 cp - s3://${BUCKET}/${POOL_UNPATH}_${SNAP_NAME}.bak \
--expected-size ${SIZE} --storage-class DEEP_ARCHIVE

echo "Snapshot created and compressed archive saved to s3://${BUCKET}/${POOL_UNPATH}_${SNAP_NAME}.bak"