Concat borg segments to larger segments and reset their numbers

borg-concat-segments.sh

#!/bin/bash
#
# Concat borg segments to larger segments and reset their numbers
#
# As usecase I concatenated my 5MB segments to 128MB segments
#
# Each borg semenent is a list of entry logs prefixed by "BORG_SEG" magic
# header. They can be concatenated by removing the magic header but the first
# original segment. The segments must preserve their original natural order.
#
# Execute this script in the repo dir or overwrite SEG_ORIG_DIR and
# SEG_CONCAT_DIR. By default it creates new concatenated segments in data.tmp.
# After concatenation please remove or rename the original data directory and
# rename data.tmp directory to data. Remove hints, index and integrity repo
# files and run 'borg check --repair' to rebuild hints, index and integrity
# files for new segment.
#
# It is advisable to backup your borg repo before concatenate segments eg. via
# btrfs snapshots
#
# Bash Example:
#   $ export BORG_REPO=/mnt/backup
#   $ SEG_ORIG_DIR=$BORG_REPO/data SEG_CONCAT_DIR=$BORG_REPO/data.concat SEGS_PER_DIR=1000 borg-concat-segments.sh
#   $ rm -rf $BORG_REPO/{hints,index,integrity}* $BORG_REPO/data
#   $ mv $BORG_REPO/data.concat $BORG_REPO/data
#   $ borg check --repair --verify-data --progress
#
# (c) 2021 [email protected] WTFPL
#

# Customizable variables
SEG_ORIG_DIR=${SEG_ORIG_DIR:-data}
SEG_CONCAT_DIR=${SEG_CONCAT_DIR:-data.tmp}

SEGS_PER_DIR=${SEGS_PER_DIR:-2000}
MAX_SEG_SIZE=${MAX_SEG_SIZE:-134217728} # 128MB

# Internal variables
SEG_ID=1
SEG_DIR=0
SEG_DIR_COUNT=1

CURRENT_SEG=$SEG_CONCAT_DIR/$SEG_DIR/$SEG_ID
mkdir -p $(dirname $CURRENT_SEG)

# Iterate over each segment in natural order (...,8,9,10,11,...)
find $SEG_ORIG_DIR -type f | sort -V | while read SEG; do
  # Create or append orig seg
  if [ ! -f "$CURRENT_SEG" ]; then
    echo "Create segment $CURRENT_SEG with $SEG"
    cp $SEG $CURRENT_SEG
  else
    echo "Append segment $SEG to $CURRENT_SEG"
    tail -c +9 $SEG >> $CURRENT_SEG
  fi
  # Uncomment carefully next line to auto-remove old segments if space is an issue
  #rm $SEG
 
  # Check size and dir
  SIZE=$(stat -c %s $CURRENT_SEG)
  if [ $SIZE -gt $MAX_SEG_SIZE ]; then
    let SEG_ID+=1
    let SEG_DIR_COUNT+=1
    
    if [ $SEG_DIR_COUNT -ge $SEGS_PER_DIR ]; then
      let SEG_DIR+=1
      SEG_DIR_COUNT=1
    fi
    CURRENT_SEG=$SEG_CONCAT_DIR/$SEG_DIR/$SEG_ID
    mkdir -p $(dirname $CURRENT_SEG)
  fi
done