backup

gistfile1.txt

# src is a staging area... copy files into src and then backup src to have them moved de-duped to dst
unset MYHOST STAGINGDIR BACKUPDIR REMOTEHOST REMOTEDIR
test -f ~/.backuprc && . ~/.backuprc
: ${MYHOST:=${HOSTNAME:-$(hostname -f)}}
: ${STAGINGDIR:=~/.backup}
STAGINGDIR=${STAGINGDIR/%//$MYHOST}; test -d $STAGINGDIR || mkdir -p $STAGINGDIR
: ${BACKUPDIR:=~/backup}
FILESTORE=$BACKUPDIR/.filestore; test -d $FILESTORE || mkdir -p $FILESTORE
BACKUPDIR=${BACKUPDIR/%//$MYHOST};

function backup::thisdir { echo $BACKUPDIR/$(date +"%Y%m%dT%H%M%S"); }
function backup::lastdir { local mydir=${1:-$BACKUPDIR}; local lastdir=$(ls -1r $mydir | sed -n '1p'); : ${lastdir:=00000000T000000}; lastdir=$mydir/$lastdir; test -d $lastdir || mkdir -p $lastdir; echo $lastdir; }
function backup::✓ { local err=0;  echo -e "\e[32m✓\e[0m $*"; return $err; }
function backup::≃ { local err=$?; echo -e "\e[33m≃\e[0m $*"; return $err; }
function backup::✗ { local err=$?; test "${err:-0}" -eq 0 && err=1; echo -e "\e[31m✗\e[0m $*"; return $err; }
function backup::¿ { [ $? == 0 ] && backup::✓ "$*" || backup::✗ "$*"; }
function back::random_chunks { :; }
function backup::dots { A=${1:-0.25} perl -MTime::HiRes=alarm -n -e 'BEGIN{$|=1;$SIG{ALRM}=sub{print".";alarm($ENV{A})};alarm($ENV{A})}END{print"\n"}'; }
function backup::spin { A=${1:-0.25} perl -MTime::HiRes=alarm -n -e 'BEGIN{$|=1;$spin=0;@spin=qw(- \\ | /);$SIG{ALRM}=sub{printf"\r%s",$spin[++$spin%4];alarm($ENV{A})};alarm($ENV{A})}END{print"\r \n"}'; }

function bup {
  unset watch
  for i in "$@"; do
    test -n "$i" && cp -fruv --parents "$i" "$STAGINGDIR"
  done | backup::spin .1
  backup
  backup-gc
  backup-ls
  backup-rsync
}

function backup {
  test $(find "$STAGINGDIR" -mindepth 1 -maxdepth 1 | wc -l) -eq 0 && backup::✗ "nothing staged" && return
  return
  unset watch
  [ "$1" == "-w" ] && local watch=-w && shift;
  local thisdir=$(backup::thisdir)
  local lastdir=$(backup::lastdir)
  test -d "$thisdir" && backup::✗ "$thisdir already exists" && return
  cp -faluv "$lastdir" "$thisdir" | backup::spin .1
  while read i; do
    test -L "$i" -a -d "$i" && continue
    sha="$(sha256sum "$i")";
    staging="${sha:66}";
    ! test "$staging" == "$i" && backup::✗ "problem" && return
    file="${staging#*/}";
    dirname="${file%/*}";
    [ "$dirname" == "$file" ] && unset dirname;
    sha=${sha:0:64};
    filestore=$FILESTORE/${sha:0:1}/${sha:0:2}/${sha:0:3};
    { 
      test -d "$filestore" || mkdir -pv "$filestore"
    } && { 
      test ! -d "$filestore/$sha" || ln -v "$staging" "$filestore/$sha"
    } && { 
      test -d "$thisdir${dirname/${STAGINGDIR:1}}" || mkdir -pv "$thisdir${dirname/${STAGINGDIR:1}}"
    } && { 
      test ! -e "$filestore/$sha" || cp -fluv "$filestore/$sha" "$thisdir${file/${STAGINGDIR:1}}"
    } && { 
      rm -fv "$staging" && find "${staging%/*}" -mindepth 1 -exec false {} + -quit && echo rmdir -v "${staging%/*}"
    };
    echo;
  done < <(if [ -n "$watch" ]; then inotifywait -rmq -e close_write --format "%w%f" "$STAGINGDIR"; else find "$STAGINGDIR" ! -type d; fi)
}

function backup-dirs {
  local s s1 b b1
  test -d "$STAGINGDIR" && s1=$(find "$STAGINGDIR" -mindepth 1 -maxdepth 1 -type d | wc -l) s=$(find "$STAGINGDIR" ! -type d | wc -l)
  backup::¿ "$STAGINGDIR: ${s1:-?} ${s:-?}"
  test -d "$BACKUPDIR" && b1=$(find "$BACKUPDIR" -mindepth 1 -maxdepth 1 -prune -o -type d | wc -l) b=$(find "$BACKUPDIR" ! -type d | wc -l)
  backup::¿ "$BACKUPDIR: ${b1:-?} ${b:-?}"
  test -d "$FILESTORE" && f=$(find "$FILESTORE" ! -type d | wc -l)
  backup::¿ "$FILESTORE: ${f:-?}"
}

function backup-find {
  [ "$1" == "-a" ] && find=() && shift;
  ! test -n "$1" && backup::✗ "usage: $FUNCNAME item" && return
  ! test -f "$1" && test "$1" == "${1%/*}" && set -- $FILESTORE/${1:0:1}/${1:0:2}/${1:0:3}/$1
  ! [[ $1 =~ ^$BACKUPDIR ]] && local rel=($(find $BACKUPDIR -regex ".*/${1#/}" -print -quit)) && set -- "$rel"
  test -n "$1" && BACKUPDIR=$BACKUPDIR find $BACKUPDIR -samefile $1 -printf "%03n %h/%f\n" -exec bash -c 'a="{}"; b="${a/$BACKUPDIR}"; test -f "$b" && stat -c "% 3h %n" "$b"' \;
}

function backup-gc {
  find "$STAGINGDIR" -mindepth 2 -type d -empty -delete
  find "$BACKUPDIR" "$FILESTORE" -links 1 -delete
  find "$BACKUPDIR" "$FILESTORE" -type d -empty -delete
}

function backup-ls {
  unset all
  [ "$1" == "-a" ] && local all=-a && shift;
  echo -e "\n$STAGINGDIR/:";
  find "$STAGINGDIR" ! -type d -printf "%03n %P\n";
  echo -e "\n$BACKUPDIR/:";
  for i in "$BACKUPDIR"/*; do
    echo -e "\n  ${i##*/}/:";
    local IFS=$'\n'
    while read k; do
      l="${k:8}"
      IFS=$'\t' m=($l)
      find $FILESTORE -samefile ${m[0]} -printf "${k:0:8}${m[1]} (%f)\n" | grep --color=never -P "${1:-.}"
    done < <(find "$i" -prune -o ! -type d -printf "    %03n %h/%f\t%P\n")
    unset IFS
  done
  test -z "$all" && return
  echo -e "\n$FILESTORE:";
  find "$FILESTORE" -type f -printf "%03n %P\n" | sed -e 's!^\(.*\)/!\1 !'
}

function backup-pop {
  echo -e "\n$BACKUPDIR/:";
  for i in "$BACKUPDIR"/*; do rm -rvf "$i"; break; done
}

function backup-rsync {
  local err=$?
  ! test -n "$REMOTEHOST" && backup::≃ "skip: backup-rsync not configured" && return $err
  rsync -vaziH -e ssh --progress "$FILESTORE" "$BACKUPDIR" "$REMOTEHOST:${REMOTEDIR:-backup}"
  rsync -vaziH -e ssh --progress --delete "$BACKUPDIR" "$REMOTEHOST:${REMOTEDIR:-backup}"
  ssh "$REMOTEHOST" find ${REMOTEDIR:-backup}/.filestore ! -type d -links 1 -delete
  ssh "$REMOTEHOST" find ${REMOTEDIR:-backup}/.filestore -empty -delete
}