git-annex-remote-goacd

#!/bin/bash
set -e

# Installation
# 1. Setup go-acd (https://github.com/go-acd/acd). You need to chmod
# acd-token.json to 0600 and manually create $HOME/.config/acd.json
# like the following:
#   {"tokenFile":"/home/you/.config/acd-token.json","cacheFile":"/home/you/.cache/go-acd.cache"}
# Then make sure the `acd` command works. Note that the first run may
# take some time as it will fetch the full list of your files on ACD.
# 2. Put this file (git-annex-remote-goacd) in your PATH.
# 3. Change your remote's externaltype from rclone to goacd:
#   $ git annex enableremote myacdremote externaltype=goacd


if which acd >/dev/null 2>&1; then
  GOACD=acd
else
  GOACD="${GOPATH:-$HOME/go}/bin/acd"
fi


ask() {
  echo "$1" "$2"
  read -r resp
  resp="${resp%$'\r'}"
  if echo $resp | grep '^VALUE '>/dev/null; then
    RET=$(echo "$resp" | cut -f2- -d' ')
  fi
}

getconfig() {
  ask GETCONFIG "$1"
}

calclocation() {
  LOC="acd://$REMOTE_PREFIX/"
  case "$RCLONE_LAYOUT" in
    lower)
      ask DIRHASH-LOWER "$1"
      LOC+="$RET"
      ;;
    directory)
      ask DIRHASH-LOWER "$1"
      LOC+="$RET$1/"
      ;;
    nodir)
      ;;
    mixed)
      ask DIRHASH "$1"
      LOC+="$RET"
      ;;
    frankencase)
      ask DIRHASH "$1"
      lret=$(echo $RET | tr A-Z a-z)
      LOC+="$lret"
      ;;
    lower1)
      ask DIRHASH-LOWER "$1"
      LOC+="${RET%%/*}/"
      ;;
    *)
      echo "invalid rclone_layout" >&2
      exit 1
      ;;
  esac
}

retry() {
  local count=$1
  shift
  while [ $count -gt 0 ]; do
    "$@" && return 0 || :
    let count--
    sleep 3
  done
  return 1
}


echo VERSION 1

while read -r line; do
  set -- $line
  cmd="$1"
  case "$cmd" in
    INITREMOTE)
      getconfig prefix
      REMOTE_PREFIX="${RET%/}"

      getconfig rclone_layout
      RCLONE_LAYOUT="$RET"

      if [[ -n $REMOTE_PREFIX ]] && [[ -n $RCLONE_LAYOUT ]]; then
        echo $cmd-SUCCESS
      else
        echo $cmd-FAILURE "prefix or rclone_layout not set"
      fi
      ;;

    PREPARE)
      getconfig prefix
      REMOTE_PREFIX="${RET%/}"

      getconfig rclone_layout
      RCLONE_LAYOUT="$RET"

      echo $cmd-SUCCESS
      ;;

    TRANSFER)
      key="$3"
      file="$4"
      calclocation "$key"
      case "$2" in
        STORE)
          if retry 3 "$GOACD" cp "$file" "$LOC$key"; then
            fsize="$(stat -c '%s' "$file")"
            if "$GOACD" ls -l "${LOC%/}" \
                | grep "^-\s$fsize\s.\+\s$key$" >/dev/null; then
              # Unfortunately acd command does not offer a simple way
              # to verify file integrity, hence this dirty hack
              annextmp="$(dirname "$file")" # .git/annex/tmp
              unique="goacd-tmp-$(echo -n "$key" | md5sum | cut -d " " -f1)-$(date "+%s")-$RANDOM"
              src="$annextmp/$unique/$(basename "$LOC")"
              dest="$(dirname "$LOC")"
              [[ $dest = "acd:" ]] && dest="acd://"
              mkdir -p "$src"
              ln "$file" "$src/"
              "$GOACD" --log-level 4 cp -R "$src" "$dest" 2>&1 \
                | grep "/$key\" already exists and has the same content" >/dev/null \
                && success=y || success=n
              rm "$src/$key"
              rmdir "$src" "$annextmp/$unique" 2>/dev/null || :
              if [[ $success = y ]]; then
                echo $cmd-SUCCESS STORE "$key"
              else
                echo $cmd-FAILURE STORE "$key" "could not verify file content"
              fi
            else
              echo $cmd-FAILURE STORE "$key" "could not verify file size"
            fi
          else
            echo $cmd-FAILURE STORE "$key" "upload failed"
          fi
          ;;

        RETRIEVE)
          if retry 3 "$GOACD" cp "$LOC$key" "$file"; then
            echo $cmd-SUCCESS RETRIEVE "$key"
          else
            echo $cmd-FAILURE RETRIEVE "$key" "download failed"
          fi
          ;;
      esac
      ;;

    CHECKPRESENT)
      key="$2"
      calclocation "$key"
      set +e
      output="$("$GOACD" ls -l "${LOC%/}" 2>&1)"
      retcode=$?
      set -e
      if [ $retcode -eq 0 ]; then
        # parent folder exists
        if echo "$output" | grep "^-"$'\t'".\+"$'\t'"$key$" >/dev/null; then
          echo $cmd-SUCCESS "$key"
        else
          echo $cmd-FAILURE "$key"
        fi
      else
        if echo "$output" | grep ' node not found$' >/dev/null; then
          echo $cmd-FAILURE "$key"
        else
          echo $cmd-UNKNOWN "$key" "'acd ls' exited unexpectedly"
        fi
      fi
      ;;

    REMOVE)
      echo $cmd-FAILURE "$2" "REMOVE is not supported"
      ;;

    *)
      echo UNSUPPORTED-REQUEST
      ;;
  esac
done