When passed a source dir and a target dir, this script will check by filename for each file in the source dir, whether it exists in the target dir. If not, it will move it to the target dir. If it exists, it verifies via sha256sum that the content is the same. If the content is the same it will delete the source file. Useful for when you have a …

dedupecp.sh

#!/usr/bin/env bash

SRC=$1
DEST=$2

echo "cp from $SRC to $DEST"

function hash ()
{
    local  __resultvar=$1
    local  myresult=$(sha256sum "$2" | awk '{ print $1; }')
    eval $__resultvar="'$myresult'"
}

ls -1 "$SRC" | while read -r file
do
	echo "[$SRC/$file]"
	if [ -f "$DEST/$file" ]
	then
		hash hashdest "$DEST/$file"
		hash hashsrc "$SRC/$file"
		echo "$hashsrc == $hashdest"
		if [ "$hashsrc" == "$hashdest" ]
		then
			echo "File exists at destination, removing."
			mkdir -p "$SRC/.moved"
			mv "$SRC/$file" "$SRC/.moved/$file"
		else
			echo "!!! Same name, different hash. Renaming at source and moving to destination."
			echo "Target is $DEST/${hashsrc}_$file"
			mv -n "$SRC/$file" "$SRC/${hashsrc}_$file"
			mv -n "$SRC/${hashsrc}_$file" "$DEST/${hashsrc}_$file"
		fi
	else
		echo "No such file at destination. Moving file to destination."
		mv -n "$SRC/$file" "$DEST"
	fi
done