A script I used to consolidate and de-duplicate lots of pictures and movies from various iPhoto libraries and random folders full of images.

dedupe-media.sh

#!/bin/sh

set -e
# set -x

src=$1
dst=$2

find "$src" -type f | while read file ; do
    chksum=$(md5 -q "$file")
    dir=$(echo $chksum | cut -b1-2)
    suffix=$(echo $file | awk -F. '{print tolower($NF)}')
    to=$dst/$dir/$chksum.$suffix
    if ! test -f "$to" ; then
        # Make sure destination dir exists...
        mkdir -p $dst/$dir
        link "$file" "$to"
    fi

    log="$chksum $file"
    echo $log >> "$dst/log"
    echo $log
done