Remove all duplicate files in the current directory that are not managed by drupal.

drupal-remove-duplicate.sh

#!/bin/bash

#
# Remove all duplicate files in current directory that are not present
# in Drupal's managed file table. If a backup directory exists then the
# files will be moved into that directory instead of being deleted.
#

#
# Determine if a file is manged by drupal. Drush is required.
#
# $1 Name of file to check against file_managed table filename field.
# $2 URI of site to query via drush.
#
# @return int
#   1 if file is managed by drupal, 0 otherwise.
#
function drupal_file_managed() {
    local FILE=$1
    if [ -z "$1" ]; then
        echo 'Parameter #1 is missing'
        exit 1
    fi
    local URI=$2
    if [ -z "$1" ]; then
        echo 'Parameter #2 is missing'
        exit 1
    fi
    local SQLCONNECT=`drush sql-connect --uri="http://$URI"`
    SQLCONNECTLEN=${#SQLCONNECT}
    if [ $SQLCONNECTLEN -le 10 ]; then
        echo 'Parameter #2 is invalid'
        exit 1
    fi
    local SQLFILEEXISTS="$SQLCONNECT -e \"SELECT filename FROM file_managed WHERE uri='public://$FILE'\""
    local RESULT=`eval $SQLFILEEXISTS`
    if [ "$RESULT" ]; then
        return 1
    else
        return 0
    fi
}

echo -ne 'Drupal domain name: '
read DOMAIN
echo -ne 'Name of backup directory: '
read BACKUP
echo ''

# Create backup directory if it does not exist.
if [ ! -z $BACKUP ]; then
    if [ ! -d "$BACKUP" ]; then
        echo 'Creating backup directory...'
        mkdir $BACKUP
    fi
fi

FILES_TOTAL=`find . -maxdepth 1 -type f -exec basename {} \; | wc -l`
# - regex '.*/[aA].*' or -name 'a*' to reduce list of files.
FILES=`find . -maxdepth 1 -type f -exec basename {} \;`
echo "Total number of files: $FILES_TOTAL"
for FIRST in $FILES; do
    # \033[0K Deletes to end of line, \r returns to begining of line.
    echo -ne "Checking $FIRST...\033[0K\r"
    for SECOND in $FILES; do
        if [ "$FIRST" != "$SECOND" ]; then
            cmp -s $FIRST $SECOND
            # Check if this file is a duplicate.
            if [ $? -eq 0 ]; then
                drupal_file_managed $FIRST $DOMAIN
                # Check if this file is manged by drupal.
                if [ $? == 1 ]; then
                    echo "Duplicate Managed by drupal: $FIRST, $SECOND"
                else
                    # Only remove duplicate files that are not managed by drupal.
                    echo "Duplicate NOT managed by drupal: $FIRST, $SECOND"
                    if [ -d "$BACKUP" ]; then
                        # A backup directory exists, move the file instead of deleting.
                        mv $FIRST $BACKUP/$FIRST
                    else
                        # No backup directory exists, delete the file forever.
                        rm $FIRST
                    fi
                    # File is no longer here, don't bother checking again.
                    break
                fi
            fi
        fi
    done
done