Linux-side deduplication

cleardupes.pl

#!/usr/bin/perl

# Duplicate finder.  Moves dupes in what directories you've specified to a dedicated "Trash" directory for you to remove.
# Double-checks those MD5 hashes!
#
# Example of usage:

# find 00.Master -type f -print0 | xargs -0 md5sum > master.hash.txt 
# ...etc... (repeat above line for other directories to dedupe against)
# cat *.hash.txt | sort | uniq -D -w 32 > dupes.txt
# perl cleardupes.pl dupes.txt | tee log.txt

# Your dupes are in 0.Trash

use strict;
use File::Basename;

$|=1;

my %hv;

mkdir "0.Trash" unless(-e "0.Trash");

while(<>) {
    chomp;
    next if(/^$/);
    next if(/^#/);
    
    next unless(/^(\S+)\s+(\S.+)$/);
    my $h=$1;
    my $f=$2;
    unless(exists $hv{$h}) {
        $hv{$h}=$f;
        print "# First: $f\n";
        next;
    } 

	unless(-e $f) {
		print "# Taken already: $f\n";
		next;
	}

    # Likely:
    print "# Likely: $f\n";
    my $s=$hv{$h};
    system('cmp','-s',$s,$f);
    if($? == -1) {
        die "cmp failed: $!";
    } elsif($? & 127) {
        die sprintf("child died w/signal %d, %s coredump",
            $? & 127, ($? & 128) ? 'with' : 'without');
    }
    
    my $rv=$? >> 8;
    if($rv) {
        print "! Total compare fail (false positive):\n";
        print "! Src: $s\n";
        print "! Dst: $f\n";
        next;
    }
    
    # Move it.
    my $p=dirname($f);
    print "> Extra: $f\n";
    system "mkdir","-p","0.Trash/$p";
    system "mv","$f","0.Trash/$p";
}

noemptydir.sh

#!/bin/bash

while [ `find . -type d -empty | wc -l` -gt 0 ]; do
	find . -type d -empty -print0 | xargs -0 rmdir
done