dupecheck.js

#!/usr/bin/env node
var fs = require('graceful-fs');
var crypto = require('crypto');
var glob = require("glob");
var pathlib = require('path');

var dict = {};
var pattern = "530755bkup082911/**/*";
console.log("Walking directory tree ...");
var paths = glob.sync(pattern);
var length = paths.length;

var print = function(message) {
  process.stdout.write(message)
};

var md5sum = function(path, callback) {
  // the file you want to get the hash
  var fd = fs.createReadStream(path);
  var hash = crypto.createHash('md5');
  hash.setEncoding('hex');

  fd.on('end', function() {
    hash.end();
    var digest = hash.read();
    if (digest.length > 0)
      callback(digest);
  });
  // read all file and pipe it (write it) to the hash object
  fd.pipe(hash);
}

console.log("Examining "+length+" paths for dupes.");

paths.forEach(function(path, index){
  if ( fs.lstatSync(path).isDirectory() ) return;
  var name = pathlib.basename(path);
  md5sum(path, function(digest) {
    var suffix = "["+index+"/"+length+"] ["+name+"] "+path;
    if (dict[digest] === name)
      print("DUPE "+suffix);
    else {
      dict[digest] = name;
      print("UNIQUE "+suffix);
    }
    print("\n");
  });
});