文件去重复

check_duplicates.rb

require 'digest/md5'

Dir['**/*'].
  select {|f| File.file? f }.
  group_by {|f| Digest::MD5.hexdigest(File.binread(f)).tap {|md5| STDERR.puts "#{f}: #{md5}" } }.
  reject {|_, list| list.size == 1 }.
  each {|_, v| puts "---\n#{v.join("\n")}" }

rm_duplicates.rb

require 'digest/md5'
require 'fileutils'
require 'awesome_print'

fail "Usage: $0 remained targets..." if ARGV.size < 2

filter_proc = ->(path) { File.file? path  }
map_proc = ->(map, path) { map.merge path => Digest::MD5.hexdigest(File.binread(path)).tap {|md5| puts "#{path}: #{md5}" } }

remained = Dir["#{ARGV.shift}/**/*"].select(&filter_proc).inject({}, &map_proc)
targets = ARGV.map {|target| Dir["#{target}/**/*"] }.flatten.select(&filter_proc).inject({}, &map_proc)

list = remained.inject({}) do |map, (path, md5)|
  to_delete = targets.select {|_, t_md5| md5 == t_md5 }.map {|t_path, _| t_path }
  next map if to_delete.empty?
  map.merge path => to_delete
end

STDERR.puts list.ai
STDERR.puts 'Confirm?'
if STDIN.gets.strip.downcase == 'y'
  list.values.each do |path| 
    begin
      FileUtils.rm path
    rescue
      STDERR.puts "Failed to delete #{path}: #{$!.message}"
    end
  end
end