checkdist.rb

# parses output from the sql query
# reports the dedups
# finally reports a summary

packages = 0
hit_packages = 0
deletions = 0

STDIN.each do |line|
  dfid, data = line.chomp.split "\t", 2

  packages +=1

  m = data.gsub('][', "]\n[").split.inject({}) do |acc, r|
    r =~ %r{\[(.+?)\|(.+?)\]}
    src = $1
    dst = $2
    
    if src != dst && src !~ /http/
      acc[src] = dst
      deletions += 1
    end
    
    acc
  end

  unless m.empty?
    
    hit_packages += 1
    
    puts dfid
    m.each do |src,dst|
      puts "#{src} : #{dst}"
    end
    puts
  end
   
end

puts "summary"
puts "sample size: #{packages}"
puts "effected packages: #{hit_packages}"
puts "deletions: #{deletions}"