gistfile1.rb

namespace :responsys do

  desc 'download and import Responsys events from FTP'
  task load: :environment do
    Resque.enqueue(LoadEventsJob)
  end

  Our Fields, unique key
  RIID, (CUSTOMER_ID_, EMAIL_)
  RIID, (CUSTOMER_ID_, EMAIL_)


  #1
  999, [email protected], 1
  998, [email protected], 1 #will delete

  #2
  999, [email protected], 1
  998, [email protected], 1 #will delete
  998, [email protected], 1 #will delete

  #3
  999, [email protected], 1
  998, [email protected], 2 
  997, [email protected], 2 #will delete
  996, [email protected], 2 #will delete

  #4
  997, [email protected], 2
  999, [email protected], NULL #will delete
  998, [email protected], 1


  desc 'remove duplicate responsys contact records'
  task dedup: :environment do
    ResponsysInteract::Client.session do |client|

      json = File.new(Rails.root.join("dups.json")).read
      groups = JSON.parse(json)
      groups.delete("EMAIL")
      puts "Processing #{groups.count} duplicate emails"

      "[email protected]": [1234, 234234]

      fetched = Hash[groups.map do |email, riids|
        puts "Retrieving Contact Data: #{email}"
        begin
          [ email , riids.map { |riid| client.retrieve_contact_by_riid(riid.to_i).body[:retrieve_list_members_response][:result][:record_data][:records][:field_values] } ]
        rescue Savon::SOAPFault => e
          [nil, nil]
        end
      end]

      "[email protected]": [[RIID, EMAIL, CUSTOMER_ID_], [RIID, EMAIL, CUSTOMER_ID_]]

      puts
      puts

      fetched.each do |email, rows|
        next unless email && rows
        if rows.count > 1
          if rows.all? { |r| !r[3].nil? }
            if rows.uniq { |r| r[3] }.count == 1
              puts email + " has multiple entries for the same CUSTOMER_ID_"
              all_but_newest = rows.sort_by { |r| r[0] }[0..-2]
            begin
              all_but_newest.each { |r| puts client.delete_contact_by_riid(r[0]) }
            rescue Savon::SOAPFault => e
              puts "Failed on: #{email}"
              puts e
            end
            end
          end
        end
      end

    end
  end
end