more_complex_chunking.rb

require 'rubygems'
require 'httparty'

class MadMimi
  include HTTParty
  base_uri 'http://api.madmimi.com/'

  def self.import(username, api_key, csv_data)
    attempts = 0
    success = false
    while !success && attempts < 3 do
      result = post('/audience_members', :body => {
        :username => username,
        :api_key => api_key,
        :csv_file => csv_data.join("\n"),
      })
      attempts += 1
      if result[0..14] == "Invalid emails:"
        success = true
        File.open("invalid.txt", "a") do |file|
          file.puts(result)
        end
        puts "Success on attempt #{attempts}: #{result}" 
      elsif result.nil? || result.strip.blank?
        success = true
        puts "Success on attempt #{attempts}." 
      else
        puts "Failed on attempt #{attempts}." 
        File.open("failed.txt", "a") do |file|
          file.puts(result)
        end
      end
    end
    if !success
      File.open("failed.csv.txt", "a") do |file|
        file.puts(csv_data[1..-1].join("\n"))
      end
    end
  end
end

class Beef
  def initialize(args, defaults)
    @args = Hash.new("")
    @files = []
    
    defaults.each do |key, val|
      @args[key] = val
    end
    
    args.each do |arg|
      key, val = arg.split("=")
      if !val.nil?
        @args[key] = val
      else
        @files << key
      end
    end
  end
  
  def [](key)
    @args[key.to_s]
  end
end

args = Beef.new(ARGV, { "ignore" => "", "start" => "0", "chunk" => 250, "list" => "Support Upload", "path" => "." })
puts "ignoring #{args[:ignore]}, starting at #{args[:start]}, chunk size #{args[:chunk]}"

start = args[:start].to_i
chunk = args[:chunk].to_i
list = args[:list]
api_key = args[:api_key]
username = args[:username]
path = args[:path]

Dir.chdir("#{path}") do
  Dir.glob("*.csv").each do |filename|
    if args[:ignore].index(filename)
      puts "Ignoring #{filename}."
      next 
    end
    
    puts "Opening #{filename}..."
    
    file = File.open(filename, "r")
    header = file.gets.strip + ",add_list"

    count = 0
    csv_data = [ header ]
    
    if start > 0
      puts "Starting at #{start}"
      start.times do 
        file.gets
      end
      count = start
      start = 0
    end

    while(line = file.gets) do
      csv_data << line.strip + ",#{list}"
      count += 1
      if count % chunk == 0
        puts "Importing #{count}"
        MadMimi.import(username, api_key, csv_data)
        csv_data = [ header ]
      end
    end
    puts "Importing #{count}"
    MadMimi.import(username, api_key, csv_data)
  end
end

simple_chunking.rb

require 'rubygems'
require 'madmimi'

unless ARGV.size == 3
  puts "Usage: ruby db/data/import_chunked_list.rb <username> <api_key> <csv_file>"
  exit
end

username, api_key, csv_file = ARGV

CHUNK = 10000

lines = File.readlines(csv_file)
header = lines.shift
count = 0
total = lines.size

mimi = MadMimi.new username, api_key

lines.each_slice(CHUNK) do |slice|
  count += 1
  puts "Importing #{count * CHUNK} of #{total}"
  send_slice = header + slice.join('')
  mimi.csv_import send_slice
end