zd_massive_full_load.rb

require 'rubygems'
require 'gooddata'
require 'pp'
require 'json'
require 'csv'
require 'pmap'
require 'set'
require 'optparse'
require 'yaml'

DONE = 'SYNCHRONIZED'
WAITING = 'Another process is already running'
UNKNOWN = 'Another process'
TRYING = 'Trying to start'
ERROR = 'ERROR'

GoodData.logger = GoodData::NilLogger.new

begin
  options = YAML.load_file('zd_config.yaml')
rescue
  options = {}
end
  
OptionParser.new do |opts|
  opts.banner = "Usage: ruby zd_massive_full_load.rb [options]"
  opts.on('-u', '--username EMAIL', 'Username') { |v| options['username'] = v }
  opts.on('-p', '--password PASSWORD', 'Password') { |v| options['password'] = v }
  opts.on('-i', '--input FILE', 'Input File') { |v| options['input'] = v }
  opts.on('-o', '--output FILE', 'Output File') { |v| options['output'] = v }
end.parse!

client = GoodData.connect(options['username'], options['password'], {:webdav_server => "https://na1-di.gooddata.com",:server => "https://analytics.zendesk.com"})
puts "Logged in as #{options['username']}."

def read_input_csv_array(path)
	res = []
	CSV.foreach(path, :headers => false, :return_headers => false) do |row|
		res << row[0]
	end
	return res
end

def read_input_csv_hash(path)
	res = {}
  p = {}
	CSV.foreach(path, :headers => false, :return_headers => false) do |row|
		res[row[0]] = row[1]
    p[row[0]] = row[2]
	end
	return res, p
end

input = read_input_csv_array(options['input'])
process, status = read_input_csv_hash(options['output'])
output = []
started = {}

begin
  input.each do |pid|
    if not process.has_key?(pid) or process[pid] == WAITING  # full load was not started yet 
      begin
        response = client.post("/gdc/projects/#{pid}/connectors/zendesk4/integration/processes", '{"process": {"incremental": false}}')
        p = response['uri']
        process[pid] = p
        status[pid] = TRYING
      rescue RestClient::BadRequest => e  # 400 Bad Request: Another process is running, cannot start full load
        process[pid] = WAITING
        response = client.get("/gdc/projects/#{pid}/connectors/zendesk4/integration")
        begin
          status[pid] = "#{UNKNOWN}: #{response['integration']['runningProcess']['status']['code']}"
        rescue
          status[pid] = UNKNOWN
        end
      rescue RestClient::Conflict => e
        process[pid] = WAITING
        status[pid] = "CONFLICT"
      end
    elsif status[pid] != DONE and status[pid] != ERROR  # full load running, update status
      begin
        response = client.get(process[pid])
        s = response["process"]["status"]["code"]
        status[pid] = s
        started[pid] = response["process"]["started"]
      rescue  # status unknown, have to restart the process
        process[pid] = WAITING
        status[pid] = "RESTART"
      end
    else  # full load completed or failed, skip project
    end
  end  
ensure
  CSV.open(options['output'], 'w') do |outputfile|
    input.each do |pid|
      outputfile << [pid, process[pid], status[pid]]
    end
  end
end

status = {}
CSV.foreach(options['output'], :headers => false, :return_headers => false) do |row|
  key = row[2]
  if status.has_key?(key)
    status[key] += 1
  else
    status[key] = 1
  end
end
status.each do |s|
  puts "#{s}"
end

puts "Downloading started at:"
started.sort_by(&:last).each do |p|
  puts "#{p[0]} #{p[1]}"
end