read_logs.rb

#!/usr/bin/env ruby -wKU

results = ""
attempted = 0
extracted = 0
exported  = 0
failed    = 0
failed_docs = []
total_failures = []

Dir.glob('*').each do |file|
  if Dir.exists? file
    Dir.glob( File.join(file, 'logs', '*.log') ).each do |log_file|
      puts "Reading #{log_file}"
      capture = false
      log = File.new log_file, 'r'
      log.each_line do |line|
        total_failures << log_file if line.include? 'More than 100 errors ocurred'
        results << line.split('-- : ')[1] if capture
        capture = true if line.include? 'Generating Status Email'
        break if line.include? 'Finished in'
        break if line.include?( 'following documents failed' ) and total_failures.include?( log_file )
      end
      log.close
    end
  end
end

puts results if $DEBUG
results.split("\n").each do |line|
  attempted += line.split(' docs ')[0].strip.to_i if line.include? 'attempted'
  extracted += line.split(' docs ')[0].strip.to_i if line.include? 'extracted'
  exported += line.split(' docs ')[0].strip.to_i if line.include? 'exported'
  failed_docs << line.split( 'Doc => ' )[1].split(' ')[0] if line.include? 'Doc =>'
end
failed = attempted - (extracted + exported)
log = File.new 'process_results.log', 'w'
log << "  Attempted: #{attempted}\n"
log << "  Extracted: #{extracted + exported}\n"
log << "  Failed: #{failed}\n"

if total_failures.count > 0
  log << "The following extracts are total failure and need to be recovered in whole:\n\n"
  total_failures.each { |f| log << "  #{f}\n" }
end

if failed > 0
  log << "\n  Error Docs:\n"
  failed_docs.each { |d| log << "    #{d}\n" }
  log << "\n  SQL Criteria: '#{failed_docs.join("', '")}'"
end

log.close