process.ex

 def try_async(files, num_process) do
    files 
      |> Enum.chunk_every(round(length(files)/num_process))
      |> Enum.each(fn(x) -> load_async(x, 3) end)
  end

  #Break that chunk into a few tasks to run parallel
  def load_async(files, chunk_by) do
    files 
      |> Enum.chunk_every(chunk_by)
      |> Enum.map(&Task.async(AdLock.Processor , :process_job, [&1]))
      |> Enum.map(&Task.await(&1))
  end

  def process_job(files) do
    files
      |> Enum.each(&process_file/1)
  end

  #process the file into the database
  def process_file(filename) do
    IO.puts "processing #{filename}"
    #:timer.sleep(1000)
    File.stream!(filename, [:read, :compressed])
      |> Stream.drop(1)
      |> CSV.decode(headers: [:identifier,	:identifier_type,	:timestamp)
      |> Enum.each(&process_record/1)
  end