Elixir download/stream large file with hackney

request_helper.ex

defmodule RequestHelper do
  @moduledoc """
  Reference from https://gist.github.com/avdi/7990684
  Stream download large file from url
  """
  require Logger

  @doc """
  Get stream data from url
  mode could be `:binary` or `:line`
  """
  def stream_url(url) do
    Stream.resource(
      fn -> begin_download(url) end,
      &continue_download/1,
      &finish_download/1
    )
  end

  def stream_url(url, :line) do
    stream_url(url)
    |> Stream.concat([:end])
    |> Stream.transform("", fn
      :end, "" ->
        {[], ""}
        
      :end, prev ->
        {[prev], ""}

      chunk, prev ->
        [last_line | lines] =
          String.split(prev <> chunk, "\n")
          |> Enum.reverse()

        {Enum.reverse(lines), last_line}
    end)
  end

  @doc """
  Stream large file and save to save_path
  """
  def download(url, save_path) do
    stream_url(url)
    |> Stream.into(File.stream!(save_path))
    |> Stream.run()
  end

  defp begin_download(url) do
    {:ok, _status, headers, client} = :hackney.get(url)
    headers = Enum.into(headers, %{})
    total_size = headers["Content-Length"] |> String.to_integer()

    {client, total_size, 0}
  end

  defp continue_download({client, total_size, size}) do
    case :hackney.stream_body(client) do
      {:ok, data} ->
        new_size = size + byte_size(data)
        {[data], {client, total_size, new_size}}

      :done ->
        {:halt, {client, total_size, size}}

      {:error, reason} ->
        raise reason
    end
  end

  defp finish_download({client, total_size, size}) do
    :hackney.close(client)
    Logger.debug("Complete download #{size} / #{total_size} bytes")
    nil
  end
end

csv_url =
  "https://support.staffbase.com/hc/en-us/article_attachments/360009197011/username-password-recovery-code.csv"


Mix.install([
  {:hackney, "~> 1.0"},
  {:csv, "~> 2.4"},
  {:benchee, "~> 1.0", only: :dev}
])

RequestHelper.stream_url(csv_url, :line)
|> CSV.decode!(separator: ?;)
|> Enum.to_list()
|> IO.inspect()