Created
January 3, 2022 14:47
-
-
Save bluzky/b16dbe415ea0aea8dd9dbcb71d9225c6 to your computer and use it in GitHub Desktop.
Stream download large file from URL in Elixir
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
defmodule Toolkit.DownloadHelper do | |
@moduledoc """ | |
Stream download large file from url | |
""" | |
require Logger | |
def stream_download(url, save_path) do | |
Stream.resource( | |
fn -> begin_download(url) end, | |
&continue_download/1, | |
&finish_download/1 | |
) | |
|> Stream.into(File.stream!(save_path)) | |
|> Stream.run() | |
end | |
defp begin_download(url) do | |
{:ok, _status, headers, client} = :hackney.get(url) | |
headers = Enum.into(headers, %{}) | |
total_size = headers["Content-Length"] |> String.to_integer() | |
{client, total_size, 0} | |
end | |
defp continue_download({client, total_size, size}) do | |
case :hackney.stream_body(client) do | |
{:ok, data} -> | |
new_size = size + byte_size(data) | |
{[data], {client, total_size, new_size}} | |
:done -> | |
{:halt, {client, total_size, size}} | |
{:error, reason} -> | |
raise reason | |
end | |
end | |
defp finish_download({client, total_size, size} = cl) do | |
Logger.debug("Complete download #{size} bytes") | |
nil | |
end | |
end |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment