Skip to content

Instantly share code, notes, and snippets.

@bvjebin
Last active August 31, 2018 13:32
Show Gist options
  • Save bvjebin/5481fd20179da953aa6b0b6f7adf6ed8 to your computer and use it in GitHub Desktop.
Save bvjebin/5481fd20179da953aa6b0b6f7adf6ed8 to your computer and use it in GitHub Desktop.
export_images_to_zip for medium post
def export_images_to_zip(data, path) do
images_to_download = Enum.map(data, fn datum ->
%{url: datum.image, name: datum.name}
end)
case images_to_download do
[] -> nil
list ->
zip_path = path<>"/images"
:ok = File.mkdir_p!(zip_path)
download_images_in_batch(list, zip_path)
end
end
def download_images_in_batch(list_of_images, zip_path) do
list_of_images
|> Stream.chunk_every(5, 5, [])
|> Stream.each(fn item ->
download_images(item, zip_path)
end)
|> Enum.to_list
end
def download_images(list_of_images, zip_path) do
try do
list_of_images
|> Enum.map(&Task.async(fn -> download_image(&1) end))
|> Enum.map(&Task.await(&1, 50000))
|> Enum.map(fn item ->
case item do
{:ok, name, body} ->
file_path = zip_path<>"/"<>name
File.write!(file_path, body)
file_path
{:error, error} ->
IO.inspect error
nil
end
end)
catch
:exit, error ->
Logger.error "Error while downloading batches #{inspect error}"
end
end
def download_image(%{url: url, name: name}) do
case HTTPotion.get url, [timeout: 50000] do
%HTTPotion.Response{body: body,status_code: 200} ->
{:ok, name<>"."<>get_image_extension(url), body}
%HTTPotion.ErrorResponse{message: message} ->
{:error, message}
error ->
{:error, error}
end
end
defp get_image_extension(url) do
String.split(url, "/") |> List.last() |> String.split(".") |> List.last()
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment