Skip to content

Instantly share code, notes, and snippets.

@jmn
Created September 19, 2019 20:00
Show Gist options
  • Save jmn/cc3e6a43fc3d061001f3db04f73c2b55 to your computer and use it in GitHub Desktop.
Save jmn/cc3e6a43fc3d061001f3db04f73c2b55 to your computer and use it in GitHub Desktop.
Concurrent elixir downloader with some fault toleration
defmodule Dl do
require Logger
def download(url) do
case HTTPoison.get(url) do
{:ok, response} ->
case response.status_code do
200 ->
{:ok, response.body}
_ ->
{:error, {response.status_code, response.request_url, response.body}}
end
end
end
def bd do
Logger.info("Starting")
["https://slashdot.org", "https://news.ycombinator.com", "http://google.com"]
|> Flow.from_enumerable()
|> Flow.map(&download/1)
|> Flow.partition()
|> Flow.map(fn result ->
case result do
{:ok, body} ->
{:ok, file} = File.open("out_path", [:write])
:ok = IO.binwrite(file, body)
:ok = File.close(file)
{:error, error} ->
Logger.error("Error: #{inspect error}")
_ ->
Logger.info("Error Unhandled")
end
end)
|> Flow.run()
end
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment