-
-
Save jschoch/5912418 to your computer and use it in GitHub Desktop.
| def fetch(path) do | |
| try do | |
| case HTTPotion.get(path, @user_agent) do | |
| Response[body: body, status_code: status, headers: _headers ] | |
| when status in 200..299 -> | |
| { :ok, body } | |
| {:error,:req_timedout} -> | |
| Lager.info "fetch: timeout detected, sleeping 30 seconds" | |
| :timer.sleep(30000) | |
| fetch(path) | |
| Response[body: body, status_code: status, headers: _headers ] -> | |
| Lager.error "fetch: something blew up #{path}\n#{status}" | |
| { :error, {status,body} } | |
| end | |
| catch | |
| value -> | |
| Lager.info inspect value | |
| end | |
| end | |
| def cfetch(url) do | |
| cfetch(url,:true) | |
| end | |
| def cfetch(url,:true) do | |
| hash = :erlang.phash(url,1000000000) | |
| file_name = "cache/#{hash}" | |
| case File.open(file_name,[:read,:utf8]) do | |
| {:ok,file} -> | |
| {:ok,html} = File.read(file_name) | |
| {html,:true} | |
| {:error,:enoent} -> | |
| Lager.info "cache miss for #{url}" | |
| cfetch(url,:false) | |
| {:error,reason} -> Lager.error inspect reason | |
| end | |
| end | |
| def cfetch(url,:false) do | |
| case fetch(url) do | |
| {:ok,html} -> | |
| cache_store(url,html) | |
| {html,:false} | |
| {reason,body} -> | |
| Lager.error "reason: #{reason} body: #{body}" | |
| :error | |
| end | |
| end |
19:49:08.656 [info] HTTPotion.HTTPError[message: "req_timedout"]
** (CaseClauseError) no case clause matching: :ok
/home/ec2-user/test/html_parsing/lib/html_parsing.ex:155: Tst.cfetch/2
/home/ec2-user/test/html_parsing/lib/html_parsing.ex:205: Tst.test_run2/2
/home/ec2-user/test/html_parsing/lib/html_parsing.ex:224: Tst."-test_run2/2-lc$^0/1-0-"/2
/home/ec2-user/test/html_parsing/lib/html_parsing.ex:217: Tst."-test_run2/2-lc$^0/1-0-"/2
/home/ec2-user/test/html_parsing/lib/html_parsing.ex:217: Tst.test_run2/2
/home/ec2-user/test/html_parsing/lib/html_parsing.ex:224: Tst."-test_run2/2-lc$^0/1-0-"/2
/home/ec2-user/test/html_parsing/lib/html_parsing.ex:217: Tst."-test_run2/2-lc$^0/1-0-"/2
/home/ec2-user/test/html_parsing/lib/html_parsing.ex:217: Tst.test_run2/2
** (HTTPotion.HTTPError) req_timedout
/home/ec2-user/test/html_parsing/lib/html_parsing.ex:8: Tst.fetch/1
/home/ec2-user/test/html_parsing/lib/html_parsing.ex:155: Tst.cfetch/2
/home/ec2-user/test/html_parsing/lib/html_parsing.ex:205: Tst.test_run2/2
/home/ec2-user/test/html_parsing/lib/html_parsing.ex:224: Tst."-test_run2/2-lc$^0/1-0-"/2
/home/ec2-user/test/html_parsing/lib/html_parsing.ex:217: Tst."-test_run2/2-lc$^0/1-0-"/2
/home/ec2-user/test/html_parsing/lib/html_parsing.ex:217: Tst.test_run2/2
/home/ec2-user/test/html_parsing/lib/html_parsing.ex:224: Tst."-test_run2/2-lc$^0/1-0-"/2