Skip to content

Instantly share code, notes, and snippets.

@astrada
Created June 24, 2019 20:28
Show Gist options
  • Save astrada/9ea1128754be4385b530f5a5f3b42c69 to your computer and use it in GitHub Desktop.
Save astrada/9ea1128754be4385b530f5a5f3b42c69 to your computer and use it in GitHub Desktop.
let with_session
config
curl_state
interact =
let curl_session = GapiCurl.init curl_state in
let writer data = String.length data in
GapiCurl.set_writefunction writer curl_session;
let cleanup () = ignore (GapiCurl.cleanup curl_session) in
let session =
{ GapiConversation.Session.curl = curl_session;
config = config;
cookies = [];
auth = GapiConversation.Session.NoAuth;
etag = ""
}
in
try
let result = interact session in
cleanup ();
result
with e ->
cleanup ();
raise e
let cleanup c =
Printf.printf "Cleanup\n%!";
ignore (GapiCurl.cleanup c)
let with_session_cleanup_gc
config
curl_state
interact =
let curl_session = GapiCurl.init curl_state in
let writer data = String.length data in
GapiCurl.set_writefunction writer curl_session;
Gc.finalise cleanup curl_session;
let session =
{ GapiConversation.Session.curl = curl_session;
config = config;
cookies = [];
auth = GapiConversation.Session.NoAuth;
etag = ""
}
in
interact session
let do_request interact =
let state = GapiCurl.global_init () in
(*with_session_cleanup_gc*)
with_session
GapiConfig.default
state
interact;
ignore (GapiCurl.global_cleanup state)
let pull_one url =
Printf.printf "Perform(%s)\n%!" url;
do_request
(fun session ->
GapiCurl.perform url session.GapiConversation.Session.curl |> ignore)
let () =
let urls = [|
"https://lastfm-img2.akamaized.net/i/u/300x300/c0abe2b16b6c49c3a240af8af1457983.png";
"https://lastfm-img2.akamaized.net/i/u/300x300/2995065b3f554a1bca444c29c0fb8530.png";
"https://lastfm-img2.akamaized.net/i/u/300x300/c0abe2b16b6c49c3a240af8af1457983.png";
"https://lastfm-img2.akamaized.net/i/u/174s/89e3eba4f1364816b86632248b7822c9.png";
"https://lastfm-img2.akamaized.net/i/u/300x300/5d3e2c10541441d8c174e5568f109b3c.png";
"https://lastfm-img2.akamaized.net/i/u/300x300/c0abe2b16b6c49c3a240af8af1457983.png";
"https://lastfm-img2.akamaized.net/i/u/300x300/c0abe2b16b6c49c3a240af8af1457983.png";
"https://lastfm-img2.akamaized.net/i/u/174s/c7b0885750f94e46a36fa41ed10691ec.png";
"https://lastfm-img2.akamaized.net/i/u/300x300/dd70b650e5cd479bc7078ade2420e2d6.png";
"https://lastfm-img2.akamaized.net/i/u/300x300/c0abe2b16b6c49c3a240af8af1457983.png";
"https://lastfm-img2.akamaized.net/i/u/300x300/c0abe2b16b6c49c3a240af8af1457983.png";
"https://lastfm-img2.akamaized.net/i/u/174s/5c84b957396c4cbdb61f22b92fc71558.png";
"https://lastfm-img2.akamaized.net/i/u/300x300/a0ece142922e4750a065e2304043c681.png";
"https://lastfm-img2.akamaized.net/i/u/300x300/c0abe2b16b6c49c3a240af8af1457983.png";
"https://lastfm-img2.akamaized.net/i/u/174s/7a08394b697945b9bfa795dfeed46921.png";
"https://lastfm-img2.akamaized.net/i/u/300x300/5d2211dbc3aa40e28e28dd2814bd318a.png";
"https://lastfm-img2.akamaized.net/i/u/300x300/c7b0885750f94e46a36fa41ed10691ec.png";
"https://lastfm-img2.akamaized.net/i/u/300x300/c0abe2b16b6c49c3a240af8af1457983.png";
"https://lastfm-img2.akamaized.net/i/u/300x300/666831aed9367cc366140dd2ffc0169e.png";
"https://lastfm-img2.akamaized.net/i/u/300x300/c0abe2b16b6c49c3a240af8af1457983.png";
"https://lastfm-img2.akamaized.net/i/u/174s/df3a1356b08e464ca51b1aa292a5c38a.png";
"https://lastfm-img2.akamaized.net/i/u/300x300/a5943f15da8880f426be5f0e8d815b2c.png";
"https://lastfm-img2.akamaized.net/i/u/300x300/c0abe2b16b6c49c3a240af8af1457983.png";
"https://lastfm-img2.akamaized.net/i/u/300x300/02c05615600743628eca3bd9df4c8a4c.png";
"https://lastfm-img2.akamaized.net/i/u/300x300/89e3eba4f1364816b86632248b7822c9.png";
"https://lastfm-img2.akamaized.net/i/u/300x300/c0abe2b16b6c49c3a240af8af1457983.png";
"https://lastfm-img2.akamaized.net/i/u/174s/407413a4bb614f1db08f1110a604f5df.png";
"https://lastfm-img2.akamaized.net/i/u/300x300/1c27be1170c04612c6b40362bb2ca60b.png";
"https://lastfm-img2.akamaized.net/i/u/300x300/29b50a52515d422abeafc3c01b3e2fab.png";
"https://lastfm-img2.akamaized.net/i/u/300x300/c0abe2b16b6c49c3a240af8af1457983.png";
"https://lastfm-img2.akamaized.net/i/u/174s/db766dc7b43d4aedb6a9fbea33980e71.png";
"https://lastfm-img2.akamaized.net/i/u/300x300/ce627d7cda714050c0e68b02defcd7d5.png";
"https://lastfm-img2.akamaized.net/i/u/300x300/c42b09eb599449639cdf0201ac80c5c9.png";
"https://lastfm-img2.akamaized.net/i/u/300x300/c0abe2b16b6c49c3a240af8af1457983.png";
"https://lastfm-img2.akamaized.net/i/u/300x300/14737855b0474575c53f259f39cd22ad.png";
"https://lastfm-img2.akamaized.net/i/u/300x300/d443f85ebdb24bf8cba7be75cc9cb0e2.png";
"https://lastfm-img2.akamaized.net/i/u/300x300/c0abe2b16b6c49c3a240af8af1457983.png";
"https://lastfm-img2.akamaized.net/i/u/300x300/b3061e8a3d2043ebc34a2a8c427a4e6f.png";
"https://lastfm-img2.akamaized.net/i/u/300x300/0254a8a3201a42ad95ffa6f64ec3fa3b.png";
"https://lastfm-img2.akamaized.net/i/u/300x300/c0abe2b16b6c49c3a240af8af1457983.png";
"https://lastfm-img2.akamaized.net/i/u/300x300/89e3eba4f1364816b86632248b7822c9.png";
|] in
Curl.global_init Curl.CURLINIT_GLOBALALL;
Printf.printf "Starting\n%!";
read_line () |> ignore;
for i = 0 to (Array.length urls - 1) do
Printf.printf "urls[%d]=%s\n%!" i urls.(i);
pull_one urls.(i);
done;
Printf.printf "Done\n%!";
read_line () |> ignore;
Printf.printf "Global cleanup\n%!";
Curl.global_cleanup ();
read_line () |> ignore;
Printf.printf "GC.full_major\n%!";
Gc.full_major ();
read_line () |> ignore
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment