A simple concurrent crawler in Haskell.

gistfile1.hs

{-# OPTIONS_GHC -Wall -O2 -threaded -with-rtsopts="-N" #-}

import Control.Concurrent
import Control.Concurrent.Async.Pool
import Data.ByteString.Lazy hiding (replicate)
import Network.HTTP.Client
import Network.HTTP.Client.TLS

getConcurrently :: [String] -> IO [ByteString]
getConcurrently urls = withManager tlsManagerSettings $ \mgr ->
    withTaskGroup 8 $ \tg -> mapConcurrently tg (\url -> do
        req <- parseUrl url
        responseBody <$> httpLbs req mgr <* threadDelay 0) urls

main :: IO ()
main = getConcurrently (replicate 1024 "https://httpbin.org/get") >>= print