async getter

asyncgetter.nim

import asyncdispatch,  httpclient, os, strutils

# ~/nimbb/nim/bin/nim c --threadAnalysis:off -d:ssl -r tcrawl3.nim 10 300 500
 
var numRunning = 0

var good = 0
var bad = 0
# not used....
var concurrency = 3
if paramCount() > 0:
  concurrency = paramStr(1).parseInt()

var sleepTime = 300
if paramCount() > 1:
  sleepTime = paramStr(2).parseInt()

var numToRun = 30
if paramCount() > 2:
  numToRun = paramStr(3).parseInt()


var totGot = 0
 
proc createRequest(url: string) {.async.} =
  numRunning += 1
  echo "numRunning: ", numRunning
  var client = newAsyncHttpClient()
  try:
    let resp = await client.request(url)
    echo("Got resp: ", resp.status)
    good += 1
    echo "good: ", good

  except:
    bad += 1
    echo "bad: ", bad
    echo("Exception!")
  numRunning -= 1
  totGot += 1
  echo "got: ", totGot

# 1000 domains
var domurl = "http://goo.gl/yzlQ5n"
var doms = getContent(domurl).split("\n")
echo doms
#quit()

proc main() {.async.} =
  for x in low(doms)..numToRun:
    asyncCheck createRequest("http://www." & doms[x])
  echo "all running"
  while numRunning > 0:
    echo "sleeping: ", sleepTime, " running: ", numRunning
    await sleepAsync(sleepTime)

when isMainModule:    
  waitFor main()