Testing async dispatch and http requests in Nim

s3_get_bucket_obj.nim

# testing async dispatch in pursuit of an efficient async s3 interface
import os, times, math, httpclient, asyncdispatch, asyncfile

let t0 = epochTime()

proc dt() : float = 
  round(epochTime() - t0,6)

const aws_url = "http://localhost:1234/"

proc s3_get_bucket_object(aws_key:string,bucket:string,obj:string) : Future[string] {.async.} = 
  var 
    client = newAsyncHttpClient()
    c_prog = false
  
  # do the HMAC here
  client.headers = newHttpHeaders({ "Authorization": "TEST KEY:SIG-"&obj })

  client.onProgressChanged = proc(total, progress, speed: BiggestInt) {.async.} =
    #echo("S3 download progress: ", progress, " of ", total)
    #echo("S3 transfer rate: ", speed div 1000, "kb/s")
    c_prog = true

  try:
    echo "get_content for ", obj, " ", dt()
    result = await client.getContent(aws_url)
    while not c_prog:
      poll()
  except HttpRequestError:
    echo "http error: "
    echo getCurrentExceptionMsg()
  except:
    echo "unknown exception"
    echo getCurrentExceptionMsg()


proc download_s3_file(aws_key, bucket, obj, output_path: string): Future[bool] {.async.} =
  let data = s3_get_bucket_object(aws_key, bucket, obj)
  var file = openAsync(output_path, fmReadWrite)
  echo "start file write ", output_path, " ", dt()
  await file.write(await data)
  echo "finish file write ", output_path, " ", dt()
  result = true
  
let bucket = "my_bucket"
var files: seq[Future[bool]] = @[]
for f in 0..9:
  let file_name = "./some_file_" & $(f+1) & ".txt"
  files.add(download_s3_file("blah", bucket, file_name, file_name))

var i=0
for file in files:
  echo "waiting for file ", i+1, " ", dt()
  discard waitFor(file)
  echo "finished for file ", i+1, " ", dt()
  inc i

echo "Transfer Complete ", dt()