SQS + Boto + Eventlet, re http://gist.github.com/434053

gistfile2.txt

(jd@XXX) /home/jd/shovel> time ./shovel.py jd_test
got 901 messages

real    0m1.617s
user    0m0.558s
sys     0m0.044s

shovel.py

#!/bin/env python26
import eventlet
import os
import sys


sqs_conn = eventlet.import_patched('boto.sqs.connection')
AWS_KEY = os.environ.get("AWS_KEY")
AWS_SECRET = os.environ.get("AWS_SECRET")


def read_message(queue_name, num_messages=10):
    sqs = sqs_conn.SQSConnection(AWS_KEY, AWS_SECRET)
    q = sqs.get_queue(queue_name)
    return q.get_messages(num_messages)


if __name__ == '__main__':
    queue_name = sys.argv[1]
    
    pool = eventlet.GreenPool(100)
    count = 0
    for result in pool.imap(read_message, [queue_name for x in xrange(0,100)]):
        count += len(result)
    print 'got %d messages' % count