Redis set performance on my local computer.

redissetperformance.py

import timeit
import random

from itertools import izip_longest

import redis

securerandom = random.SystemRandom()

r = redis.StrictRedis(host='localhost', port=6379, db=0)

all_records = ['abc%0100d' % i for i in xrange(500000)]

records_with_misses = set(all_records)
records_with_misses.difference_update(securerandom.sample(all_records, 10000))


def grouper(iterable, n, fillvalue=None):
    "Collect data into fixed-length chunks or blocks"
    # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx
    args = [iter(iterable)] * n
    return izip_longest(fillvalue=fillvalue, *args)


def benchmark_set():
    r.delete('set-key-poll', 'set-key-match')
    for group in grouper(records_with_misses, 5000):
        r.sadd('set-key-poll', *[x for x in group if x is not None])

    for group in grouper(all_records, 5000):
        r.sadd('set-key-match', *[x for x in group if x is not None])

    missing = r.sdiff('set-key-match', 'set-key-poll')

    r.delete('set-key-poll', 'set-key-match')


def benchmark_list():
    r.delete('list-key-poll')
    for group in grouper(records_with_misses, 5000):
        r.rpush('list-key-poll', *[x for x in group if x is not None])

    missing = set(all_records) - set(r.lrange('list-key-poll', 0, -1))

    r.delete('list-key-poll')

print 'set'
print timeit.timeit(benchmark_set, number=10)

print 'list'
print timeit.timeit(benchmark_list, number=10)

# output:
# set
# 37.1569800377
# list
# 33.0010311604

# also set version pinned CPU 50/50 python and redis, list version 10% redis/90% python