File system read vs CSV iteration vs Redis cache read vs Memcached read

min_max_avg.py

import sys

import numpy as np


def get_reads_and_writes(lines, wanted):
    result = []
    for line in lines:
        _, name, value = line.split(':')
        if name == wanted:
            result.append(float(value))
    return result


def main(numbers):
    return min(numbers), max(numbers), np.mean(numbers)


if __name__ == '__main__':
    lines = sys.stdin.read().strip().split('\n')
    reads = get_reads_and_writes(lines, 'test_read')
    print 'Reads: min %.5f, max %.5f, mean %.5f' % main(reads)
    writes = get_reads_and_writes(lines, 'test_write')
    print 'Writes: min %.5f, max %.5f, mean %.5f' % main(writes)

scenario.md

Test scenario

Environment info:

$ python --version
Python 2.7.10
$ pip freeze |grep -E 'redis|memcached'
python-memcached==1.59
redis==2.10.6

Test data:

$ ll *.csv
-rw-r--r--@ 1 nesusvet  staff   500M 19 авг 12:33 big.csv
-rw-r--r--  1 nesusvet  staff   1,4M 26 авг 11:01 small.csv
$ head -n 1 big.csv
Brent,60,20170813,230000,52.0200000,52.0500000,51.9700000,52.0500000,203

Redis

$ docker run --name test-redis -p 6379:6379 -d redis
$ for i in {1..10}; do python test_read.py redis big.csv; done
$ cat test.log | python min_max_avg.py
Reads: min 4.76063, max 7.77215, mean 6.07756
Writes: min 3.81259, max 5.38883, mean 4.62716
$ rm test.log
$ for i in {1..10}; do python test_read.py redis small.csv; done
$ cat test.log | python min_max_avg.py
Reads: min 0.01125, max 0.02440, mean 0.01428
Writes: min 0.01470, max 0.03029, mean 0.01910

Memcached

$ docker run --name test-memcached -p 11211:11211 -d memcached memcached -m 1024 -I $((512 * 1024 * 1024))
$ for i in {1..10}; do python test_read.py memcached big.csv; done
$ cat test.log | python min_max_avg.py
Reads: min 3.03730, max 6.46400, mean 3.94183
Writes: min 3.71722, max 5.98671, mean 4.24129
$ rm test.log
$ for i in {1..10}; do python test_read.py memcached small.csv; done
$ cat test.log | python min_max_avg.py
Reads: min 0.01060, max 0.02742, mean 0.02015
Writes: min 0.01181, max 0.02065, mean 0.01472

test_read.py

"""
Uasge:
    python test_read.py <storage_name> <filename>

Storage name must be one of `redis` or `memcached` or `memcached-compressed`
"""

import csv
import logging
import sys
import time

import redis
import memcache

logging.basicConfig(
    level=logging.DEBUG,
    filename='test.log',
)


def timer(fn):
    def wrapper(*args, **kwargs):
        logger = logging.getLogger(fn.__name__)
        start = time.time()
        result = fn(*args, **kwargs)
        speed = time.time() - start
        logger.debug('%s', speed)
        return result

    return wrapper


@timer
def file_read(filename):
    with open(filename) as f:
        return f.read()


def csv_read(filename):
    with open(filename) as f:
        reader = csv.reader(f)
        for row in reader:
            yield row


@timer
def test_write(client, key, data, **kwargs):
    return client.set(key, data, **kwargs)


@timer
def test_read(client, key):
    return client.get(key)


if __name__ == '__main__':
    if len(sys.argv) < 3:
        print __doc__
        exit(1)

    filename = sys.argv[2]
    text = file_read(filename)

    name = sys.argv[1]
    if name == 'redis':
        client = redis.Redis()
        test_write(client, filename, text)
        assert test_read(client, filename) == text

    elif name == 'memcached':
        client = memcache.Client(['localhost'], server_max_value_length=len(text))
        data = list(csv_read(filename))
        assert test_write(client, filename, data) is True
        assert test_read(client, filename) == data

    elif name == 'memcached-compressed':
        client = memcache.Client(['localhost'], server_max_value_length=len(text))
        assert test_write(client, filename, text, min_compress_len=1024) is True
        assert test_read(client, filename) == text