Script to load all ngrams as keys to redis

loading_data_0.py

import time
import os
import redis
path = '/path/to/unigrams'
client = redis.Redis(host = 'host-ip-here', port = 6385, db = 0)
pipeline = client.pipeline(transaction = False)
for f in os.listdir(path):
    
    print ' starting with file ', f
    start = time.time()
    with os.popen("zcat " + os.path.join(path,f)) as ngram_file:
        for index, line in enumerate(ngram_file):
            parts = line.strip().split('\t')
            word, year, raw_count, volume_count = parts
            pipeline.hset(word, year, (raw_count, volume_count))
            if index > 0 and index%10000 == 0:
                pipeline.execute()
        print ' file ', f, 'is is done in ', time.time() - start, ' seconds '
        print ' key_size now is ', client.dbsize()