file_test.py

#!/usr/bin/python
# -*- coding: utf-8 -*-
import os
import codecs
import random
import string
from hashlib import md5
from multiprocessing import Pool
PROCESSES = 10
LENGTH = 8192*5+777
LINES = 100
FILENAME = 'test.log'
def randomstr(length):
    return ''.join(random.choice(string.lowercase) for i in range(length))
def write(_):
    with codecs.open(FILENAME, 'ab', encoding='utf-8', buffering=0) as f:
        rndstr = randomstr(LENGTH-2 - len(md5('x').hexdigest())) # -2 is to account for : and \n
        hash = md5(rndstr).hexdigest()
        line = rndstr + ':' + hash + '\n'
        for i in xrange(LINES):
            f.write(line)
def verify():
    wrong = 0
    lines = 0
    with codecs.open(FILENAME, 'rb', encoding='utf-8') as f:
        for l in f:
            lines += 1
            try:
                line, hash = l.strip('\n').split(':')
            except:
                wrong += 1
            if hash != md5(line).hexdigest():
                wrong += 1
        print('Line length {} total {} missing {} wrong {}'.format(LENGTH, lines, PROCESSES * LINES-lines, wrong))
if __name__ == '__main__':
    try:
        os.unlink(FILENAME)
    except:
        pass
    pool = Pool(PROCESSES)
    pool.map(write, [None] * PROCESSES)
    pool.close()
    pool.join()
    verify()