hash.py

import os
import hashlib
import argparse

MAX_HASH = 500

def dump(hashes, f):
    as_str = "\n".join(["{},{},{}".format(*data) for data in hashes])
    f.write(as_str + "\n")
    hashes.clear()
    print("Dumped 500")

def md5(fname):
    hash_md5 = hashlib.md5()
    count = 0
    with open(fname, "rb") as f:
        for chunk in iter(lambda: f.read(4096), b""):
            count += len(chunk)
            hash_md5.update(chunk)
    return hash_md5.hexdigest(), count


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Build file hash table')
    parser.add_argument('root', type=str, help='directory to iter from')

    args = parser.parse_args()

    hashes = []
    with open("output.txt", "w") as outputf:
        for root, dirs, filenames in os.walk(args.root):
            for filename in filenames:
                path = os.path.join(root, filename)
                hash, size= md5(path)
                hashes.append((hash, size, path))

                if len(hashes) == MAX_HASH:
                    dump(hashes, outputf)

        dump(hashes, outputf)