Remove duplicate files

dups.py

#! /usr/bin/env python

import os
import sys
import hashlib


def md5(fname):
    hash_md5 = hashlib.md5()
    with open(fname, "rb") as f:
        for chunk in iter(lambda: f.read(4096), b""):
            hash_md5.update(chunk)
    return hash_md5.hexdigest()


def traverse(path, f):
    for root, dirs, files in os.walk(path):
        for name in files:
            f(os.path.join(root, name))


def remove_dup(hm, path):
    h = md5(path)
    if h in hm:
        print(path)
        os.remove(path)
    hm[h] = True


if __name__ == '__main__':
    hm = dict()
    if len(sys.argv) < 2:
        print("Removes duplicates. Usage:\n\tdups.py root")
    else:
        traverse((sys.argv[1]), lambda p: remove_dup(hm, p))