duplicate_finder.py

import hashlib
from collections import defaultdict
from pathlib import Path
from typing import List


def find_duplicates(directory: Path) -> List[List[Path]]:
    fingerprint2paths = defaultdict(list)
    for path in directory.glob("**/*"):
        if not path.is_file():
            continue
        fingerprint = get_fingerprint(path)
        fingerprint2paths[fingerprint].append(path)
    return [paths for paths in fingerprint2paths.values() if len(paths) > 1]


def get_fingerprint(path: Path) -> str:
    hash_md5 = hashlib.md5()
    with open(path, "rb") as f:
        for chunk in iter(lambda: f.read(4096), b""):
            hash_md5.update(chunk)
    return hash_md5.hexdigest()


for duplicate_set in find_duplicates(Path(".")):
    print(duplicate_set)