Fast duplicate file finder written in Python (3)

duplicates.py

#!/usr/bin/env python3
"""
Fast duplicate file finder.
Usage: duplicates.py <folder> [<folder>...] [--output <path>]
Based on https://stackoverflow.com/a/36113168/300783 and https://gist.github.com/tfeldmann/fc875e6630d11f2256e746f67a09c1ae
Modified for Python3 with some small code improvements.
"""
import argparse
import hashlib
import os
from collections import defaultdict


def chunk_reader(fobj, chunk_size=1024):
    """Generator that reads a file in chunks of bytes."""
    while True:
        chunk = fobj.read(chunk_size)
        if not chunk:
            return
        yield chunk


def get_hash(filename, first_chunk_only=False, hash_algo=hashlib.sha256):
    hashobj = hash_algo()
    with open(filename, "rb") as f:
        if first_chunk_only:
            hashobj.update(f.read(1024))
        else:
            for chunk in chunk_reader(f):
                hashobj.update(chunk)
    return hashobj.digest()


def check_for_duplicates(paths, output_path=None):
    files_by_size = defaultdict(list)
    files_by_small_hash = defaultdict(list)
    files_by_full_hash = dict()
    duplicates = []

    for path in paths:
        for dirpath, _, filenames in os.walk(path):
            for filename in filenames:
                full_path = os.path.join(dirpath, filename)
                try:
                    # if the target is a symlink (soft one), this will
                    # dereference it: change the value to the actual target file
                    full_path = os.path.realpath(full_path)
                    file_size = os.path.getsize(full_path)
                except OSError:
                    # not accessible (permissions, etc): pass on
                    continue
                files_by_size[file_size].append(full_path)

    # for all files with the same file size, get their hash on the first 1024 bytes
    for file_size, files in files_by_size.items():
        if len(files) < 2:
            continue  # this file size is unique, no need to spend cpu cycles on it, lemme play minecraft plz

        for filename in files:
            try:
                small_hash = get_hash(filename, first_chunk_only=True)
            except OSError:
                # the file access might've changed till the exec point got here
                continue
            files_by_small_hash[(file_size, small_hash)].append(filename)

    # for all files with the hash on the first 1024 bytes, get their hash on the full
    # file: collisions will be duplicates
    for files in files_by_small_hash.values():
        if len(files) < 2:
            # the hash of the first 1kb is unique, skip this file
            continue

        for filename in files:
            try:
                full_hash = get_hash(filename, first_chunk_only=False)
            except OSError:
                # the file access might've changed till the exec point got here
                continue

            if full_hash in files_by_full_hash:
                duplicate = files_by_full_hash[full_hash]
                duplicates.append((filename, duplicate))
                print("Duplicate found:\n - %s\n - %s\n" % (filename, duplicate))
            else:
                files_by_full_hash[full_hash] = filename

    if output_path:
        with open(output_path, "w", encoding="utf-8") as output_file:
            for filename, duplicate in duplicates:
                output_file.write(f"{filename}\n{duplicate}\n\n")

    return duplicates


def parse_args():
    parser = argparse.ArgumentParser(description="Find duplicate files without deleting anything.")
    parser.add_argument("paths", nargs="*", help="Folder(s) to scan.")
    parser.add_argument(
        "--output",
        "-o",
        dest="output_path",
        help="Write duplicate file pairs to a text file for manual review.",
    )
    return parser.parse_args()


def main():
    args = parse_args()
    if not args.paths:
        print("Usage: duplicates.py <folder> [<folder>...] [--output <path>]")
        return
    check_for_duplicates(args.paths, output_path=args.output_path)


if __name__ == "__main__":
    main()
    
# it work