Recursive saving of file contents from a directory in Markdown format

dump_tree_to_md.py

#!/usr/bin/env python3
import argparse
import fnmatch
from pathlib import Path
from typing import List, Optional, Union

try:
    from pathspec import PathSpec

    _HAS_PATHSPEC = True
except ImportError:
    _HAS_PATHSPEC = False


def is_binary(fp: Path, sniff: int = 1024) -> bool:
    try:
        with fp.open("rb") as f:
            return b"\0" in f.read(sniff)
    except OSError:
        return True


def is_hidden(path: Path) -> bool:
    return any(part.startswith(".") for part in path.parts)


def load_gitignore(root: Path) -> Optional[Union["PathSpec", List[str]]]:
    gitignore = root / ".gitignore"
    if not gitignore.is_file():
        return None

    lines = gitignore.read_text(encoding="utf-8", errors="ignore").splitlines()
    patterns = [ln.strip() for ln in lines if ln.strip() and not ln.lstrip().startswith("#")]

    if _HAS_PATHSPEC:
        return PathSpec.from_lines("gitwildmatch", patterns)

    return patterns


def matches_gitignore(rel: Path, matcher: Optional[Union["PathSpec", List[str]]]) -> bool:
    if matcher is None:
        return False

    rel_str = rel.as_posix()

    if _HAS_PATHSPEC and hasattr(matcher, "match_file"):
        return matcher.match_file(rel_str)

    for pat in matcher:
        if pat.endswith("/") and rel_str.startswith(pat.rstrip("/")):
            return True
        if fnmatch.fnmatch(rel_str, pat):
            return True
    return False


def dump_tree(root: Path, md_out: Path, include_hidden: bool) -> None:
    matcher = load_gitignore(root)
    md_out_resolved = md_out.resolve()

    with md_out.open("w", encoding="utf-8") as out:
        for file in sorted(root.rglob("*")):
            if not file.is_file():
                continue

            if file.resolve() == md_out_resolved:
                continue

            rel = file.relative_to(root)

            if not include_hidden and is_hidden(rel):
                continue

            if matches_gitignore(rel, matcher):
                continue

            if is_binary(file):
                continue

            try:
                text = file.read_text(encoding="utf-8", errors="replace")
            except OSError:
                continue

            out.write(f"```{rel}\n")
            out.write(text.rstrip("\n"))
            out.write("\n```\n\n")


def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("directory", nargs="?", default=".")
    parser.add_argument("-o", "--output", default="dump.md")
    parser.add_argument("-a", "--all", action="store_true", help="Include hidden ‘dot’ files as well.")
    args = parser.parse_args()

    dump_tree(Path(args.directory).resolve(), Path(args.output).resolve(), args.all)


if __name__ == "__main__":
    main()