Read-only Reddit thread capture utility

reddit_full_capture.py

import json
import os
import sys
import getpass
from datetime import datetime, timezone

import praw
from praw.models import MoreComments


def prompt_value(env_name, prompt, secret=False):
    value = os.getenv(env_name)
    if value:
        return value
    if secret:
        return getpass.getpass(prompt)
    return input(prompt).strip()


def comment_to_dict(comment):
    author = str(comment.author) if comment.author else None

    return {
        "id": comment.id,
        "name": f"t1_{comment.id}",
        "parent_id": comment.parent_id,
        "author": author,
        "body": getattr(comment, "body", None),
        "score": getattr(comment, "score", None),
        "created_utc": getattr(comment, "created_utc", None),
        "edited": getattr(comment, "edited", False),
        "distinguished": getattr(comment, "distinguished", None),
        "is_submitter": getattr(comment, "is_submitter", False),
        "permalink": getattr(comment, "permalink", None),
        "replies": [],
    }


def build_tree(comment_records, post_id):
    nodes = {c["id"]: c.copy() for c in comment_records}

    # Make sure reply arrays are independent objects.
    for node in nodes.values():
        node["replies"] = []

    roots = []

    for node in nodes.values():
        parent_id = node["parent_id"]

        if parent_id == f"t3_{post_id}":
            roots.append(node)

        elif parent_id and parent_id.startswith("t1_"):
            parent_comment_id = parent_id[3:]

            if parent_comment_id in nodes:
                nodes[parent_comment_id]["replies"].append(node)
            else:
                # Parent unavailable/deleted from API response.
                roots.append(node)

        else:
            roots.append(node)

    return roots


def main():
    if len(sys.argv) < 2:
        print(
            'Usage:\n'
            '  python reddit_full_capture.py '
            '"https://www.reddit.com/r/.../comments/POST_ID/..."'
        )
        sys.exit(1)

    reddit_url = sys.argv[1]

    client_id = prompt_value(
        "REDDIT_CLIENT_ID",
        "Reddit client ID: "
    )

    client_secret = prompt_value(
        "REDDIT_CLIENT_SECRET",
        "Reddit client secret: ",
        secret=True
    )

    reddit_username = prompt_value(
        "REDDIT_USERNAME",
        "Your Reddit username (for the user-agent): "
    )

    user_agent = (
        f"desktop:chatgpt-research-capture:1.0 "
        f"(by /u/{reddit_username})"
    )

    reddit = praw.Reddit(
        client_id=client_id,
        client_secret=client_secret,
        user_agent=user_agent,
        ratelimit_seconds=600,
        timeout=30,
    )

    reddit.read_only = True

    print("\nOpening submission...")
    submission = reddit.submission(url=reddit_url)

    # Force submission metadata retrieval.
    post_id = submission.id
    title = submission.title
    reported = submission.num_comments

    print(f"Post: {title}")
    print(f"Post ID: {post_id}")
    print(f"Reddit reported comments: {reported}")
    print()
    print("Expanding every MoreComments object...")
    print("This may issue many Reddit API requests on a large thread.")

    # None = keep replacing MoreComments until there are none left.
    remaining_more = submission.comments.replace_more(limit=None)

    print("Expansion finished. Building snapshot...")

    items = submission.comments.list()

    unresolved_in_forest = [
        x for x in items if isinstance(x, MoreComments)
    ]

    real_comments = [
        x for x in items if not isinstance(x, MoreComments)
    ]

    # Deduplicate defensively.
    by_id = {}
    for comment in real_comments:
        by_id[comment.id] = comment

    comments = [
        comment_to_dict(comment)
        for comment in by_id.values()
    ]

    deleted_or_removed = sum(
        1
        for c in comments
        if c["author"] is None
        or c["body"] in ("[deleted]", "[removed]", None)
    )

    human_authors = {
        c["author"]
        for c in comments
        if c["author"]
        and c["author"] != "AutoModerator"
    }

    automod = sum(
        1 for c in comments
        if c["author"] == "AutoModerator"
    )

    tree = build_tree(comments, post_id)

    unresolved_count = (
        len(remaining_more)
        + len(unresolved_in_forest)
    )

    unique_count = len(comments)
    residual_gap = reported - unique_count

    captured_at = datetime.now(timezone.utc).isoformat()

    snapshot = {
        "capture": {
            "type": "praw_full_comment_expansion",
            "captured_at_utc": captured_at,
            "source_url": reddit_url,
            "praw_read_only": True,
        },
        "post": {
            "id": post_id,
            "name": f"t3_{post_id}",
            "title": title,
            "author": (
                str(submission.author)
                if submission.author
                else None
            ),
            "subreddit": str(submission.subreddit),
            "selftext": submission.selftext,
            "permalink": submission.permalink,
            "created_utc": submission.created_utc,
            "score": submission.score,
            "upvote_ratio": submission.upvote_ratio,
            "reported_num_comments": reported,
        },
        "completeness": {
            "reddit_reported_comments": reported,
            "unique_comments_captured": unique_count,
            "residual_gap": residual_gap,
            "unresolved_more_objects": unresolved_count,
            "deleted_or_removed_comments": deleted_or_removed,
            "automoderator_comments": automod,
            "identifiable_non_automod_authors": len(human_authors),
        },
        "comments_flat": comments,
        "comments_tree": tree,
    }

    filename = f"reddit_{post_id}_full.json"

    with open(filename, "w", encoding="utf-8") as f:
        json.dump(
            snapshot,
            f,
            ensure_ascii=False,
            indent=2
        )

    print()
    print("=" * 60)
    print("CAPTURE REPORT")
    print("=" * 60)
    print(f"Reddit reported:            {reported}")
    print(f"Unique comments captured:   {unique_count}")
    print(f"Residual gap:               {residual_gap}")
    print(f"Unresolved MoreComments:    {unresolved_count}")
    print(f"Deleted/removed:            {deleted_or_removed}")
    print(f"AutoModerator:              {automod}")
    print(
        f"Identifiable human authors: {len(human_authors)}"
    )
    print()
    print(f"Saved: {filename}")

    if unresolved_count == 0:
        print("\nNo unresolved MoreComments remain.")
    else:
        print(
            "\nWARNING: unresolved MoreComments remain; "
            "do not treat this as a complete capture."
        )


if __name__ == "__main__":
    main()