Simple script to convert DriveThruRPG library data into CSV

dtrpg_convert.py

import csv
import json
import sys

# This is simple converter for DriveThruRPG JSON data 
# from library into CSV file, so you can manage your
# library

# How to use
# 1. Go to DriveThruRPG site and login
# 2. Open: https://www.drivethrurpg.com/api/products/mylibrary/search?show_all=1&draw=9&columns[0][data]=null&columns[0][name]=&columns[0][searchable]=false&columns[0][orderable]=false&columns[0][search][value]=&columns[0][search][regex]=false&columns[1][data]=product.title&columns[1][name]=name&columns[1][searchable]=true&columns[1][orderable]=true&columns[1][search][value]=&columns[1][search][regex]=false&order[0][column]=1&order[0][dir]=asc&start=0&length=-1&search[value]=&search[regex]=false&show_archived=false&show_all=0&show_new=&show_updated=0&filter_string=&oneclick=true
# 3. Copy and paste json response into data.json file
# 4. Run: `python dtrpg_convert.py data.json export.json`
# 5. Your data is in `export.csv` file

# Arguments for the script:
# - input_file -- required
# - output_file -- required
# - old_output_file -- optional - if you run this script after some time, you may want to pull only new products. By providing old csv file you will get in the output_file only products that do not match already existing ones (by title)

columns = ["title", "publisher", "updated", "url"]


def load_data(input_file):
    with open(input_file, "r", encoding='utf8') as f:
        data = json.load(f)
    return data["data"]


def load_old_titles(old_file):
    titles = []
    with open(old_file, "r", encoding='utf8') as file:
        file.readline()  # headers
        reader = csv.DictReader(file, fieldnames=columns)
        for data_dict in reader:
            titles.append(data_dict["title"])
    return titles


def process(input_file, output_file, old_file=None):
    if old_file:
        old_titles = load_old_titles(old_file)

    data = load_data(input_file)

    export_data = []
    with open(output_file, "w", newline='', encoding='utf8') as file:
        writer = csv.DictWriter(file, fieldnames=columns)
        writer.writeheader()

        for item in data:
            if old_file and item["product"]["title"] in old_titles:
                continue
            row_dict = {
                "title": item["product"]["title"],
                "url": item["product"]["url"],
                "publisher": item["publisher"]["title"],
                "updated": item["updated"]
            }
            writer.writerow(row_dict)


if __name__ == "__main__":
    # Arguments:
    # - input file
    # - output file
    # - old output file for updates
    args = sys.argv
    if len(args) < 3 or len(args) > 5:
        print("Wrong number of arguments. Check what you run")
    process(*args[1:])

I started poking around the logs from the official DTRPG library app, and found that they're pretty verbose. I started documenting what I could find from that and poking at the API here. I'm not very experienced with this, but I hope that maybe this can help someone out.

FWIW, the app uses different endpoints than the search embedded in the page. The app API endpoints are much easier to authenticate into, though, and don't require logging into the website (well, except to get your API key).