Demo python script to remove duplicate objects in a JSON file, write output

remove_duplicates.py

# Remove duplicate objects in a JSON file.

import json


INPATH = "offending.json"
OUTPATH = "corrected.json"


with open(INPATH, "r") as fl:
    json_in = json.load(fl)

# Remove duplicates...
hashables = set(tuple(g.items()) for g in json_in)
# Reinflate to list of dicts...
unique_output = [dict(h) for h in hashables]

with open(OUTPATH, "w") as fl:
    json.dump(unique_output, fl)