Skip to content

Instantly share code, notes, and snippets.

@sri
Last active October 1, 2022 13:31
Show Gist options
  • Save sri/fed2d5f9faec01f7df78717eed3e68b9 to your computer and use it in GitHub Desktop.
Save sri/fed2d5f9faec01f7df78717eed3e68b9 to your computer and use it in GitHub Desktop.
Parse JSON into objects
from dataclasses import fields
def init_from_json(cls, json):
return cls(**{f.name: json.get(f.name) for f in fields(cls)})
from dataclass_helper import init_from_json
@dataclass
class Item:
id: str
kind: str
created_utc: float
permalink: str
title: str
selftext_html: str
link_title: str
body: str
link_url: str
def comment_url(self):
return f"https://beta.reddit.com/{self.permalink}"
def posted_on(self):
return datetime.datetime.fromtimestamp(self.created_utc)
def is_comment(self):
return self.kind == "t1"
def is_post(self):
return self.kind == "t3"
def parse(...):
...
item = init_from_json(Item, raw_item)
# Uses another intermediary dict, but this felt OK for now.
#
args = dict(
site=site,
fetched_on=datetime.date.today(),
posted_on=item.posted_on(),
comment_url=item.comment_url(),
)
if item.is_post():
fetched += 1
args["title"] = item.title
args["content"] = item.selftext_html or ""
elif item.is_comment():
fetched += 1
args["title"] = item.link_title
args["link_url"] = item.link_url
args["content"] = item.body
else:
print(f"unknown link: {item}")
if debug:
# Don't create db
return
Post.create(**args)
...
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment