4chan thread archiver

4rchive.py

import requests, sys, re, os, shutil

if len(sys.argv) < 2:
	sys.exit("Usage: 4rchive.py <URL>")


url = re.search("4chan.org\/(\w+)\/\w+\/(\d+)\/?", sys.argv[1])
board = url.group(1)
path = "downloads/{}/{}/".format(board, url.group(2))

if not os.path.exists(path):
	os.makedirs(path)


r = requests.get(sys.argv[1].replace("boards.4chan.org", "a.4cdn.org") + ".json")

if r.status_code != 200:
	sys.exit("4chan API failure")

response = r.json()


total = len(response['posts'])

print("Downloading " + str(total) + " post(s)") 


# allowed file extensions
# (ignoring text posts on an image board)
exts = [".webm", ".gif", ".jpg", ".jpeg", ".png"]


for i, post in enumerate(response['posts']):

	if ('tim' not in post) or (post['ext'] not in exts):
		continue

	# discard postage stamp sized images, roughly
	if post['w'] + post['h'] < 800:
		continue

	filename = str(post['tim']) + post['filename'] + post['ext']
	iurl = "https://i.4cdn.org/{}/{}".format(board, filename)

	print("[{}/{}] ".format(i+1, total) + iurl)

	if os.path.exists(path + filename):
		continue

	r = requests.get(iurl, stream=True)

	with open(path + filename, 'wb') as outf:
		shutil.copyfileobj(r.raw, outf)