ssh... it's a secret

_crawl.py

#!/usr/bin/env python3

from urllib.request import Request, urlopen
import sys
import os
import json

class color:
    PURPLE = "\033[95m"
    CYAN = "\033[96m"
    DARKCYAN = "\033[36m"
    BLUE = "\033[94m"
    GREEN = "\033[92m"
    YELLOW = "\033[93m"
    RED = "\033[91m"
    BOLD = "\033[1m"
    UNDERLINE = "\033[4m"
    END = "\033[0m"

def pad_int(int, pad):
    return(str(int).zfill(pad))

def make_url(url, id):
    return(url.format(id=pad_int(id_int, id_length)))

config_file = open("config.json")
config = json.load(config_file)
config_file.close()

id_int = config["start"]
id_max = config["end"]
id_length = len(str(id_max))

url_template = config["url"]
cookie = config["cookie"]

output_dir = config["output_dir"]

while id_int <= id_max:
    url = make_url(url_template, id_int)
    
    print("trying " + url)
    
    req = Request(url)
    
    req.add_header("Cookie", cookie)

    try:
        res = urlopen(req).read()
        
        if sys.getsizeof(res) > 5000:
            f = open(output_dir + "/" + str(id_int) + ".jpg", "w+b")
            f.write(res)
            f.close()
            
            print(color.BOLD + color.GREEN + "downloaded %s" % str(id_int) + color.END)
        else:
            print(color.RED + "placeholder at %s" % str(id_int) + color.END)
    except:
        print(color.RED + ";_; error ;_;" + color.END)
    
    id_int += 1
    
    print("%(numfiles)s files found\n" % {
        "numfiles": str(len(os.listdir(output_dir)))
    })

config.json

{
    "url": "http://www.example.com/{id}/something.txt",
    "cookie": "all_of_the=cookies; including=allTheOtherOnes",
    "output_dir": "output",

    "start": 0,
    "end": 99999
}