A web scraper to pull historical item price data from rl.insider.gg

RLInsiderScraper.py

#
# Scraper.py
#
# Copyright Harry Day 2021
#
# Twitter: https://twitter.com/realharryday
# Github: https://github.com/harryday123
#

import json
from datetime import datetime
from typing import Any, List, Optional, Union, TypeVar, Callable, Type, cast
import dateutil.parser
import csv
import requests
from bs4 import BeautifulSoup
import re

def fixDates(dict: dict) -> dict:
    newDict = {}

    for key in dict:
        ts = int(key)
        fixed = datetime.utcfromtimestamp(ts).strftime('%Y-%m-%d')

        newDict[fixed] = dict[key]

    return newDict


def CreateOutputFromJSONObject(json_string: str) -> str:
    global label
    global color

    objectDict = None

    try:
        objectDict = json.loads(json_string)
    except:
        print("Could Not Decode")
        return

    name = objectDict.get("itemName")
    color = objectDict.get("itemColor")

    if name == None or color == None:
        print("Name or Color not found")
        print("Skipping...")
        return None

    if color == "":
        color = "Unpainted"
        label = name
    elif color == "bm":
        color = "Black Market"
        label = color + " " + name
    else:
        label = color + " " + name

    label = label.strip()

    itemHistoryData = objectDict.get("itemHistoryData")
    if itemHistoryData != None:
        allTimeObject = next((x for x in itemHistoryData if x.get("label") == "all-time"), None)
    else:
        return None

    if allTimeObject != None:
        dataResultDict = {}

        for pointInfo in allTimeObject.get("pointInfo"):
            if pointInfo != None:
                try:
                    dataResultDict[pointInfo.get("timestamp")] = pointInfo.get("price")[2]
                except TypeError:
                    print("That weird price = false thing appeared again")
        if dataResultDict != {}:
            return fixDates(dataResultDict)
        else:
            return None
    else:
        return None


def DoesContainItemData(s):
    if s != None:
        return "itemData" in s
    else:
        return False


def ExtractJSONFromScript(string: str):
    try:
        return re.search('itemData = (.+?);\n', string).group(1)
    except AttributeError:
        # itemData = , ; not found in the original string
        return None


def SaveFromURL(url: str):
    page = requests.get(url)
    soup = BeautifulSoup(page.content, "html.parser")
    scriptTag = soup.head.find("script", string=DoesContainItemData)
    if scriptTag != None:
        print("Found Script Tag")
        itemDataJSON = ExtractJSONFromScript(scriptTag.string)
    else:
        print("No Script Tag Found")
        return False

    if itemDataJSON != None:
        print("Found JSON")
        # print(itemDataJSON)
        output = CreateOutputFromJSONObject(itemDataJSON)
    else:
        print("Could not decode JSON")
        return None

    if output != None:
        print("Converted to Output Format")
        filled = FillBlanks(output)
    else:
        print("Could not convert to output format")
        return None

    if filled != None:
        filled["Name"] = label
        filled["Colour"] = color
        print("Added Name and Color")

        AppendToCSV(filled)
        print("Appended To CSV")
        return True
    else:
        return False


def AppendToCSV(obj: dict) -> dict:
    keys = ["Name", "Colour"] + headers
    with open('outputData.csv', mode='a') as csv_file:
        writer = csv.DictWriter(csv_file, fieldnames=keys)
        writer.writerow(obj)


def FillBlanks(obj: dict) -> dict:
    keys = obj.keys()

    # Delete last entry
    del obj[list(keys)[-1]]

    for key in list(keys):
        if key not in headers:
            del obj[key]

    if obj == {}:
        return None

    firstHeaderIndex = headers.index(list(keys)[0])

    # From first known value forwards
    for i in range(firstHeaderIndex, len(headers)):
        if headers[i] not in keys:
            obj[headers[i]] = obj[headers[i-1]]

    # Add values backwards of first known value as 0
    for i in range(0, firstHeaderIndex):
        obj[headers[i]] = 0

    return obj


def GenerateHeaders(overwrite: bool):
    start = 1478131200
    end = 1625316023
    interval = 604800
    current = start

    headers = ["Name", "Colour"]

    while current < end:
        headers.append(datetime.utcfromtimestamp(current).strftime('%Y-%m-%d'))
        current += interval

    if overwrite:
        with open('outputData.csv', mode='w') as csv_file:
            writer = csv.DictWriter(csv_file, fieldnames=headers)
            writer.writeheader()

    return headers[2:]


def FormatItemNames(names: list):
    # f = lambda x: x.replace(" (", "_").replace(": ", "_").replace(" ", "_").replace("-", "_").strip(")")
    f = lambda x: x.strip("_")
    print(list(map(f, names)))


def LoopItems():
    global finishedPainted
    global failedPainted
    global notFoundPainted

    baseURL = "https://rl.insider.gg/en/pc/"

    for item in painted:
        for color in colors:
            url = baseURL + item + "/" + color
            print("======================")
            print("Now Running:", color, item)
            print(url)
            result = SaveFromURL(url)
            if result:
                finishedPainted.append(url)
            elif not result:
                notFoundPainted.append(url)
            else:
                failedPainted.append(url)


def LoopUnPaintedItems():
    global finishedUnPainted
    global failedUnpainted

    baseURL = "https://rl.insider.gg/en/pc/"

    for item in unpainted:
        url = baseURL + item
        print("======================")
        print("Now Running:", item)
        print(url)
        result = SaveFromURL(url)
        if result:
            finishedUnPainted.append(url)
        else:
            failedUnpainted.append(url)



colors = ["", "black", "white", "grey", "pink", "cobalt", "sblue", "sienna", "saffron", "lime", "fgreen", "orange", "purple", "crimson"]

painted = [
'Exalter', 'Interstellar', 'Hex_Tide', 'Wet_Paint', 'Trigon', 'Mainframe', '20xx', 'Hexed', 'Slipstream', 'Air_Strike', 'Buffy_Sugo', 'Gravity_Bomb', 'Meteor_Storm', 'Neuro_Agitator', 'Shattered', 'Voxel', 'Singularity', 'Atomizer', 'Toon', 'Dueling_Dragons', 'Poly_Pop', 'Solar_Flare', 'Hellfire', 'Sub_Zero', 'Fireworks', 'Electroshock', 'Vampire_Bat', 'Reaper', 'Light_Show', 'Riser', 'Mister_Monsoon', 'Spatial_Rift', 'Rad_Rock', 'Overgrowth', 'Holodata', 'Floppy_Fish', 'Kablooey', 'Meta_Blast', 'Dust_Cloud', 'Wall_Breaker_II', 'Force_Razor_II', 'Quasar_III', 'Supernova_III', 'Ballistic', 'Chequered_Flag', 'Komodo', 'Tygris', 'Insidio', 'Peregrine_TT', 'Sentinel', 'Fennec', 'Diestro', 'Nimbus', 'Cyclone', 'Twinzer', 'Samurai', 'Werewolf', 'Imperator_DT5', 'Jager_619', 'Octane', 'Breakout', 'Merc', 'Road_Hog', 'Venom', 'X_Devil', 'Centio', 'Animus_GP', 'Mantis', 'Endo', 'Takumi_RX_T', 'X_Devil_Mk2', 'Dominus', 'Hotshot', 'Paladin', 'Hotshot', 'Paladin', 'R3MX_GXT', 'Harbinger_GXT', 'Ronin_GXT', 'Chikara_GXT', 'Mudcat_GXT', 'Guardian_GXT', 'Artemis_GXT', 'Maverick_GXT', 'Tyranno_GXT', 'Bravado_Infinite', 'Dire_Wolf', 'HNY_Inverted', 'Cutter_Inverted', 'Tremor', 'Ultralex', 'Tremor_Inverted', 'NeYoYo', 'Creeper', 'Zefram', 'Zefram_Infinite', 'P_SIMM', 'P_SIMM_Inverted', 'Pyrrhos', 'Zowie', 'Zowie_Infinite', 'Gernot', 'Raijin', 'Dynamo', 'Reactor', 'Cruxe', 'Santa_Fe', 'Centro', 'REEVRB', 'Hypnotik', 'Illuminata', 'Infinium', 'Balla_Carra', 'Wonderment', 'Equalizer', 'Hikari_P5', 'Clockwork', 'Chrono', 'FGSP', 'Turbine', 'Draco', 'K2', 'Kalos', 'Roulette', 'Zomba', 'ARA_51', 'Discotheque', 'Pulsus', 'Looper', 'Photon', 'Voltaic', 'Lobo', 'Sunburst', 'Tunica', 'Vortex', 'Veloce', 'Spyder', 'Alchemist', 'OEM', 'Invader', 'Stern', 'Octavian', 'Neptune', 'Falco', 'Almas', 'Lowrider', 'Rat_Rod', 'Trahere', 'Dieci', 'Ault_SPL', 'Parabolic', 'SPN', '3_Lobe', 'DYR_II', 'Esoto_4R', 'Woofer', 'Polyergic', 'Polyergic_Inverted', 'Esoto_4R_Inverted', '3_Lobe_Infinite', 'Wrathogen', 'Hephaestus', 'ANDR01D', 'String_Theory', 'BLSSM', 'HRB_20', 'Hamster', 'Ferris', 'Sk8ter', 'Propeller', 'Mandala', 'Propeller_Inverted', 'Hamster_Holographic', 'Mandala_Infinite', 'Q_Runner', 'Shortwire', 'Jandertek', 'Muscle_Boy', 'Tri_2050', 'Cyberware', 'Blade_Wave', 'Z_RO', 'Z_RO_Inverted', 'Jandertek_Holographic', 'Blade_Wave_Inverted', 'Decopunk', 'Carbon', 'Petacio', 'Reticle', 'Shima', 'Dimonix', 'Tube_Tank', 'Morrowhatch', 'Startis', 'Yankii_RL', 'Holosphere', 'Helicoprion', 'Dimonix_Inverted', 'Yankii_RL_Infinite', 'Shima_Infinite', 'Gyre', 'Mendoza', 'Metalstar', 'Patriarch', 'Automaton', 'Zhou_GFP', 'DRN', 'Cephalo', 'Irradiator', 'Philoscope_III', 'Camo', 'Automaton_Inverted', 'DRN_Infinite', 'Irradiator_Infinite', 'Camo_Inverted', 'Decennium', 'Emerald', 'Tungsten', 'Metalwork', 'Metalwork_Inverted', 'Galvan', 'Galvan_Holographic', 'Ulterior', 'Generator_II', 'Visionary', 'Celestial_II', 'Polaris', 'Polaris_Remixed', 'Ved_ava_II', 'Rocket_Forge_II', 'Plasmatic', 'Madness_II', 'Teller', 'Daemon_Kelpie', 'Gripstride_HX', 'Sprocket', 'Sprocket_Infinite', 'Gripstride_HX_Inverted', 'Rocket_Forge_II_Holographic', 'Troublemaker_IV', 'Capacitor_IV', 'Aero_Mage', 'Pyramidia', 'Meridian', 'Aether', 'Razzle', 'Apex', 'Ninja', 'Maxle_PA', 'Zadeh_S3', 'Traction_Hatch', 'Imptekk', 'Joko_XL', 'Zadeh_S3_Inverted', 'X_Tempo_Radiant', 'Spanner_Inverted', 'Shortquarter_Inverted', 'Flim_Flam_Inverted', 'Hydraul1K', 'HNY', 'CNTCT_1_Infinite', 'IO', 'IO_Infinite', 'Spiralis_R2', 'Apparatus', 'Apparatus_Inverted', 'Stella', 'Stella_Inverted', 'Asik', 'Asik_Infinite', 'Finny', 'Libertine', 'Sovereign_A/T', 'Kyrios', 'Lustrum_X10', 'Christmas_Wreath', 'Hiro', 'Cauldron', 'Saptarishi', 'Triplex', 'FSL', 'Vortex', 'OEM', 'Invader', 'Stern', 'Falco', 'Bravado', 'X_Tempo', 'Medianic', 'Spanner', 'Flim_Flam', 'Shortquarter', 'Playock', 'Spinner', 'Cristiano', 'CNTCT_1', 'Cutter', 'Grappler', 'Orbit', 'Glaive', 'Glaive_Inverted', 'FSL_B', 'Picket', 'Truncheon', 'Picket_Holographic', 'Blender', 'Jayvyn', 'Yamane', 'Thread_X2', 'Quimby', 'Masato', 'Reaper', 'Diomedes', 'Doughnut', 'Nipper', 'Peppermint', 'Yuzo', 'Revenant', 'Grimalkin', 'Gaiden', 'SLK', 'Fireplug', 'Septem', 'Spiralis', 'Troika', 'Chakram', 'Wrench_Roller', 'Clodhopper', 'Asterias', 'Zeta', 'Dieci', 'Sunburst', 'Tunica', 'Vortex', 'Veloce', 'Spyder', 'Alchemist', 'OEM', 'Invader', 'Stern', 'Octavian', 'Neptune', 'Almas', 'Falco', 'Lowrider', 'Rat_Rod', 'Trahere', 'Ambustion', 'Candyfloss', 'Flamerate', 'Ninja_Star', 'Krackle', 'Glitch', 'Synthwave', 'Virtual_Wave', 'Sun_Ray', 'Enchanter', 'Cirrus', 'Comet', 'Geo_Soul', 'Tsunami_Beam', 'Toon_Sketch', 'Fractal_Fire', 'Power_Shot', 'Datastream', 'Flamethrower', 'Ion', 'Lightning', 'Plasma', 'Sacred', 'Sparkles', 'Standard', 'Thermal', 'Magic_Missile', 'Neo_Thermal', 'Tachyon', 'Proton', 'Hypernova', 'Dark_Matter', 'Trinity', 'Pixel_Fire', 'Quasar', 'Torrent', 'Springtime_Flowers', 'Hexphase', 'Polygonal', 'Frostbite', 'Toon_Smoke', 'Monsoon', 'HRT_Beat', 'Anispray', 'Toon_Flame', 'Supernova', 'Rad_Rock', 'Glimmerslag_II', 'Holodata', 'Shining_Barrage_II', 'Radiant_Gush', 'Fiber_Optic_II', 'Fusion', 'Slash_Beam_III', 'Super_Manga_Bolt_III', 'Cloudburst_III', 'Hot_Rod', 'Blast_Ray', 'Helios', 'Pylon', 'Chequered_Flag', 'Diecast_Spark', 'Star_Princess', 'Pixelated_Shades', 'Catfish', 'MMS_Headphones', 'Octopus', 'Wildcat_Ears', 'Drink_Helmet', 'Bycocket', 'Surfboard', 'Foam_Hat', 'Halo', 'Hawaiian_Lei', 'Heart_Glasses', 'Jack_in_the_Box', 'Latte', 'Stegosaur', 'Cromulon', 'Unicorn', 'Beret', 'Royal_Crown', 'Pork_Pie', "Pirate's_Hat", 'Little_Bow', "Chef's_Hat", 'Ivy_Cap', 'Wizard_Hat', 'Derby', 'Mariachi_Hat', 'Devil_Horns', 'Visor', "Witch's_Hat", 'Traffic_Cone', 'Chainsaw', 'Fire_Helmet', 'Trucker_Hat', 'Birthday_Cake', 'Bowler', 'Brodie_Helmet', 'Fez', 'Fruit_Hat', 'Hard_Hat', 'Homburg', 'Top_Hat', 'Drone_III', 'Phoenix_Wings_III', 'Jolt_Bangle_III', 'LFT', 'Cuckoo_Clock', 'Lollipop', 'Macaron', 'Mage_Glass_III', 'Painted_Trails', 'Toon_Sketch', 'Truth_Beam', 'Discotheque', 'Binary', 'EKG_OMG', 'Zigzag', 'Lightspeed', 'Excavator', 'Rad_Rock', 'Holodata', 'Hack_Swerve_III', 'Tachyon_III', 'Laser_Wave_III', 'EQ', 'Pixel_Pointer', 'Pulse_Crawler', "Bob's_Ramen", 'Burnout', 'Lift_Off', 'Doughnut_Eater', 'Zombie', 'Sweet_Dreams', 'Ouchie', 'NeOctane', 'Call_Sign_RL', 'Dead_Serious', 'Narwhal', 'Tagged', 'Crown', 'Almagest', 'Z_Current', 'Octane_10dril', 'Deep_Six', 'Smokescreen', 'Octane_Tumbling_Blocks', 'Endo_MG_88', 'Octane_MG_88', 'Fennec_Bodacious', 'Octane_Bodacious', 'Fennec_Ombre', 'Komodo_Takeover', 'Octane_Kana', 'Goop', 'Tygris_Silencer', 'Fennec_Zeus', 'Insidio_Silencer', 'Fennec_Slimline', 'Insidio_Mr._Coney', 'Insidio_Sticker_Bomb', 'Insidio_Ternion', 'Fennec_Hardline', 'Sentinel_Faceted', 'Octane_Vertebrate', 'Dominus_OR_AISE', 'Octane_Abtruse', 'Fennec_Edge_Burst', 'Twinzer_Smooth_Jazz', 'Dominus_Mixtape', 'Octane_Vaporwave', 'Diestro_Rico', 'Breakout_Taniwha', 'Twinzer_Good_Shape', 'Venom_Retro_Sun', 'Twinzer_Muddy', 'Octane_Sticker_Bomb', 'Octane_ZSR_Mechaceph', 'Octane_Chantico', 'Jäger_619_StarLighter', 'Breakout_Astaroth', 'Breakout_9_Lives', 'Animus_GP_Peacock', 'Octane_Island_King', 'Dominus_Proptosis', 'Dominus_GT_Staredown', 'Centio_Splashback', 'Imperator_DT5_Windswept', 'Road_Hog_Sundae', 'Masamune_Min_Spec', 'Dominus_GT_NNTR', 'Centio_Machina', 'Animus_GP_Rose_King', 'Jäger_619_XVI', 'Octane_Royal_Tyrant', 'Dominus_GT_Callous_Bros.', 'Centio_Pollinator', 'Jäger_619_Hip_Hop', 'Breakout_Egged', 'Merc_Athena', 'Imperator_DT5_Mosher', 'Jäger_619_Mister_Monsoon', 'Marauder_XVIII', 'Dominus_Funny_Book', 'Mantis_Cold_Front', 'Octane_Christmas_Tree', 'Dominus_Holiday_Deco', 'Breakout_Rad_Reindeer', 'Octane_Slimline', "Breakout_Type_S_S'mored", 'Animus_GP_Odd_Fish', 'Endo_Mummified', "Mantis_Widow's_Web", 'Breakout_Stitches', 'Dominus_Afterlife', 'Octane_Thanatos', 'Mantis_Critters', 'Dominus_GT_Splatter', 'Breakout_Chainsaw', 'Centio_Gigapede', 'Octane_ZSR_Funny_Book', 'Endo_Spatter', 'Octane_Lone_Wolf', 'Breakout_Type_S_Mobo', 'Takumi_Sticker_Bomb', 'Dominus_MDGA', 'Octane_ZSR_Jiangshi', 'Octane_Buzz_Kill', 'Breakout_Heiwa', 'Masamune_Kawaii', 'Octane_Griffon', 'Dominus_Suji', 'Takumi_Aqueous', 'Breakout_Type_S_Funny_Book', 'Octane_ZSR_Tribal', 'X_Devil_Mk2_Cobra', 'Dominus_GT_Unmasked', 'Takumi_RX_T_Super_RX_T', 'Dominus_Mondo', 'Dominus_Arcana', 'Breakout_Shibuya', 'Octane_Dragon_Lord', 'Triton_Glyphtrix', 'Paladin_ShapeRacer', 'Aftershock_Tiger_Tiger', 'Spatiotemporal', 'Filiformer', 'EQ_RL', 'Octane_RLCS_X', 'Fennec_RLCS_X', 'Dominus_RLCS_X', 'Liquid_Camo', 'Auger', 'Tigress', 'Compound_C', 'Dune_Sweeper', 'Fennec_Yorebands', 'Dominus_Stratum_Badge', 'Luster_Edge', 'Octane_Metarudia', 'Dominus_Nightmare_Fuel', 'Formation', 'Hydro_Paint', 'Shield_Glitch', 'Percussion', 'Mainliner', 'Streak_Wave', 'Future_Shock', 'Octane_Noooo!', 'Breakout_What_A_Save!', 'Breakout_Nice_Shot', 'Octane_Nice_Shot', 'Dominus_Savage', 'Dominus_CRL_Eastern', 'Dominus_CRL_Southern', 'Octane_CRL_Northern', 'Breakout_CRL_Western', 'Octane_Dune_Racer', 'Dominus_Fantasmo', 'Breakout_Froggy', 'Octane_RLCS', 'Octane_ZSR_RLCS', 'Dominus_RLCS', 'Dominus_GT_RLCS', 'Breakout_RLCS', 'Breakout_Type_S_RLCS']

unpainted = [
'Stipple_Gait', 'Tidal_Stream', 'Glorifier', 'Magma', 'Intrudium', 'Streamline', 'Chameleon', 'Storm_Watch', 'Dissolver', 'Fire_God', 'Bubbly', 'Spectre', 'Tora', 'Biomass', 'Heatwave', 'Parallax', 'Labyrinth', 'Beta_Reward_Gold_Nugget', 'Alpha_Reward_Gold_Cap', 'Alpha_Reward_Gold_Rush', 'Alpha_Reward', 'Goldstone', 'Juiced', 'Party_Time', 'Popcorn', 'Beach_Party', 'Butterflies', 'Happy_Holidays', 'Big_Splash', 'Supernova_I', 'Supernova_II', 'Octane_ZSR', 'Breakout_Type_S', 'Dominus_GT', 'Road_Hog_XL', 'Vulcan', 'Scarab', 'Esper', 'Ripper', 'Takumi', 'Backfire', 'Proteus', 'Grog', 'Zippy', 'Masamune', 'Gizmo', 'Marauder', 'Triton', 'Aftershock', 'Maverick', 'Maverick_G1', 'Sanchez_DC_137', 'Mothership', 'Grog', 'Masamune', 'Proteus', 'Esper', 'Ripper', 'Aftershock', 'Marauder', 'Vulcan', 'Triton', 'Zippy', 'Scarab', 'Gearlock', 'Tomahawk', 'Tempest', 'Stallion', 'Lucci', 'Mountaineer', 'Foreman', 'Paper_Cut', 'CDJ', 'Stay_Puft', 'Ghostbusters', 'SARPBC', 'League_Shield', 'Cupcake', 'Sunrise_1986', 'Psyonix_II', 'Low_Poly_TE', 'Ton_Eighty', 'Stuffed_Crust', 'Fortune', 'Mistletoe', 'Yin_Yang', 'Shinkei', 'Timbertyre', 'Starcourt', 'Shutterbug', 'High_Poly', 'FLT', 'Low_Poly', 'Cassette', 'Sunset_1986', 'E_T', 'MainStreet', 'Sovereign', 'Holiday_Hearth', 'Moko', 'Polychrome', 'Demon_Disc', 'Troublemaker_I', 'Capacitor_I', 'Troublemaker_II', 'Capacitor_II', 'Troublemaker_III', 'Capacitor_III', 'Rhino', 'Watermelon', 'SARPBC_10', 'Psyonix', 'WWE', 'ZT_17', 'Carriage', 'Z_Plate', 'Roskilde', 'W_I_P', 'Breakout_Ripped_Comic', 'Leviathan', 'Kaleidoscope', 'Octane_Ripped_Comic', 'Mantis_Snakeskin', 'Jäger_619_Snakeskin', 'Octane_Snakeskin', 'Breakout_Distortion', 'Road_Hog_Snakeskin', 'Takumi_Distortion', 'Breakout_Snakeskin', 'Octane_Distortion', 'X_Devil_Snakeskin', 'Dominus_Huntress', 'Fennec_Distortion', 'Fennec_Huntress', 'Dominus_Vector', 'Dominus_Dot_Matrix', 'Octane_Dot_Matrix', 'Takumi_Vector', 'Breakout_Vector', 'Dominus_GT_Distortion', 'Takumi_RX_T_Distortion', 'Breakout_Type_S_Distortion', 'Octane_ZSR_Distortion', 'Octane_Kilowatt', 'Takumi_Anubis', 'Breakout_Dot_Matrix', 'Dominus_Snakeskin', 'Merc_Warlock', 'Tygris_Platformer', 'Tygris_Mister_Rad', 'Peregrine_TT_Hoodbar', 'Peregrine_TT_Crisis', 'Octane_Gale_Fire', 'Octane_Windblast', 'Octane_Roadkill', 'Road_Hog_Wildfire', 'Hotshot_Hi_Tech', 'Gizmo_Mean_Streak', 'X_Devil_Maximon', 'Breakout_Falchion', 'Breakout_Turbo', 'Octane_Shisa', 'Masamune_Oni', 'Takumi_Combo', 'Breakout_Vice', 'Dominus_Pollo_Caliente', 'Venom_Nine_Lives', 'Road_Hog_Carbonated', 'Takumi_Whizzle', 'Merc_Narwhal', 'Venom_Flex', 'Merc_Flower_Power', 'Breakout_Junk_Food', 'Dominus_Royalty', 'Octane_Racer', 'Merc_Dragon', 'Breakout_Dragon', 'Xmas_Lights', 'Merc_Watermelon', 'Gizmo_Wannabee', 'Octane_SARP_Stripe', 'Dominus_Gentleman_Beef', 'Breakout_Mammoth', 'Gizmo_Hive_Mind', 'Caution_Waves', 'Octane_Dragon', 'Breakout_Yaojing', 'Dominus_Yaojing', 'Octane_Sleet_Creeps', 'Breakout_Snow_Devil', 'Dominus_Arboreal_Nightmare', 'Gift_Storm', 'Breakout_Lycan', 'Breakout_Type_S_Zero_Sum', 'Twinzer_Quick_Fix_II', 'Breakout_Watermelon', 'Germophile', 'Spider_Boogie', 'Octane_Jacktide', 'Dominus_Candy_Monster', 'Hawaiian_Hopper', 'Leafstream', 'Octane_Rocket_League_Esports', 'Dominus_Rocket_League_Esports', 'Breakout_Rocket_League_Esports', 'Mudcat_Rocket_League_Esports', 'Dominus_Funky_Medusa', 'Octane_Conundrum', 'Dominus_Whitewash', 'Octane_Killer_Griller', 'Octane_Voltron', 'Breakout_TV_Time', 'Octane_Ultimate_Warrior', 'Octane_Cobra_Kai', 'Octane_The_Goonies', 'Breakout_Easter_Bunny', 'Fireworks', 'X_Devil_Disco_Monster', 'Octane_Quetzalcoatl', 'Octane_Sweater_Fest', 'Breakout_Nisse', 'Dominus_Winter_Waddle', 'Snowstorm', 'Marauder_Alley_Cat', 'Octane_Watermelon', 'Road_Hog_Quick_Fix', 'Breakout_Nitro_Circus', 'Octane_Trick_or_Treat', 'Breakout_Ghost_Fever', 'Mantis_Megabat', 'Arachnophobia', 'Octane_Dr_Thrash', 'Maverick_Circuit_Pro', 'Breakout_Bobcat', 'Dominus_Templar', 'Maverick_Gale_Fire', 'Partly_Cloudy', 'Jäger_619_Shark_Tooth', 'X_Devil_Dot_Matrix', 'Maverick_Jetstream', 'Octane_Space_Cadet', 'Merc_B_B_Wolf', "X_Devil_Devil's_Advocate", 'Humble_Pie', 'Octane_Turtle_Tribe', 'Breakout_Sneaky_Slither', 'Boo_ya!', 'Mantis_Hammerhead', 'Swirls', 'Endo_Polar_Force', 'Cold_Sweater', 'Breakout_BOO!', 'Twisted_Tree', 'Peregrine_TT_Mechwing', 'Harbinger_Quickline', 'Hydro', 'Winter_Storm', 'Bubbles', 'Yuletide', 'Scary_Pumpkin', 'Feather', 'Magmus', 'Taco', 'Xenosplash', 'Hearts', 'Ink', 'Lightning_Yellow', 'Treasure', 'Interdimensional_GB', 'Nitrous', 'Sandstorm', 'Accelerato', 'Battle_Stars', 'Nuts_&_Bolts', 'Burnout', 'Rainbow', 'Slime', 'Snowflakes', 'Grass', 'Money', 'Confetti', 'Flowers', 'Lantern_Lift', 'Cherry_Blossom', 'Mood_Slime', 'League_Legacy', 'Party_Horn', 'Vaporwave_Remix', 'Red_Envelopes', 'Xmas_Lights', 'Cupid', 'Stranger_Things', 'Autumn', 'Watermelon', 'Voltron', 'Vaporwave', 'Ectoplasm', 'Fireworks', 'Pickle', 'Night_Terror', 'Cloudburst_I', 'Cloudburst_II', 'Popcorn', 'Cold_Fusion', 'Xmas', 'Candy_Corn', 'Netherworld', 'Mohawk', 'Shark_Fin', 'Goldfish', 'Mr_Hot_Pepper', 'Mr_Hot_Dog', 'Mr_Banana', 'Mrs_Avocado', 'Chafed_Cherry', 'Worried_Watermelon', 'Sad_Strawberry', 'Butterfly', 'Fishbowl', 'Dragon_Wings', 'Rooster_Comb', 'Clamshell', 'ROBO_Visor', 'Propellerhead', 'deadmau5', 'Bobby_Helmet', 'Brünnehilde', 'Cherry_Top', 'Pizza_Topper', 'Portal_Cake', 'Sombrero', 'Taxi_Topper', 'Cow_Skull', 'Boombox', 'Shark_Bite', 'Cavalier', 'Succulents', 'Locomotive', 'Palm_Tree', 'Swim_Ring', 'Little_Elephant', 'Little_Dog', 'Little_Bunny', 'Little_Cow', 'Little_Owl', 'Little_Sloth', 'Uncle_Sam', 'Hotcakes', 'Antlers', 'Baseball_Cap_B', 'Baseball_Cap_F', 'Biker_Cap', "Captain's_Hat", 'Cattleman', 'Cockroach', 'Deerstalker', 'Graduation_Cap', 'Light_Bulb', 'Mouse_Trap', 'Paper_Boat', 'Party_Hat', 'Pigeon', 'Plunger', 'Police_Hat', 'Rasta', 'Rhino_Horns', 'Shuriken', 'Tiara', 'Work_Boot', 'Lucky_Cat', 'Ox_Horn', 'Ozzy_Ox', 'X_Skis', 'X_Boards', 'Accordion', 'Keyboard', 'Slimer', 'Stay_Puft', 'Ghost_Trap', 'AquaDome_Buoy', 'OGBP', 'Cactus_Cutie', 'Pizza_Pixel_2P', 'Phat_Hat_LIVE', 'Pastel_Eggs', 'Alpinist', 'Sure_Shot', 'Tranquil_Tangerine', 'Checkmate', 'Richie_Rat', 'Paper_Dragon', 'Lantern_Fest', 'Gingerbread_House', 'Happy_New_Year', 'Tee_Time', 'Crabby', 'Kitten_Cup', 'Flat_Cap', 'Tea_Kettle', 'Leaf_Litter', 'Camp_Know_Where', 'Sea_Turtle', 'Silent_Panda', "S'more", 'Starfish', 'Flower_Lotus', 'Macho_Man', 'Voltron', 'VCR', 'Phat_Hat', 'Brick_Phone', 'Pizza_Pixel', 'Mixtape', 'Hoverboard', 'Marty_Jr', "Sloth's_Hat", 'Miyagi_Do', 'Hammerhead', 'Battle_Axe', 'Turtle_Shell', 'Snare_Drum', 'Periscope', 'Shamrock', 'Christmas_Cocoa', 'Sleigh_Bailed', 'Alarm_Clock', 'Udder', 'Mad_Bomber', 'Camera', 'Juice_Box', 'Snail', 'Candy_Jack', 'Haunted_House', 'Jolt_Bangle_I', 'Phoenix_Wings_I', 'Jolt_Bangle_II', 'Phoenix_Wings_II', 'Cutlass', 'Great_White', 'Campaign_Hat', 'Macaron', 'Drip_Drop', 'Battle_Cars_Anniversary', 'Birthday_Bash', 'Kayak', 'Buzzer', 'Sand_Castle', 'Mr_Pineapple', 'GG_the_Clown', 'Dizzy_Birds', 'Two_Timer', 'Stoic_Muffin', 'Happy_Penguin', 'Fallen_Tree', 'Angel_Wings', 'Holiday_Bow', 'Snowboards', 'Scarecrow_Jack', 'Grave_Robber', 'Gargoyle', 'Undying_Love', 'Hustle_Brows', 'Mount_Champion', 'Bunny_Ears', 'Easter_Basket', 'White_Hat', 'Blitzen', 'Christmas_Tree', 'Helper_Elf', "New_Year's_2017", 'Sad_Sapling', 'Santa', 'Bone_King', 'Ghost', 'Pumpkin', 'Jester', 'Salty_Peanut', 'Bomb_Pole', 'Radioactive', 'Camo_Flag', 'Safety_Flag', 'Retro_Ball_Urban', 'Gingerbread_Man', 'Lightning_Bolt', 'Planetoid', 'Peace', 'Retro_Ball_Utopia', 'Horseshoe', 'Star', "Dave's_Bread", 'Tennis_Ball', 'Dollar_Sign', 'Retro_Ball_Wasteland', 'White_Flag', 'Heart', '8_Ball', 'UFO', 'Jolly_Roger', 'Snowman', 'Giant_Panda', 'Flower_Orchid', 'Flower_Tulip', 'Flower_Daisy', 'Pinwheel', 'Hot_Dog', 'Alien', 'Balloon_Dog', 'Candle', 'Chick_Magnet', 'Cupcake', 'Disco_Ball', 'Doughnut', 'Foam_Finger', 'Genie_Lamp', 'Harpoon', 'Hula_Girl', 'Parrot', 'Piñata', 'Rainbow_Flag', 'Rocket', 'Flower_Rose', 'Rubber_Duckie', 'Satellite', 'Seastar', 'Flower_Sunflower', 'Trident', 'Venus_Flytrap', 'Waffle', 'Mr_Floeberg', 'RL_Streamer', 'Argyled_Egg', 'Floppy', 'Celebration_Kite', 'Paper_Dragon', 'Naginata', 'Floppy_Disk', 'Bell', 'Big_Checker', 'Fortune_Fan', 'Dreidel', 'Peggy_Panda', 'Nitro_Circus', 'Mage_Glass_I', 'Mage_Glass_II', 'Calculated', 'Siiick_Reads', 'Warm_Watermelon', 'SARPBC_10', 'Mr_Toasty', 'Bubble_Wand', 'WWE', 'WWE_Monday_Night_Raw', 'WWE_NXT', 'WWE_SmackDown_Live!', 'WWE_WrestleMania_34', 'Koinobori', 'Dandelion_Seed', 'Holiday_Stocking', 'Peppermint', 'Arachnotenna', 'Shadow_Witch', 'Easter_Egg', 'Fuzzy_Vamp', 'Candy_Cane', 'Holiday_Gift', 'Nutcracker', 'Calavera', 'Fuzzy_Brute', 'Fuzzy_Skull', 'Luminous', 'Candy_Cane', 'Lightning', 'Hot_Rocks', 'Friction', 'Blazer', 'SARPBC', 'Hex_Fade', 'Ruckus', 'Xmas_Lights', 'Winter_Storm', 'Laser_Wave_I', 'Laser_Wave_II', 'Toon_Hydro', 'Rainbow', 'Hallowtide', 'Rumble_Strip', 'Triangle', 'S_Watercolour', 'Taxi', 'Sharp_Flames', 'Circuit_Pro', 'Coral', 'Leopard', 'Crystal', 'Island_Scales', 'Crisscross', 'Feathered', 'Topographic', 'Block', 'Old_Flame', 'Ishihara', 'Mrs_Avocado', 'Circuit_Board', 'Migraine', "Winter's_Warmth", 'Tranquil_Toff', 'Pixel_Fire', 'Hat_Trick', 'Lucky_Stars', 'Calculated', 'Salty', 'Unicorn', 'Pigeon', 'Nice_Slice', 'Starbase_ARC', 'Moai', 'Junk_Food', 'Beaten_Egg', 'Turtle', 'MDGA', 'Ball_King', 'Soccer_Ball', 'Hearts', 'Shattered', 'Sticker_Bomb', 'Rockat', 'Maximon', 'Dendritic', 'Hexed', 'Temple_Guardian', 'Knuckle_Huck', 'Faboo', 'RLCS_X', 'Ecto_1', 'Ghostbusters', 'Picnic', 'Wet_Wash', 'Frosted_Cake', 'Hypnoteks', 'Hornet', 'RNSM', 'Metropolis', 'Statesman', 'Fallen_Angel', 'Goodbye_Nian', 'Xmas_Lights', 'Tranquility', 'Kaze', 'Hibiscus', 'Valorous', 'Scoops_Ahoy', 'Demogorgon', 'Copper_Crisis', 'Neon_Runner', 'Checkered_Flag', 'Avant_Garde', 'Venom_Ride', 'Vaporwave', 'Knight_Rider', 'Sunset_1986', 'BKR', 'Retro_Fresh', 'Fighter_Junk', 'Sticker_Shock', 'Aloha', 'Blueprint_II', 'Frostline', 'Gamer_Pad', 'Tread_Heavily', 'Gold_Plated', 'Ghost_Story', 'Soccar_Nebula', 'Shark_Attack', 'Sushi_Roll', 'Hoagie', 'Trash_It!', 'Rocket_Labs', 'Blueprint', 'Camo', 'Vice', 'Triplicate', 'Solar_System', 'Birthday_Bash', 'Battle_Cars_Anniversary', 'Ice_Pop', "S'mored", 'WWE', 'WWE_NXT', 'WWE_Monday_Night_Raw', 'WWE_SmackDown_Live!', 'WWE_WrestleMania_34', 'Soccer_Splash', 'Shooting_Star', 'RL_Esports', 'Winning_Streak', 'Cold_Sweater', 'Howler', 'Ghostbusters', 'Frosted_Cake', 'Jolted', 'Tranquility', 'Spring_Pagoda', 'Happy_Holidays', 'Snowman', 'Scoops_Ahoy', 'Paladin', 'SDTV', 'Damaged', 'Fire_Main', 'Twinkle_Box', 'Free_Ride', 'Initializer', 'Electro_Mag', 'Muscle_Car', 'Go_Kart', 'Motormouth', 'Heinrich', 'Bootlegger', "Golden_Gift_'20", "Golden_Pumpkin_'20", "Golden_Gift_'19", "Golden_Pumpkin_'19", "Golden_Egg_'19", "Golden_Egg_'18", "Golden_Pumpkin_'18", "Golden_Gift_'18", "Golden_Lantern_'19", "Golden_Egg_'20", "Golden_Lantern_'21", 'Furry', 'Anodized_Pearl', 'Straight_Line', 'Anodized', 'Metallic_smooth', 'Metallic_Pearl_smooth', 'Metallic_Pearl', 'Pearlescent', 'Stiletto', 'Toon_Sketch', 'Dino', 'Pearlescent_Matte', 'Burlap', 'Moon_Rock', 'Knitted_Yarn', 'Cookie_Dough', 'Glossy_Block', 'Circuit_Board', 'Camo', 'Wood', 'Carbon_Fiber', 'Sun_Damaged', 'Toon_Glossy', 'Toon_Matte', 'Toon_Wood', 'Brushed_Metal', 'Metallic_Flake', 'Zebra', 'Fiberboard']

finishedPainted = []
failedPainted = []
notFoundPainted = []
finishedUnPainted = []
failedUnpainted = []

headers = GenerateHeaders(True)

# FormatItemNames(unpainted)

# SaveFromURL("https://rl.insider.gg/en/pc/Backfire")
LoopItems()
LoopUnPaintedItems()

print("Finished Painted Items ======================================")
print(finishedPainted)

print("Failed Painted Items ======================================")
print(failedPainted)

print("Not Found Painted Items ======================================")
print(notFoundPainted)

print("Finished Unpainted Items ======================================")
print(finishedUnPainted)

print("Failed Unpainted Items ======================================")
print(failedUnpainted)