A script for automatically downloading all your Glitch projects, and their associated CDN assets

download.py

import sys, os, shutil, json, subprocess
from http.client import InvalidURL
from urllib.request import Request, urlopen, urlretrieve, URLError
from urllib.parse import quote, unquote
from time import time, sleep

print("\nWelcome to the Glitch.com bulk project downloader.")

# Verify that we have tar available, otherwise thing
bypass_tar = False
try:
    subprocess.run(["tar", "--help"], capture_output=True, text=True)
except:
    print("")
    print("WARNING: could not find 'tar'!")
    print("")
    print("This script works best when the 'tar' command is available,")
    print("would you like to continue in download-only mode?")
    print("")
    print("Note: you will need to unpack each download manually, and rename")
    print("the resulting 'app' dir manually. You cannot run a bulk-unpack,")
    print("every archive will unpack to the same directory name!!")
    print("")
    try:
        no_tar = input("Proceed in download-only mode? (Y/n) ").lower()
        if no_tar != "y":
            exit(1)
        bypass_tar = True
    except KeyboardInterrupt:
        exit()

args = sys.argv
no_assets = "--no-assets" in args
no_skip = "--no-skip" in args
fetch_archived = "--archived" in args # New flag to fetch archived projects																		   

def get_values():
    """
    Ask for user credentials, unless they were already provided on the command line
    """
    if len(args) > 1:
        user_id = args[1]
    else:
        print("\nPlease enter your user id. You can find this in the browser with glitch.com open, by opening your dev tools and running:\n")
        print("  JSON.parse(localStorage.cachedUser).id\n")
        user_id = input("Your user id: ")

    if len(args) > 2:
        user_token = args[2]
    else:
        print("\nPlease enter your persisten token. You can find this in the browser with glitch.com open, by opening your dev tools and running:\n")
        print("  JSON.parse(localStorage.cachedUser).persistentToken\n")
        user_token = input("Your persistent token: ")
    user_token = user_token.replace("'",'').replace('"','')

    return (user_id, user_token,)

def get_project_list(user_id, user_token, get_archived=False): # Added get_archived parameter
    """
    Ask for user credentials, unless they were already provided on the command line
	If get_archived is True, fetches deleted/archived projects.
    """

    if get_archived:
        # Use the URL for deleted/archived projects
        url = f"https://api.glitch.com/v1/users/by/id/deletedProjects?id={user_id}&limit=400"
        print("Fetching archived project list...")
    else:
        # Use the URL for active projects
        url = f"https://api.glitch.com/v1/users/by/id/projects?id={user_id}&limit=400"
        print("Fetching active project list...")

    req = Request(url)
    req.add_header('Authorization', user_token)

    try:
        text = urlopen(req).read().decode("utf-8")
        try:
            return json.loads(text)
        except:
            print("could not parse JSON")
    except URLError as e:
        print(f"could not open {url}")
        print(e)
    return {}

def wait_for_dir(dir_path, timeout=10, poll_interval=1):
    """
    Waits until a directory exists or a timeout is reached.
    """
    start_time = time()
    while time() - start_time < timeout:
        if os.path.isdir(dir_path):
            return True
        sleep(poll_interval)
    return False

def download_project(user_token, project):
    """
    Download a project archive from Glitch, unpack it, and rename the dir from "app" to the project domain.
    """
    project_id = project.get("id")
    project_title = project.get("domain", project_id)
    if os.path.exists(project_title):
        if not no_skip:
            print(f"Skipping {project_title} (already downloaded)")
            return
        else:
            shutil.rmtree(f"./{project_title}", ignore_errors=False, onerror=None)
    url = f"https://api.glitch.com/project/download/?authorization={user_token}&projectId={project_id}"
    file = f"./{project_title}.tgz"
    print(f"\nDownloading '{project_title}'...")
    result = urlretrieve(url, file)
    if bypass_tar is False:
        print("Unpacking...")
        unpacked_dir = './app'
        if os.path.exists(unpacked_dir):
            shutil.rmtree(unpacked_dir, ignore_errors=False, onerror=None)
        command = f"tar -xvzf {file}"
        unpack = subprocess.run(command.split(), capture_output=True, text=True)
        wait_for_dir(unpacked_dir)
        if not os.path.isdir(unpacked_dir):
            print(f"ERROR: {project_title} did not extract to {unpacked_dir}!")
        else:
            shutil.move(unpacked_dir, f"./{project_title}")
            os.remove(file)
            if no_assets is False:
                download_assets(project_title)

def download_assets(project_title):
    """
    Download all assets associated with this project
    """
    # It is a major failing of Python that we can't tell
    # it to halt execution until shutils is done...
    while not os.path.exists(project_title):
        sleep(0.1)  # Check every 100ms
    dir = f"./{project_title}/glitch-assets"
    os.makedirs(dir, exist_ok=True)
    print(f"Downloading all assets into {dir}...")
    assets = {}
    try:
        with open(f"./{project_title}/.glitch-assets") as asset_file:
            for line in asset_file:
                if line.isspace():
                    continue
                """
                Aggregate our asset records, keyed on uuid, invalidating
                any record that has a "deleted" record.
                """
                record = json.loads(line)
                uuid = record["uuid"]
                deleted = record.get("deleted", False)
                have_record = assets.get(uuid, None)
                if have_record is not None and deleted is not False:
                    assets[uuid] = False
                else:
                    assets[uuid] = record
    except Exception as e:
        print(f"glitch-assets error for {project_title}: {e}")
    for entry in  [x for x in assets.values() if x is not False]:
        # Do a bit of URL hackery because there's a surprising number
        # of bad URLs in people's glitch assets files...
        name = entry["name"]
        url = entry["url"].replace("%3A", ":").replace("%2F", "/").replace(" ", "%20")
        dest = f"{dir}/{name}"
        print(f"Downloading {name} from {url}...")
        try:
            urlretrieve(url, dest)
        except URLError as e:
            print(f"error getting url: {e}")
        except ValueError as e:
            print(f"bad url: {e}")
        except InvalidURL as e:
            print(f"invalid url: {e}")


"""
Let's get this bulk download going:
"""

try:
    (user_id, user_token) = get_values()
    
    # Pass the fetch_archived flag to get_project_list
    data = get_project_list(user_id, user_token, fetch_archived) 
    
    # The structure of the response for deletedProjects might be different.
    # Assuming it's similar to active projects with an 'items' list.
    # If 'items' is not present, or has a different name, this needs adjustment.
    items = []
    if isinstance(data, list): # The deletedProjects endpoint returns a list directly
    	items = data
    	print(f"Found {len(items)} archived projects.")
    elif isinstance(data, dict) and 'items' in data: # The active projects endpoint returns a dict with 'items'
    	items = data.get('items', [])
    	print(f"Found {len(items)} active projects.")
    else:
    	print("Could not find project items in the response. The API response structure might have changed or an error occurred.")
    	if data:
    		print(f"API Response (first 500 chars): {str(data)[:500]}")
    
    
    if not items:
    	project_type = "archived" if fetch_archived else "active"
    	print(f"No {project_type} projects found or an error occurred fetching them.")
    else:
    	project_type_plural = "archived projects" if fetch_archived else "projects"
    	print(f"Starting the download for {len(items)} {project_type_plural}...")
    
    	for project in items:
    		download_project(user_token, project)

except KeyboardInterrupt:
    exit(1)

print("")
print("*** Finished downloading all your projects ***")
print("")
print("NOTE: asset URLs were not automatically replaced in any source")
print("      code, so you will still need to replace CDN URLs in your")
print("      code with relative links to the ./glitch-assets directory.")
print("")

Enhanced Glitch Project Downloader

This script is an enhanced version of the Glitch project downloader originally created by Pomax. It allows users to bulk download their Glitch projects, including their assets.

Key Enhancements

This version introduces a significant new feature to the Glitch project downloader:

• Archived Project Download (--archived flag):

  • The most notable enhancement is the ability to download your archived (deleted) Glitch projects.

  • You can now use the --archived command-line flag to specify that you want to fetch and download projects that have been moved to your archive.

  • This provides a crucial way to recover or back up projects that are no longer active on your Glitch dashboard.

How to Use the New Feature

To download your archived projects, simply run the script with the --archived flag:

python download.py <your_user_id> <your_persistent_token> --archived

Other Improvements

• Improved Project List Handling: The script now includes more robust logic to handle the different API responses for active and archived project lists, ensuring it correctly identifies and processes projects from both endpoints.

@chriskyfung to prevent confusion for users, mind if I fold these changes into the main script, with an attribution comment to point out you helped out? (we'll be adding instructions for using those to the official help docs)

@Pomax Feel free to fold in the changes—appreciate the attribution. Looking forward to the updated help docs!