A script to download all assets from Quixel (not properly tested/incomplete)

quixel.py

import requests
import time

token = TOKEN

def get_assets(page):
    url = "https://6UJ1I5A072-2.algolianet.com/1/indexes/assets/query?x-algolia-application-id=6UJ1I5A072&x-algolia-api-key=e93907f4f65fb1d9f813957bdc344892"

    params = {
        "page": page,
        "maxValuesPerFacet": 1000,
        "hitsPerPage": 1000,
        "attributesToRetrieve": ",".join(["id", "name"]),
    }

    headers = {
        "X-Api-Key": "2Zg8!d2WAHIUW?pCO28cVjfOt9seOWPx@2j",
        "Referer": "https://quixel.com/",
    }

    print(f"Getting asset IDs for page {page}...")
    assets_response = requests.post(url, headers=headers, json=params)
    if assets_response.status_code == 200:
        return assets_response.json()
    else:
        print(f"Failed to get assets for page {page}")
        print(assets_response.json())
        return None


def get_asset_download_id(asset_id):
    url = "https://quixel.com/v1/downloads"

    # You can adjust the `config` property based on your needs by
    # choosing the proper options on the website and then inspecting 
    # the payload sent to the servers on the same endpoint (/v1/downloads)
    params = {
        "asset": asset_id,
        "config": {
            "highpoly": False,
            "ztool": False,
            "lowerlod_normals": True,
            "albedo_lods": True,
            "meshMimeType": "application/x-fbx",
            "brushes": False,
            "lods": [29155, 13541, 6289, 2921],
        },
    }

    headers = {
        "X-Api-Key": "2Zg8!d2WAHIUW?pCO28cVjfOt9seOWPx@2j",
        "Authorization": "Bearer " + token,
    }

    print(f"Getting download ID for asset: {asset_id}")
    download_url_response = requests.post(url, headers=headers, json=params)
    if download_url_response.status_code == 200:
        return download_url_response.json()["id"]
    else:
        print(f"Failed to get asset download url for id: {asset_id}")
        print(download_url_response.json())
        return None


def download_asset(download_id):
    url = f"https://assetdownloads.quixel.com/download/{download_id}?preserveStructure=true&url=https%3A%2F%2Fquixel.com%2Fv1%2Fdownloads"

    delay = 5
    attempt_count = 0
    while True:
        response = requests.get(url, stream=True)
        if response.status_code != 200:
            print(f"Failed to download asset with id: {download_id}")
            return False

        filename = download_id
        content_disposition = response.headers.get("Content-Disposition")
        if content_disposition:
            filename = content_disposition.split("filename=")[-1].strip('"')

        print(f"Downloading file: {filename}")

        try:
            with open(filename, "wb") as file:
                for chunk in response.iter_content(chunk_size=8192):
                    file.write(chunk)
            break
        except requests.exceptions.ChunkedEncodingError as e:
            print(f"Caught ChunkedEncodingError: {e}")
            print(f"Retrying with attempt number {attempt_count}...")

            time.sleep(delay)

            delay += 5
            attempt_count += 1

    print(f"File downloaded successfully: {filename}")

    return True


cached_assets = set()

cache_file = open("cache.txt", "a+")
cache_file.seek(0)

for asset_id in cache_file.read().splitlines():
    cached_assets.add(asset_id)

hits = []
total_pages = 1
page = 0

while page < total_pages:
    assets = get_assets(page)
    if assets is None:
        exit()

    total_pages = assets["nbPages"]
    hits += assets["hits"]

    page += 1

for hit in hits:
    asset_id = hit["id"]
    if asset_id in cached_assets:
        print(f"{asset_id} already downloaded, skipping...")
        continue

    download_id = get_asset_download_id(asset_id)
    if download_id is None:
        continue

    downloaded = download_asset(download_id)
    if downloaded:
        cache_file.write(f"{asset_id}\n")
        cache_file.flush()

cool, but it only fetches little over 1000 items out of the available ~18800 items.
The reset give i.e.

Getting download ID for asset: vmnmbhv
Failed to get asset download url for id: vmnmbhv

hello! not sure what you mean by it fetching only 1k items? i tried to test it out by filtering the unique ids inside hits and it was rougly 18k. if you mean downloading, the error message Failed to get asset download url for id: vmnmbhv should also include an error message if it was returned from the server (make sure you're using a recent version of the script). perhaps that can give another clue.

I only get 1k and then the rest only give that same kind of error message for each and every id after the initial 1k.
There's no additional information in the cli window. (I have not checked the cache file just yet)

i just tried calling the get_asset_download_id function for vmnmbhv specifically and it worked on my end, so i'm assuming it's some form of rate limiting/authorization issue. what i find strange is that there's no error being printed afterwards in your case - the server should respond with a message as to why it 4xx'd, so perhaps try looking into what the response looks like.

hello! i only wrote this script quickly for a buddy of mine, i don't actually do 3d modeling as of now so i'm afraid i can't be of any help, sorry.

yes you can help, if you want surely... for example you could ask your friend if he managed to import the assets into the 3d software that he uses, cause if you download a bunch of files - as big as 50Tb as somebody estimated - maybe it is important to know if the assets can be used afterwards in 3d modeling

strange, are you sure they're done downloading by the time you try to open them?

yes, I'm sure

works on his end.

works on his end.

your response is partially helpful... so the conclusion is this script is partially working: it downloads files, but it does not guarantee that it downloads all the assets from Quixel or that the assets will work on your end

Hey folks, I modified this script to make it faster and per category (and less queries on their server).

https://gist.github.com/maalrron/877b2edb23cc5d99d6a6b4c22f708e58

I just made a script for this have a look if that would help