Scrapper for unity shaders

Process-Data.ps1

$json = Get-Content "$PSScriptRoot\unity_shaders_full_details.json" | ConvertFrom-Json

$shaders = $json | Where-Object { $_.url -like "*/vfx/shaders/*" }

$shadersByTagSortedByName = $shaders | ForEach-Object { $_.tags } | Group-Object | Select-Object Name, Count | Sort-Object Name

$shadersByTagSortedByCount = $shaders | ForEach-Object { $_.tags } | Group-Object | Select-Object Name, Count | Sort-Object Count -Descending

$shadersWithFiveStarts = $shaders | Where-Object { $_.metrics.rating -eq 5 -and $_.metrics.review_count -ge 10 } `
| Sort-Object { $_.metrics.favorites } -Descending

$shadersCountByPublisher = $shaders | Group-Object { $_.publisher.name } `
| Select-Object Name, Count `
| Sort-Object Count -Descending

# Calculate total sales volume per publisher using total review count as a proxy
$shadersSoldByPublisher = $shaders | Group-Object { $_.publisher.name } | ForEach-Object {
    $reviews = $_.Group | ForEach-Object { $_.metrics.review_count }
    $totalReviews = ($reviews | Measure-Object -Sum).Sum
    [PSCustomObject]@{
        Publisher        = $_.Name
        TotalShaders     = $_.Count
        TotalReviewCount = $totalReviews
    }
} | Sort-Object TotalReviewCount -Descending

$shadersWithHighInterestButMedicoreRatings = $shaders | Where-Object { 
    $_.metrics.favorites -gt 50 -and 
    $_.metrics.rating -lt 4.0 -and 
    $_.metrics.rating -gt 0 
} `
| Select-Object title, @{N = "Rating"; E = { $_.metrics.rating } }, @{N = "Favs"; E = { $_.metrics.favorites } }, url


# Use your own keywords
$keywords = @("Fog", "Hologram", "Dissolve", "sci fi", "Sci-Fi", "Scifi")

$tendencyAnalysis = $keywords | ForEach-Object {
    $kw = $_
    $matching = $shaders | Where-Object { $_.title -like "*$kw*" }
    if ($matching) {
        $avgFavs = ($matching | Measure-Object -Property { $_.metrics.favorites } -Average).Average
        [PSCustomObject]@{
            Keyword      = $kw
            TotalAssets  = $matching.Count
            AvgFavorites = [Math]::Round($avgFavs, 1)
        }
    }
} `
| Sort-Object AvgFavorites -Descending

$highlySuccessFullShadersToAnalyzePricingSweetSpots = $shaders | Where-Object { 
    $_.metrics.rating -ge 4.5 -and 
    $_.metrics.review_count -ge 10 
} `
| Select-Object title, @{N = "Price"; E = { $_.pricing.current_price } }, @{N = "Favs"; E = { $_.metrics.favorites } }, @{N = "Url"; E = { $_.url } }`
| Sort-Object Favs -Descending


$shadersWithHighWhislistFactor = $shaders | Where-Object { $_.metrics.review_count -gt 0 } `
| Select-Object title, @{N = "WishlistFactor"; E = { [Math]::Round($_.metrics.favorites / $_.metrics.review_count, 1) } }, @{N = "Url"; E = { $_.url } } `
| Sort-Object WishlistFactor -Descending 


# MODIFIED METRIC: Aggregate support counts strictly grouped by Unity MAJOR version
$majorVersionMetricsMap = @{}

foreach ($shader in $shaders) {
    if ($null -ne $shader.compatibility) {
        foreach ($comp in $shader.compatibility) {
            # Extract only the major part (e.g., "2022.3.43f1" -> "2022" or "6000.0.43f1" -> "6000")
            $rawVersion = $comp.unity_version
            if ([string]::IsNullOrEmpty($rawVersion)) { continue }
            
            $majorVersion = $rawVersion.Split('.')[0].Trim()
            
            if (-not $majorVersionMetricsMap.ContainsKey($majorVersion)) {
                $majorVersionMetricsMap[$majorVersion] = [ordered] @{
                    UnityMajorVersion      = $majorVersion
                    TotalCompatibleAssets  = 0
                    BuiltInCompatibleCount = 0
                    UrpCompatibleCount     = 0
                    HdrpCompatibleCount    = 0
                }
            }

            $hasAnyCompatibility = $false
            if ($comp.built_in -eq "Compatible") {
                $majorVersionMetricsMap[$majorVersion].BuiltInCompatibleCount++
                $hasAnyCompatibility = $true
            }
            if ($comp.urp -eq "Compatible") {
                $majorVersionMetricsMap[$majorVersion].UrpCompatibleCount++
                $hasAnyCompatibility = $true
            }
            if ($comp.hdrp -eq "Compatible") {
                $majorVersionMetricsMap[$majorVersion].HdrpCompatibleCount++
                $hasAnyCompatibility = $true
            }

            if ($hasAnyCompatibility) {
                $majorVersionMetricsMap[$majorVersion].TotalCompatibleAssets++
            }
        }
    }
}

$unityVersionCompatibilityAnalysis = $majorVersionMetricsMap.Values | ForEach-Object {
    [PSCustomObject]$_
} | Sort-Object UnityMajorVersion -Descending

$unity6MigrationGaps = $shaders | Where-Object {
    # Check if Unity 6 (6000) is missing from the compatibility list
    $hasUnity6 = $_.compatibility | Where-Object { $_.unity_version.Split('.')[0] -eq "6000" }
    $_.metrics.favorites -gt 200 -and -not $hasUnity6
} | Select-Object title, @{N = "Favs"; E = { $_.metrics.favorites } }, url | Sort-Object Favs -Descending

# Context: Current year is 2026, looking back 12 months
$oneYearAgo = (Get-Date).AddYears(-1)

$risingStars = $shaders | Where-Object {
    $_.release_date -ne "Unknown" -and 
    [datetime]$_.release_date -ge $oneYearAgo -and 
    $_.metrics.favorites -gt 30
} | Select-Object title, release_date, @{N = "Favs"; E = { $_.metrics.favorites } }, url | Sort-Object Favs -Descending

# Assets performance grouped by size
$sizeTiersAnalysis = $shaders | ForEach-Object {
    $tier = if ($_.file_size_bytes -eq 0) { "01. Unknown / Untracked" }
            elseif ($_.file_size_bytes -lt 5MB) { "02. Micro-Utility / Pure Code (<5MB)" }
            elseif ($_.file_size_bytes -lt 50MB) { "03. Standard Asset Pack (5-50MB)" }
            else { "04. Mega-Kit / Environment (>50MB)" }
            
    [PSCustomObject]@{
        Tier      = $tier
        Favorites = $_.metrics.favorites
    }
} | Group-Object Tier | ForEach-Object {
    $avgFavs = ($_.Group | Measure-Object -Property Favorites -Average).Average
    [PSCustomObject]@{
        SizeTier      = $_.Name
        TotalAssets   = $_.Count
        AvgFavorites  = [Math]::Round($avgFavs, 1)
    }
} | Sort-Object SizeTier

# Low size shader with high favs
$pureMathShaders = $shaders | Where-Object {
    $_.file_size_bytes -gt 0 -and 
    $_.file_size_bytes -lt 2MB -and 
    $_.pricing.current_price -notlike "*Free*" -and 
    $_.metrics.favorites -gt 100
} | Select-Object title, @{N = "Size_MB"; E = { [Math]::Round($_.file_size_bytes / 1MB, 2) } }, 
    @{N = "Price"; E = { $_.pricing.current_price } }, 
    @{N = "Favs"; E = { $_.metrics.favorites } }, url | Sort-Object Favs -Descending



# --- Export to CSV section ---

# Set the output directory to the script root
New-Item -Path "$PSScriptRoot/shader-data" -ItemType Directory -Force -Verbose | Out-Null
$outputDir = "$PSScriptRoot/shader-data/"

$shadersByTagSortedByName | Export-Csv -Path "$outputDir\shaders_tags_by_name.csv" -NoTypeInformation -Encoding utf8
$shadersByTagSortedByCount | Export-Csv -Path "$outputDir\shaders_tags_by_count.csv" -NoTypeInformation -Encoding utf8

$shadersWithFiveStarts | Select-Object title, url, 
    @{N = "Rating"; E = { $_.metrics.rating } }, 
    @{N = "ReviewCount"; E = { $_.metrics.review_count } }, 
    @{N = "Favorites"; E = { $_.metrics.favorites } },
    @{N = "Price"; E = { $_.pricing.current_price } } `
| Export-Csv -Path "$outputDir\shaders_top_rated.csv" -NoTypeInformation -Encoding utf8

$shadersCountByPublisher | Export-Csv -Path "$outputDir\shaders_by_publisher_count.csv" -NoTypeInformation -Encoding utf8

$shadersSoldByPublisher | Export-Csv -Path "$outputDir\shaders_by_publisher_sales_proxy.csv" -NoTypeInformation -Encoding utf8

$shadersWithHighInterestButMedicoreRatings | Export-Csv -Path "$outputDir\shaders_opportunity_high_interest_low_rating.csv" -NoTypeInformation -Encoding utf8

$tendencyAnalysis | Export-Csv -Path "$outputDir\shaders_keyword_tendency.csv" -NoTypeInformation -Encoding utf8

$highlySuccessFullShadersToAnalyzePricingSweetSpots | Export-Csv -Path "$outputDir\shaders_pricing_sweet_spots.csv" -NoTypeInformation -Encoding utf8

$shadersWithHighWhislistFactor | Export-Csv -Path "$outputDir\shaders_high_wishlist_factor.csv" -NoTypeInformation -Encoding utf8

$unityVersionCompatibilityAnalysis | Export-Csv -Path "$outputDir\shaders_compatibility_by_version.csv" -NoTypeInformation -Encoding utf8

$unity6MigrationGaps | Export-Csv -Path "$outputDir\shaders_gap_unity6_missing.csv" -NoTypeInformation -Encoding utf8

$risingStars | Export-Csv -Path "$outputDir\shaders_trending_rising_stars.csv" -NoTypeInformation -Encoding utf8

$sizeTiersAnalysis | Export-Csv -Path "$outputDir\shaders_market_performance_by_size.csv" -NoTypeInformation -Encoding utf8

$pureMathShaders | Export-Csv -Path "$outputDir\shaders_pure_technical_efficiency.csv" -NoTypeInformation -Encoding utf8


Write-Host "All analysis exported successfully to CSV files in: $outputDir" -ForegroundColor Green

unity-shader-scrapper.py

import re
import time
import json
import requests
from datetime import datetime
from concurrent.futures import ThreadPoolExecutor, as_completed
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

class UnityAsset:
    def __init__(self, title, url, publisher_name, publisher_url, discount, 
                 current_price, original_price, rating, review_count, 
                 favorites, tags=None, compatibility=None, file_size_bytes=0, release_date="Unknown"):
        self.title = title
        self.url = url
        self.publisher_name = publisher_name
        self.publisher_url = publisher_url
        self.discount = discount
        self.current_price = current_price
        self.original_price = original_price
        self.rating = rating
        self.review_count = review_count
        self.favorites = favorites
        self.tags = tags if tags is not None else []
        self.compatibility = compatibility if compatibility is not None else []
        self.file_size_bytes = file_size_bytes
        self.release_date = release_date

    @classmethod
    def from_soup(cls, card_soup, tags=None, compatibility=None):
        title_el = card_soup.find(attrs={"data-test": "package-title"})
        title = title_el.text.strip() if title_el else "Unknown"

        link_el = card_soup.find('a', href=re.compile(r'/packages/'))
        asset_url = "https://assetstore.unity.com" + link_el['href'] if link_el else None

        pub_el = card_soup.find('a', href=re.compile(r'/publishers/'))
        publisher_name = pub_el.text.strip() if pub_el else "Unknown"
        publisher_url = "https://assetstore.unity.com" + pub_el['href'] if pub_el else None

        rating_el = card_soup.find(attrs={"data-rating": True})
        rating = float(rating_el['data-rating']) if rating_el else 0.0
        review_count = int(rating_el['data-count']) if rating_el else 0

        fav_el = card_soup.find('div', class_='tjsFV')
        try:
            favorites = int(fav_el.text.strip().strip('()')) if fav_el else 0
        except ValueError:
            favorites = 0

        discount_el = card_soup.find(string=re.compile(r'-\d+%'))
        discount = discount_el.strip() if discount_el else "None"

        prices = card_soup.find_all(string=re.compile(r'€\d+'))
        current_price = prices[0].strip() if len(prices) > 0 else "Free"
        original_price = prices[1].strip() if len(prices) > 1 else current_price    

        return cls(title, asset_url, publisher_name, publisher_url, discount, 
                   current_price, original_price, rating, review_count, favorites, tags, compatibility)

    def to_dict(self):
        return {
            "title": self.title,
            "url": self.url,
            "tags": self.tags,
            "compatibility": self.compatibility,
            "file_size_bytes": self.file_size_bytes,
            "release_date": self.release_date,
            "publisher": {"name": self.publisher_name, "url": self.publisher_url},
            "pricing": {"discount": self.discount, "current_price": self.current_price, "original_price": self.original_price},
            "metrics": {"rating": self.rating, "review_count": self.review_count, "favorites": self.favorites},
        }

def parse_date_to_iso(date_str):
    """Converts a date string like 'Jun 3, 2026' into ISO format '2026-06-03'."""
    if not date_str or date_str == "Unknown":
        return "Unknown"
    try:
        # Clean any extra whitespace or line breaks
        cleaned_date = date_str.strip()
        # Parse the English format "Jun 3, 2026"
        parsed_date = datetime.strptime(cleaned_date, "%b %d, %Y")
        return parsed_date.strftime("%Y-%m-%d")
    except ValueError:
        # Fallback in case of a different format (e.g. Full month name "June 3, 2026")
        try:
            parsed_date = datetime.strptime(cleaned_date, "%B %d, %Y")
            return parsed_date.strftime("%Y-%m-%d")
        except ValueError:
            return "Unknown"

def parse_size_to_bytes(size_str):
    """Converts standard asset sizes strings (e.g., '58.4 MB', '1.2 GB') into integer bytes."""
    if not size_str:
        return 0
    match = re.search(r"([\d.,]+)\s*([a-zA-Z]+)", size_str)
    if not match:
        return 0
    
    raw_number = float(match.group(1).replace(',', ''))
    unit = match.group(2).upper()
    
    multipliers = {
        'B': 1,
        'KB': 1024,
        'MB': 1024**2,
        'GB': 1024**3,
        'TB': 1024**4
    }
    
    multiplier = multipliers.get(unit, 1)
    if multiplier == 1:
        if 'KB' in unit or 'K' in unit: multiplier = 1024
        elif 'MB' in unit or 'M' in unit: multiplier = 1024**2
        elif 'GB' in unit or 'G' in unit: multiplier = 1024**3
        
    return int(raw_number * multiplier)


def extract_total_pages(soup):
    nav = soup.find('nav', attrs={"aria-label": "Pagination"})
    if not nav:
        return 1
    buttons = nav.find_all('button', label=True)
    page_numbers = [int(btn['label']) for btn in buttons if btn['label'].isdigit()]
    return max(page_numbers) if page_numbers else 1


def create_headless_driver():
    chrome_options = Options()
    chrome_options.add_argument("--headless")
    chrome_options.add_argument("--disable-gpu")
    chrome_options.add_argument("--log-level=3")
    chrome_options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
    
    prefs = {"profile.managed_default_content_settings.images": 2}
    chrome_options.add_experimental_option("prefs", prefs)
    return webdriver.Chrome(options=chrome_options)


def scrape_grid_range(worker_id, page_indices, url_template):
    worker_assets = []
    driver = create_headless_driver()
    wait = WebDriverWait(driver, 10)
    
    try:
        for page_idx in page_indices:
            display_page_num = page_idx + 1
            driver.get(url_template.format(page_index=page_idx))
            try:
                wait.until(EC.presence_of_element_located((By.XPATH, "//a[contains(@href, '/packages/')]")))
            except Exception:
                continue
                
            soup = BeautifulSoup(driver.page_source, 'html.parser')
            title_elements = soup.find_all(attrs={"data-test": "package-title"})
            
            count = 0
            for title_el in title_elements:
                card_container = title_el.find_parent('div', class_=lambda x: x and '_1OM2f' in x)
                if card_container:
                    worker_assets.append(UnityAsset.from_soup(card_container))
                    count += 1
            print(f"[Chrome-Worker {worker_id}] Indexed page {display_page_num} -> Logged {count} assets.")
    finally:
        driver.quit()
        
    return worker_assets


def fetch_extended_details_pipeline(asset_object, http_session):
    if not asset_object.url:
        return asset_object

    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
        "Accept-Language": "en-US,en;q=0.9"
    }
    try:
        response = http_session.get(asset_object.url, headers=headers, timeout=6)
        if response.status_code == 200:
            detail_soup = BeautifulSoup(response.text, 'html.parser')
            
            # 1. Parsing Keywords
            h2_el = detail_soup.find('h2', string=re.compile(r'Related keywords', re.I))
            if h2_el and h2_el.find_parent():
                anchors = h2_el.find_parent().find_all('a', href=re.compile(r'/\?q='))
                asset_object.tags = [a.text.strip() for a in anchors if a.text.strip()]
            
            # 2. Parsing Unity Versions Compatibility Table
            th_version = detail_soup.find('th', string=re.compile(r'Unity Version', re.I))
            if th_version:
                table_el = th_version.find_parent('table')
                if table_el:
                    tbody_el = table_el.find('tbody')
                    if tbody_el:
                        compatibility_list = []
                        for row in tbody_el.find_all('tr'):
                            tds = row.find_all('td')
                            if len(tds) >= 4:
                                compatibility_list.append({
                                    "unity_version": tds[0].text.strip(),
                                    "built_in": tds[1].text.strip(),
                                    "urp": tds[2].text.strip(),
                                    "hdrp": tds[3].text.strip()
                                })
                        asset_object.compatibility = compatibility_list

            # 3. Parsing Metadata: File Size & Release Date (Hybrid CSS and Content Selector)
            size_container = detail_soup.find('div', class_='product-size')
            if not size_container:
                h4_size = detail_soup.find('h4', string=re.compile(r'File size', re.I))
                if h4_size: size_container = h4_size.find_parent('div')
            if size_container:
                size_val = size_container.find('div', class_='SoNzt') or size_container.find_all('div')[-1]
                if size_val:
                    asset_object.file_size_bytes = parse_size_to_bytes(size_val.text.strip())

            date_container = detail_soup.find('div', class_='product-date')
            if not date_container:
                h4_date = detail_soup.find('h4', string=re.compile(r'Latest release date', re.I))
                if h4_date: date_container = h4_date.find_parent('div')
            if date_container:
                date_val = date_container.find('div', class_='SoNzt') or date_container.find_all('div')[-1]
                if date_val:
                    raw_date = date_val.text.strip()
                    asset_object.release_date = parse_date_to_iso(raw_date)

    except Exception:
        pass
    return asset_object


def split_into_chunks(lst, chunk_size):
    for i in range(0, len(lst), chunk_size):
        yield lst[i:i + chunk_size]


if __name__ == "__main__":
    url_template = "https://assetstore.unity.com/?category=vfx%2Fshaders&orderBy=1&page={page_index}"
    output_filename = "unity_shaders_full_details.json"
    
    global_start_time = time.time()
    
    print("Checking total pagination scale...")
    temp_driver = create_headless_driver()
    try:
        temp_driver.get(url_template.format(page_index=0))
        WebDriverWait(temp_driver, 10).until(EC.presence_of_element_located((By.XPATH, "//a[contains(@href, '/packages/')]")))
        initial_soup = BeautifulSoup(temp_driver.page_source, 'html.parser')
        total_pages = extract_total_pages(initial_soup)
    finally:
        temp_driver.quit()
        
    print(f"Targeting {total_pages} pages in total.\n")
    
    # --- PHASE 1: DISCOVERY ---
    MAX_CHROME_WORKERS = 5
    all_pages = list(range(total_pages))
    chunk_size = max(1, len(all_pages) // MAX_CHROME_WORKERS)
    page_chunks = list(split_into_chunks(all_pages, chunk_size))
    
    discovered_assets = []
    print(f"--- PHASE 1: Starting grid discovery with {MAX_CHROME_WORKERS} Chrome instances ---")
    
    with ThreadPoolExecutor(max_workers=MAX_CHROME_WORKERS) as selenium_executor:
        futures = {
            selenium_executor.submit(scrape_grid_range, i + 1, chunk, url_template): i + 1 
            for i, chunk in enumerate(page_chunks)
        }
        for future in as_completed(futures):
            discovered_assets.extend(future.result())
            
    print(f"\nPhase 1 Complete. Found {len(discovered_assets)} assets to process.")
    
    # --- PHASE 2: EXTENDED EXTRACTION ---
    MAX_HTTP_WORKERS = 40  
    processed_assets = []
    
    print(f"--- PHASE 2: Injecting tags, compatibility, sizes & dates using {MAX_HTTP_WORKERS} threads ---")
    phase2_start = time.time()
    
    with requests.Session() as shared_session:
        with ThreadPoolExecutor(max_workers=MAX_HTTP_WORKERS) as http_executor:
            tag_futures = {
                http_executor.submit(fetch_extended_details_pipeline, asset, shared_session): asset 
                for asset in discovered_assets
            }
            
            completed_count = 0
            for future in as_completed(tag_futures):
                processed_assets.append(future.result())
                completed_count += 1
                if completed_count % 250 == 0 or completed_count == len(discovered_assets):
                    print(f"[HTTP-Engine] Extracted data for {completed_count}/{len(discovered_assets)} assets...")

    # --- EXPORT ---
    end_time = time.time()
    print(f"\nScraping workflow fully completed in {round(end_time - global_start_time, 2)} seconds.")
    
    print(f"Packaging dataset into JSON layout...")
    json_payload = [asset.to_dict() for asset in processed_assets]
    
    with open(output_filename, "w", encoding="utf-8") as json_file:
        json.dump(json_payload, json_file, indent=4, ensure_ascii=False)
        
    print(f"Success! Optimized dataset exported to: .\\{output_filename}")