scrape assets with categories ects.

gistfile1.txt

import fetch from 'node-fetch';
import { createObjectCsvWriter } from 'csv-writer';

const GRAPHQL_ENDPOINT = 'https://source-api.substance3d.com/beta/graphql';
const PAGE_SIZE = 100;
const MAX_RETRIES = 3;

// Updated query with categories and collections
const query = `
  query Assets($page: Int!) {
    assets(limit: ${PAGE_SIZE}, sortDir: desc, sort: byPublicationDate, page: $page) {
      total
      hasMore
      items {
        id
        title
        categories
        collections {
          id
          title
        }
        attachments {
          ... on DownloadAttachment {
            url
            filename
            mimetype
            size
            md5
          }
        }
      }
    }
  }
`;

// CSV writer configuration
const csvWriter = createObjectCsvWriter({
  path: 'assets.csv',
  header: [
    { id: 'id', title: 'ID' },
    { id: 'title', title: 'Title' },
    { id: 'categories', title: 'Categories' },
    { id: 'collections', title: 'Collections' },
    { id: 'size', title: 'Size' },
    { id: 'url', title: 'URL' },
    { id: 'filename', title: 'Filename' },
    { id: 'mimetype', title: 'Mimetype' },
    { id: 'md5', title: 'MD5' }
  ]
});

async function fetchPage(page) {
  for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
    try {
      const response = await fetch(GRAPHQL_ENDPOINT, {
        method: 'POST',
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify({ query, variables: { page } })
      });

      if (!response.ok) {
        console.warn(`⚠️ Retry ${attempt}/${MAX_RETRIES} for page ${page}: HTTP ${response.status}`);
        await new Promise(r => setTimeout(r, 1000 * attempt));
        continue;
      }

      const json = await response.json();

      if (json.errors) {
        console.warn(`⚠️ Retry ${attempt}/${MAX_RETRIES} for page ${page}: GraphQL errors`, json.errors);
        await new Promise(r => setTimeout(r, 1000 * attempt));
        continue;
      }

      return json.data.assets;
    } catch (e) {
      console.warn(`⚠️ Retry ${attempt}/${MAX_RETRIES} for page ${page}: ${e.message}`);
      await new Promise(r => setTimeout(r, 1000 * attempt));
    }
  }

  console.error(`❌ Failed to fetch page ${page} after ${MAX_RETRIES} retries, skipping.`);
  return null;
}

async function fetchAllAssets() {
  let allRecords = [];
  let currentPage = 0;
  let hasMore = true;

  while (hasMore) {
    console.log(`Fetching page ${currentPage}...`);
    const assets = await fetchPage(currentPage);
    if (!assets) {
      // skip page on failure
      currentPage++;
      continue;
    }

    const records = assets.items.flatMap(item => {
      const cats = Array.isArray(item.categories) ? item.categories.join(', ') : '';
      const cols = Array.isArray(item.collections) ? item.collections.map(c => c.title).join(', ') : '';

      return item.attachments
        .filter(att => att.url)
        .map(att => ({
          id: item.id,
          title: item.title,
          categories: cats,
          collections: cols,
          size: att.size,
          url: att.url,
          filename: att.filename,
          mimetype: att.mimetype,
          md5: att.md5
        }));
    });

    allRecords = allRecords.concat(records);

    console.log(`Fetched page ${currentPage + 1}, ${records.length} items`);

    hasMore = assets.hasMore;
    currentPage++;

    await new Promise(r => setTimeout(r, 800)); // avoid API flooding
  }

  // sort ascending by size
  allRecords.sort((a, b) => a.size - b.size);

  try {
    await csvWriter.writeRecords(allRecords);
    console.log(`Successfully saved ${allRecords.length} records to assets.csv`);
  } catch (err) {
    console.error('Error writing CSV:', err);
  }
}

fetchAllAssets();