buildwith scraper using spider browser

buildwith_test.ts

/**
 * BuiltWith detailed-profile scraper — uses the `spider-browser` SDK.
 *
 * For now this just fetches the target page and writes the raw HTML
 * to ./output/. No DOM extraction yet — we'll layer that on once we
 * confirm the page loads cleanly through Spider.
 *
 * Usage (from repo root):
 *   npx tsx builtwith_test.ts
 */

import { config as loadEnv } from "dotenv";
import { writeFileSync, mkdirSync } from "node:fs";
import { join } from "node:path";
import { SpiderBrowser, BlockedError } from "spider-browser";

// Walk up from this script's dir until we find a .env (repo-root fallback).
loadEnv({ path: join(__dirname, ".env") });
loadEnv({ path: join(__dirname, "..", ".env") });

// ── Configuration ─────────────────────────────────────────────────────

const SPIDER_API_KEY = process.env.SPIDER_API_KEY;
if (!SPIDER_API_KEY) {
  throw new Error("SPIDER_API_KEY is required in .env");
}

const OUTPUT_DIR = join(__dirname, "output");
const TARGET_URL = "https://builtwith.com/detailed/hardlotion.com";

// ── Block detection ───────────────────────────────────────────────────

/**
 * BuiltWith fronts with Cloudflare. A real detailed page is ~200KB+ of
 * HTML; a challenge shell is a tiny page with a Turnstile iframe. Anything
 * suspiciously small that mentions the challenge endpoints is a block.
 */
function isBlockedPage(html: string, title: string): boolean {
  if (html.length < 5000) {
    if (
      html.includes("challenges.cloudflare.com") ||
      html.includes("captcha-delivery.com") ||
      /just a moment/i.test(title) ||
      /attention required/i.test(title)
    ) {
      return true;
    }
  }
  return false;
}

// ── Main ──────────────────────────────────────────────────────────────

async function main(): Promise<void> {
  const browser = new SpiderBrowser({
    apiKey: SPIDER_API_KEY!,
    // BuiltWith uses Cloudflare but isn't as aggressive as G2's DataDome.
    // Start at stealth 2 and let smartRetry escalate if we get blocked —
    // cheaper than opening at max stealth every run.
    stealth: 2,
    captcha: "solve",
    smartRetry: true,
    maxRetries: 5,
    record: false,
    country: "US",
    // Pre-warms the server's proxy/profile selection for this domain.
    url: TARGET_URL,
    logLevel: "info",
  });

  // Observability — same hooks as the G2 scraper. These surface what
  // smart-retry is doing under the hood if the page doesn't load cleanly.
  browser.on("captcha.detected", ({ types }) =>
    console.log(`   ⚠ captcha detected: ${types.join(", ")}`));
  browser.on("captcha.solved", () =>
    console.log(`   ✓ captcha solved`));
  browser.on("stealth.escalated", ({ from, to, reason }) =>
    console.log(`   ↑ stealth ${from} → ${to} (${reason})`));
  browser.on("browser.switching", ({ from, to, reason }) =>
    console.log(`   ⇄ browser ${from} → ${to} (${reason})`));
  browser.on("retry.attempt", ({ attempt, maxRetries, error }) =>
    console.log(`   ↻ retry ${attempt}/${maxRetries}: ${error}`));

  await browser.init();

  let html = "";
  let title = "";
  let finalUrl = "";

  try {
    console.log(`\n[Fetching] ${TARGET_URL}`);

    await browser.withRetry(async () => {
      await browser.goto(TARGET_URL);
      try {
        await browser.page.waitForNetworkIdle(15_000);
      } catch {
        /* non-fatal — page may still be complete */
      }

      html = await browser.page.rawContent();
      title = await browser.page.title();
      finalUrl = await browser.page.url();

      if (isBlockedPage(html, title)) {
        throw new BlockedError(
          `Interstitial detected (${html.length} chars, title=${JSON.stringify(title)})`,
        );
      }
    });

    console.log(`   title: ${JSON.stringify(title)}`);
    console.log(`   url:   ${finalUrl}`);
    console.log(`   html:  ${html.length} chars`);

    // ── Write raw HTML ──
    mkdirSync(OUTPUT_DIR, { recursive: true });
    const htmlPath = join(OUTPUT_DIR, "builtwith_hardlotion.html");
    writeFileSync(htmlPath, html);
    console.log(`\n  Saved: ${htmlPath}`);
  } finally {
    try {
      const credits = await browser.getSessionCredits();
      console.log(`\n  Credits used this session: ${credits}`);
    } catch {
      // getSessionCredits may fail if the WS is already gone — ignore.
    }
    await browser.close();
  }

  console.log("\nDone!");
}

main().catch((err) => {
  console.error("\nFATAL:", err);
  process.exit(1);
});