Download random/curated images from Unsplash

download.ts

import { writeFile, mkdir } from "fs/promises";
import { join } from "path";
import PQueue from "p-queue";
import ora from "ora";
import chalk from "chalk";

// Add these type declarations at the top
declare module "fs/promises";
declare module "path";
declare global {
  var Buffer: any;
}

// Add slugify function
function slugify(text: string): string {
  return text
    .toString()
    .toLowerCase()
    .trim()
    .replace(/\s+/g, "-") // Replace spaces with -
    .replace(/[^\w\-]+/g, "") // Remove all non-word chars
    .replace(/\-\-+/g, "-") // Replace multiple - with single -
    .replace(/^-+/, "") // Trim - from start of text
    .replace(/-+$/, ""); // Trim - from end of text
}

// Add this helper function near the top
function addParamsToUrl(
  url: string,
  params: Record<string, string | number>
): string {
  const urlObj = new URL(url);
  Object.entries(params).forEach(([key, value]) => {
    urlObj.searchParams.set(key, value.toString());
  });
  return urlObj.toString();
}

const UNSPLASH_ACCESS_KEY =
  "<your unsplash key>";
const BASE_DIR = "./unsplash_images";
const IMAGE_COUNT_RANDOM = 49;
const IMAGE_COUNT_EDITORIAL = 200;
const MAX_FILENAME_LENGTH = 255; // Maximum filename length for most filesystems
const MAX_PATH_LENGTH = 255; // Maximum path length for most filesystems
const DEBUG = process.argv.includes("--debug");

// Add this helper function
function debugLog(message: string, data?: any) {
  if (DEBUG) {
    console.log("[DEBUG]", message);
    if (data) {
      console.log(data);
    }
  }
}

// Add helper function to get reset time
function getResetTime(): Date {
  const resetTime = new Date();
  resetTime.setHours(resetTime.getHours() + 1);
  return resetTime;
}

// Add these constants near the top with other constants
const CONCURRENT_DOWNLOADS = 8;
const queue = new PQueue({ concurrency: CONCURRENT_DOWNLOADS });
const spinner = ora();

// Add this helper function for formatted logging
function formatLog(
  type: "success" | "error" | "info" | "warning",
  message: string
): void {
  const icons = {
    success: chalk.green("✓"),
    error: chalk.red("✗"),
    info: chalk.blue("ℹ"),
    warning: chalk.yellow("⚠"),
  };
  console.log(`${icons[type]} ${message}`);
}

// Update downloadImage function
async function downloadImage(
  url: string,
  filepath: string,
  retries = 2
): Promise<{ success: boolean; error?: string }> {
  try {
    const response = await fetch(url);
    if (!response.ok) {
      throw new Error(`HTTP error! status: ${response.status}`);
    }
    const buffer = await response.arrayBuffer();
    await writeFile(filepath, Buffer.from(buffer));
    return { success: true };
  } catch (error) {
    if (retries > 0) {
      formatLog("warning", `Retrying download for ${chalk.cyan(filepath)}...`);
      return downloadImage(url, filepath, retries - 1);
    }
    return {
      success: false,
      error: error instanceof Error ? error.message : "Unknown error",
    };
  }
}

// Add new constants
const MODE = process.argv.includes("--editorial") ? "editorial" : "random";
const PHOTOS_PER_PAGE = 30; // Unsplash default
const PAGES_TO_FETCH = Math.ceil(IMAGE_COUNT_EDITORIAL / PHOTOS_PER_PAGE);

// Update downloadRandomPhotos function
async function downloadRandomPhotos(): Promise<void> {
  let remainingRequests: string | null = null;
  let successCount = 0;
  let failureCount = 0;

  spinner.start("Preparing to download random photos...");

  const downloadTasks = Array.from(
    { length: IMAGE_COUNT_RANDOM },
    async (_, i) => {
      try {
        spinner.text = `Fetching photo information ${
          i + 1
        }/${IMAGE_COUNT_RANDOM}`;
        const response = await fetch("https://api.unsplash.com/photos/random", {
          headers: {
            Authorization: `Client-ID ${UNSPLASH_ACCESS_KEY}`,
          },
        });

        debugLog("Response headers:", Object.fromEntries(response.headers));

        if (response.status === 429) {
          throw new Error("rate limit");
        }

        remainingRequests = response.headers.get("x-ratelimit-remaining");
        if (remainingRequests && parseInt(remainingRequests) <= 0) {
          throw new Error("rate limit");
        }

        const photo = await response.json();
        debugLog("Response body:", photo);

        const category = photo.topic_submissions
          ? Object.keys(photo.topic_submissions)[0] || "uncategorized"
          : "uncategorized";
        const dirPath = join(BASE_DIR, category);

        await mkdir(dirPath, { recursive: true });

        // Create filename using id, username and title
        const title = photo.description || "untitled";
        let filename = `${photo.id}-${photo.user.username}-${slugify(
          title
        )}.jpg`;
        let filepath = join(dirPath, filename);

        // Truncate filename if the full path is too long
        if (filepath.length > MAX_PATH_LENGTH) {
          const extension = ".jpg";
          const idAndUsername = `${photo.id}-${photo.user.username}-`;
          const basePathLength = dirPath.length + 1; // +1 for path separator
          const maxTitleLength =
            MAX_PATH_LENGTH -
            basePathLength -
            idAndUsername.length -
            extension.length;
          const truncatedTitle = slugify(title).slice(
            0,
            Math.max(0, maxTitleLength)
          );
          filename = `${idAndUsername}${truncatedTitle}${extension}`;
          filepath = join(dirPath, filename);
        }

        return queue.add(async () => {
          try {
            const rawUrlWithParams = addParamsToUrl(photo.urls.raw, {
              w: 2500,
              dpr: 2,
              auto: "format",
            });

            spinner.text = `Downloading image ${
              successCount + failureCount + 1
            }/${IMAGE_COUNT_RANDOM}`;
            const result = await downloadImage(rawUrlWithParams, filepath);
            if (result.success) {
              successCount++;
              spinner.stop();
              formatLog(
                "success",
                `Downloaded raw resolution image ${chalk.bold(
                  successCount + failureCount
                )}: ${chalk.cyan(filepath)}`
              );
              spinner.start();
              return;
            }

            const fullResult = await downloadImage(photo.urls.full, filepath);
            if (fullResult.success) {
              successCount++;
              spinner.stop();
              formatLog(
                "success",
                `Downloaded full resolution image ${chalk.bold(
                  successCount + failureCount
                )}: ${chalk.cyan(filepath)}`
              );
              spinner.start();
              return;
            }

            const regularResult = await downloadImage(
              photo.urls.regular,
              filepath
            );
            if (regularResult.success) {
              successCount++;
              spinner.stop();
              formatLog(
                "success",
                `Downloaded regular resolution image ${chalk.bold(
                  successCount + failureCount
                )}: ${chalk.cyan(filepath)}`
              );
              spinner.start();
              return;
            }

            failureCount++;
            spinner.stop();
            formatLog(
              "error",
              `Failed to download image after all attempts: ${chalk.red(
                filepath
              )}`
            );
            spinner.start();
          } catch (error) {
            failureCount++;
            spinner.stop();
            formatLog(
              "error",
              `Error downloading image: ${chalk.red(filepath)}`
            );
            spinner.start();
          }
        });
      } catch (error) {
        if (error instanceof Error && error.message.includes("rate limit")) {
          throw error;
        }
        failureCount++;
        spinner.stop();
        formatLog("error", `Error processing image ${chalk.bold(i + 1)}`);
        spinner.start();
      }
    }
  );

  try {
    await Promise.all(downloadTasks);
  } catch (error) {
    spinner.stop();
    if (error instanceof Error && error.message.includes("rate limit")) {
      const resetTime = getResetTime();
      formatLog(
        "warning",
        `Rate limit reached. Resets at: ${chalk.yellow(
          resetTime.toLocaleString()
        )}`
      );
      return;
    }
    throw error;
  }

  spinner.stop();
  console.log("\n" + chalk.bold("Download Summary:"));
  formatLog(
    "success",
    `Successfully downloaded: ${chalk.green(successCount)} images`
  );
  formatLog("error", `Failed downloads: ${chalk.red(failureCount)} images`);
  if (remainingRequests) {
    formatLog(
      "info",
      `Remaining API requests: ${chalk.blue(remainingRequests)}`
    );
  }
}

// Update downloadEditorialPhotos function
async function downloadEditorialPhotos(): Promise<void> {
  let remainingRequests: string | null = null;
  let downloadedCount = 0;
  let successCount = 0;
  let failureCount = 0;

  spinner.start("Starting editorial photo downloads...");

  for (
    let page = 1;
    page <= PAGES_TO_FETCH && downloadedCount < IMAGE_COUNT_EDITORIAL;
    page++
  ) {
    try {
      spinner.text = `Fetching page ${page} of ${PAGES_TO_FETCH}...`;
      const response = await fetch(
        `https://api.unsplash.com/photos?page=${page}&per_page=${PHOTOS_PER_PAGE}`,
        {
          headers: {
            Authorization: `Client-ID ${UNSPLASH_ACCESS_KEY}`,
          },
        }
      );

      debugLog("Response headers:", Object.fromEntries(response.headers));

      if (response.status === 429) {
        throw new Error("rate limit");
      }

      remainingRequests = response.headers.get("x-ratelimit-remaining");
      if (remainingRequests && parseInt(remainingRequests) <= 0) {
        throw new Error("rate limit");
      }

      const photos = await response.json();
      spinner.stop();
      formatLog(
        "info",
        `Processing ${chalk.blue(
          Math.min(photos.length, IMAGE_COUNT_EDITORIAL - downloadedCount)
        )} images from page ${chalk.bold(page)}`
      );
      spinner.start();

      const pageDownloadTasks = photos.map(async (photo) => {
        if (downloadedCount >= IMAGE_COUNT_EDITORIAL) return;
        downloadedCount++;

        const category = photo.topic_submissions
          ? Object.keys(photo.topic_submissions)[0] || "uncategorized"
          : "uncategorized";
        const dirPath = join(BASE_DIR, category);

        await mkdir(dirPath, { recursive: true });

        const title = photo.description || "untitled";
        let filename = `${photo.id}-${photo.user.username}-${slugify(
          title
        )}.jpg`;
        let filepath = join(dirPath, filename);

        if (filepath.length > MAX_PATH_LENGTH) {
          const extension = ".jpg";
          const idAndUsername = `${photo.id}-${photo.user.username}-`;
          const basePathLength = dirPath.length + 1;
          const maxTitleLength =
            MAX_PATH_LENGTH -
            basePathLength -
            idAndUsername.length -
            extension.length;
          const truncatedTitle = slugify(title).slice(
            0,
            Math.max(0, maxTitleLength)
          );
          filename = `${idAndUsername}${truncatedTitle}${extension}`;
          filepath = join(dirPath, filename);
        }

        return queue.add(async () => {
          try {
            spinner.text = `Downloading image ${
              successCount + failureCount + 1
            }/${IMAGE_COUNT_EDITORIAL}`;
            const rawUrlWithParams = addParamsToUrl(photo.urls.raw, {
              w: 2500,
              dpr: 2,
              auto: "format",
            });

            const result = await downloadImage(rawUrlWithParams, filepath);
            if (result.success) {
              successCount++;
              spinner.stop();
              formatLog(
                "success",
                `Downloaded raw resolution image ${chalk.bold(
                  successCount + failureCount
                )}: ${chalk.cyan(filepath)}`
              );
              spinner.start();
              return;
            }

            const fullResult = await downloadImage(photo.urls.full, filepath);
            if (fullResult.success) {
              successCount++;
              spinner.stop();
              formatLog(
                "success",
                `Downloaded full resolution image ${chalk.bold(
                  successCount + failureCount
                )}: ${chalk.cyan(filepath)}`
              );
              spinner.start();
              return;
            }

            const regularResult = await downloadImage(
              photo.urls.regular,
              filepath
            );
            if (regularResult.success) {
              successCount++;
              spinner.stop();
              formatLog(
                "success",
                `Downloaded regular resolution image ${chalk.bold(
                  successCount + failureCount
                )}: ${chalk.cyan(filepath)}`
              );
              spinner.start();
              return;
            }

            failureCount++;
            spinner.stop();
            formatLog(
              "error",
              `Failed to download image after all attempts: ${chalk.red(
                filepath
              )}`
            );
            spinner.start();
          } catch (error) {
            failureCount++;
            spinner.stop();
            formatLog(
              "error",
              `Error downloading image: ${chalk.red(filepath)}`
            );
            spinner.start();
          }
        });
      });

      await Promise.all(pageDownloadTasks);
      spinner.stop();
      formatLog("success", `Completed page ${chalk.bold(page)}`);
      spinner.start();
    } catch (error) {
      spinner.stop();
      if (error instanceof Error && error.message.includes("rate limit")) {
        const resetTime = getResetTime();
        formatLog(
          "warning",
          `Rate limit reached. Resets at: ${chalk.yellow(
            resetTime.toLocaleString()
          )}`
        );
        break;
      }
      formatLog("error", `Error processing page ${chalk.bold(page)}`);
    }
  }

  spinner.stop();
  console.log("\n" + chalk.bold("Download Summary:"));
  formatLog(
    "success",
    `Successfully downloaded: ${chalk.green(successCount)} images`
  );
  formatLog("error", `Failed downloads: ${chalk.red(failureCount)} images`);
  if (remainingRequests) {
    formatLog(
      "info",
      `Remaining API requests: ${chalk.blue(remainingRequests)}`
    );
  }
}

// Update the main execution
if (MODE === "editorial") {
  downloadEditorialPhotos().then(() => console.log("Download complete!"));
} else {
  downloadRandomPhotos().then(() => console.log("Download complete!"));
}

package.json

{
  "name": "unsplash-random-downloader",
  "version": "1.0.0",
  "description": "",
  "main": "index.js",
  "scripts": {
    "test": "echo \"Error: no test specified\" && exit 1"
  },
  "keywords": [],
  "author": "",
  "license": "ISC",
  "dependencies": {
    "chalk": "^5.4.1",
    "ora": "^8.2.0",
    "p-queue": "^8.1.0"
  },
  "devDependencies": {
    "@types/node": "^22.13.5"
  }
}