Node.js script that interacts with the GitHub API to fetch information about my repositories

fetch-github-repos.js

import fs from 'fs/promises';

/**
 * Prompt:
 *
 * I need a Node.js script that interacts with the GitHub API to fetch information about my repositories. The script should:
 *
 * - Use the GitHub API to fetch all repositories for a specific user, separating them into own repositories and forked repositories.
 * - For each repository, fetch the package.json file and extract the dependencies and devDependencies.
 * - If the repository is a monorepo (contains a workspaces field in package.json), recursively search for package.json files in subdirectories, excluding certain directories.
 * - Store the dependencies and devDependencies for each repository and sub-package in an array.
 * - Save the collected data into a packages.json file, including the total number of repositories, own repositories, and forked repositories.
 * - Handle errors gracefully and log appropriate messages.
 *
 * The script should use modern JavaScript features (ES6+), including async/await for asynchronous operations. It should also use the node fetch ntaive for making HTTP requests and the fs/promises module for file operations.
 *
 * Here are the specific requirements:
 *
 * - Use the GitHub username and a personal access token for authentication.
 * - Exclude certain directories (e.g., src, define, styles, test, demo, dev, dist) when searching for sub-packages.
 * - If a repository is a fork, do not include its dependencies and devDependencies in the output.
 * - The output JSON file should be formatted with 2-space indentation.
 * - Include the namespace (name field from package.json) in the output for each repository and sub-package.
 * - Implement pagination to handle large numbers of repositories.
 * - Ensure that the script can handle errors gracefully, including network errors and JSON parsing errors.
 * - Log appropriate messages for debugging and information purposes.
 * - Use modular functions to separate concerns, such as fetching repositories, fetching package.json files, and processing dependencies.
 */

const GITHUB_USERNAME = 'xx';
const GITHUB_TOKEN =
  'xxx';

const headers = {
  Authorization: `token ${GITHUB_TOKEN}`,
  'User-Agent': GITHUB_USERNAME,
};

const EXCLUDED_DIRS = ['src', 'define', 'styles', 'test', 'demo', 'dev', 'dist'];

/**
 * Get all user repositories with pagination and separation of forks.
 * @param {number} [page=1] - The page number to fetch.
 * @param {Object} [repos={ ownRepos: [], forkRepos: [] }] - The repositories object to store results.
 * @returns {Promise<Object>} The repositories object containing ownRepos and forkRepos arrays.
 */
async function getRepos(page = 1, repos = {ownRepos: [], forkRepos: []}) {
  const url = `https://api.github.com/users/${GITHUB_USERNAME}/repos?type=owner&per_page=100&page=${page}`;
  const response = await fetch(url, {headers});
  if (!response.ok) {
    throw new Error(`Error fetching repos: ${response.statusText}`);
  }
  const data = await response.json();

  data.forEach(
    /**
     * @param {Object} repo
     */ (repo) => {
      if (repo.fork) {
        repos.forkRepos.push(repo);
      } else {
        repos.ownRepos.push(repo);
      }
    }
  );

  if (data.length === 100) {
    return getRepos(page + 1, repos);
  } else {
    return repos;
  }
}

/**
 * Get the content of a package.json file given a repository and path.
 * @param {Object} repo - The repository object.
 * @param {string} [path='package.json'] - The path to the package.json file.
 * @returns {Promise<Object|null>} The parsed package.json content or null if not found.
 */
async function getPackageJson(repo, path = 'package.json') {
  const url = `https://api.github.com/repos/${repo.owner.login}/${repo.name}/contents/${path}`;
  try {
    const response = await fetch(url, {headers});
    if (!response.ok) {
      throw new Error(`Error fetching ${path} for repo ${repo.name}: ${response.statusText}`);
    }
    const data = await response.json();
    const content = Buffer.from(data.content, 'base64').toString('utf8');
    return JSON.parse(content);
  } catch (error) {
    console.error(`Could not fetch ${path} for repository ${repo.name}: ${error.message}`);
    return null;
  }
}

/**
 * Search for package.json in normal repositories and, for own repos, in possible monorepo folders.
 * @param {Object} repo - The repository object.
 * @param {boolean} [isFork=false] - Whether the repository is a fork.
 * @returns {Promise<Array>} The dependencies array.
 */
async function getDependencies(repo, isFork = false) {
  const packageJson = await getPackageJson(repo);
  const dependencies = [];

  if (packageJson) {
    dependencies.push({
      repo: repo.name,
      namespace: packageJson.name,
      path: 'package.json',
      dependencies: isFork ? undefined : packageJson.dependencies,
      devDependencies: isFork ? undefined : packageJson.devDependencies,
    });
  }

  if (!isFork && packageJson?.workspaces) {
    const subPackages = await getSubPackages(repo);
    if (subPackages.length > 0) {
      for (const subPackage of subPackages) {
        const subPackageJson = await getPackageJson(repo, subPackage.path);
        if (subPackageJson) {
          dependencies.push({
            repo: repo.name,
            namespace: subPackageJson.name,
            path: subPackage.path,
            dependencies: subPackageJson.dependencies,
            devDependencies: subPackageJson.devDependencies,
          });
        }
      }
    }
  }

  return dependencies;
}

/**
 * Detect folders that might contain package.json in monorepos.
 * @param {Object} repo - The repository object.
 * @param {string} [path='packages'] - The path to start searching from.
 * @returns {Promise<Array>} The sub-packages array.
 */
async function getSubPackages(repo, path = 'packages') {
  const url = `https://api.github.com/repos/${repo.owner.login}/${repo.name}/contents/${path}`;

  try {
    const response = await fetch(url, {headers});
    if (!response.ok) {
      return [];
    }
    const data = await response.json();
    const subPackages = [];
    for (const item of data) {
      if (item.type === 'dir' && !EXCLUDED_DIRS.includes(item.name)) {
        const subDirPackages = await getSubPackages(repo, `${path}/${item.name}`);
        subPackages.push(...subDirPackages);
      } else if (item.name === 'package.json') {
        subPackages.push({path: `${path}/package.json`});
      }
    }

    return subPackages;
  } catch (error) {
    console.error(`Error fetching sub-packages for ${repo.name}: ${error.message}`);
    return [];
  }
}

async function main() {
  try {
    const repos = await getRepos();

    const ownDependencies = await Promise.all(
      repos.ownRepos.map(async (repo) => await getDependencies(repo))
    );

    const forkDependencies = await Promise.all(
      repos.forkRepos.map(async (repo) => await getDependencies(repo, true))
    );

    const packagesData = {
      totalRepos: ownDependencies.length + forkDependencies.length,
      ownRepos: ownDependencies.flat(),
      forkRepos: forkDependencies.flat(),
    };

    try {
      await fs.writeFile('packages.json', JSON.stringify(packagesData, null, 2));
      console.info("Saved 'packages.json' with dependencies of all repositories.");
    } catch (writeError) {
      console.error(`Error writing packages.json: ${writeError.message}`);
    }
  } catch (error) {
    console.error(`Error in main: ${error.message}`);
  }
}

main();