Curiouscat API Scrapper

curiouscat-api-scrapper.js

const axios = require("axios");
const fs = require("fs").promises;
const path = require("path");
const { open } = require("fs/promises");

function pgbr(prcnt, dn = "█", rm = "░", length = 100, hint = "") {
  const d = Math.floor(prcnt * length);
  const r = length - d;
  process.stdout.write(
    `\r${hint} |${dn.repeat(d)}${rm.repeat(r)}| ${Math.floor(prcnt * 100)}%`
  );
}

async function gtall(name, d, fileIndex) {
  const add = (x, y) =>
    x
      .map(
        (post, n) =>
          `\n\n${n + y}:\nTime: ${new Date(post.timestamp * 1000)
            .toISOString()
            .replace("T", " ")
            .substr(0, 19)}\nMessage: ${post.comment}\nReply: ${post.reply}`
      )
      .join("");

  let dct,
    n = 0,
    last = 0;
  const dbPath = path.join(name, `answers_${fileIndex}.txt`);

  async function writeInitialData() {
    const db = await open(dbPath, "w");
    try {
      await db.write(
        `Name: ${d.username}\nAnswers: ${d.answers}\nFollowers: ${d.followers}\nFollowing: ${d.following}\nTwitter: https://twitter.com/intent/user?user_id=${d.userData.twitterid}`
      );
      if (!d.posts) {
        await db.write("\n\n[No Messages _/(0_0)_]");
      }
    } finally {
      await db.close();
    }
  }

  await writeInitialData();

  while (n < 2086) {
    let db = await open(dbPath, "a");
    try {
      while (n < 2086) {
        try {
          const response = await axios.get(
            `https://curiouscat.live/api/v2/profile?username=${name}&count=100${
              last !== 0 ? `&max_timestamp=${last}` : ""
            }`
          );
          dct = response.data;

          await db.write(add(dct.posts, n + 1));
          n += dct.posts.length;
          pgbr(n / 2086, `Answers fetched (${fileIndex}):`);

          if (dct.posts.length === 0) return n;
          last = dct.posts[dct.posts.length - 1].timestamp - 1;
        } catch (error) {
          if (error.response && error.response.status === 500) {
            console.log(
              "\nReceived 500 status. Starting new data collection..."
            );
            return n;
          } else {
            await db.write("\n\n[Unexpected Error]");
            return n;
          }
        }
      }
    } finally {
      await db.close();
    }
  }
  return n;
}

async function foloDetail(name, identifier) {
  const response = await axios.get(
    `https://curiouscat.live/api/v2/profile/${identifier}?username=${name}`
  );
  const fls = response.data;
  const fsPath = path.join(name, `${identifier}.txt`);
  const fs = await open(fsPath, "w");

  try {
    for (const [fn, flwr] of fls.result.entries()) {
      await fs.write(
        `\n\n${fn + 1}:\n${flwr.username}\nhttps://curiouscat.live/${
          flwr.username
        }`
      );
    }
  } finally {
    await fs.close();
  }
}

async function cc(fileIndex) {
  const name = "periakteon";

  try {
    const response = await axios.get(
      `https://curiouscat.live/api/v2/profile?username=${name}`
    );
    const dct = response.data;

    if ("error" in dct) {
      console.log("\nProfile doesn't exist!");
      return 0;
    }

    await fs.mkdir(name, { recursive: true });

    if (fileIndex === 1) {
      await foloDetail(name, "followers");
      await foloDetail(name, "following");

      const avatarResponse = await axios.get(dct.avatar, {
        responseType: "arraybuffer",
      });
      await fs.writeFile(path.join(name, "avatar.jpg"), avatarResponse.data);
    }

    const answersCount = await gtall(name, dct, fileIndex);
    return answersCount;
  } catch (error) {
    console.error("An error occurred:", error);
    return 0;
  }
}

async function main() {
  await fs.mkdir("DATA", { recursive: true });
  process.chdir("DATA");

  let fileIndex = 1;
  let totalAnswers = 0;

  while (totalAnswers < 2086) {
    try {
      console.clear();
      const answersCount = await cc(fileIndex);
      totalAnswers += answersCount;

      if (answersCount > 0) {
        const name = "periakteon";
        const oldPath = path.join(name, `answers_${fileIndex}.txt`);
        const newPath = path.join(
          name,
          `answers_${fileIndex}_${answersCount}.txt`
        );
        await fs.rename(oldPath, newPath);

        console.log(
          `\nFile saved as: answers_${fileIndex}_${answersCount}.txt`
        );
        fileIndex++;
      }

      if (totalAnswers >= 2086) {
        console.log("\n2086 or more answers have been recorded. Exiting...");
        break;
      }

      console.log("\nContinuing to next iteration...");
      await new Promise((resolve) => setTimeout(resolve, 1500));
    } catch (error) {
      if (error.code === "SIGINT") {
        console.log("Why Ctrl+C? (-_-)");
      } else {
        console.error("An error occurred:", error);
      }
      break;
    }
  }

  console.log(
    "\nScript completed. Data is saved in the DATA\periakteon directory."
  );
}

main();

inspired by:

https://github.com/elmoiv/ccapi/blob/master/curiouscat.py