script to put ffprobe info to trace.moe api database

analyze.js

import "dotenv/config.js";

import child_process from "child_process";
import cluster from "cluster";
import fs from "fs-extra";
import Knex from "knex";

const { ANIME_PATH, DB_NAME, DB_USER, DB_PASS, DB_HOST } = process.env;

const knex = Knex({
  client: "mysql",
  connection: {
    host: DB_HOST,
    user: DB_USER,
    password: DB_PASS,
    database: DB_NAME,
  },
});

if (cluster.isPrimary) {
  console.log("Cleaning up deleted files");
  for (const media of await knex("mediainfo")) {
    if (!fs.existsSync(media.path)) {
      await knex("mediainfo").where("path", media.path).del();
      console.log(`Deleted: ${media.path}`);
    }
  }
  const mediainfo = await knex("mediainfo");
  let opts = "";
  if (mediainfo.length) {
    const [last] = await knex("mediainfo").orderBy("created", "desc").limit(1).select("created");
    console.log(`Last Update: ${last.created.toISOString()}`);
    opts = `-newerct "${last.created.toISOString()}"`;
  }

  console.log("Reading file list...");
  const fileList = child_process
    .execSync(`find -L ${ANIME_PATH} -type f -name "*.mp4" ${opts}`, {
      maxBuffer: 1024 * 1024 * 100,
    })
    .toString()
    .split("\n")
    .filter((each) => each);

  let finished = 0;
  const concurrency = 8;
  const displayInterval = 500; // ms
  let speedRecord = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
  let time = 0;
  let mark = fileList.length;
  let total = fileList.length;

  for (let i = 0; i < concurrency; i++) {
    const worker = cluster.fork();
    worker.on("message", async (message) => {
      if (message) {
        const { filePath, result } = JSON.parse(message);
        const { mtime, ctime } = fs.lstatSync(filePath);
        await knex.raw(
          knex("mediainfo")
            .insert({
              path: filePath,
              json: result,
              created: ctime,
              updated: mtime,
            })
            .toString()
            .replace(/^insert/i, "insert ignore")
        );
        if (Date.now() - time > displayInterval) {
          const speed = (mark - fileList.length) / (displayInterval / 1000);
          speedRecord.push(speed);
          speedRecord = speedRecord.slice(1);
          const averageSpeed = speedRecord.reduce((a, b) => a + b, 0) / speedRecord.length;
          const ETA = fileList.length / averageSpeed;
          const completed = total - fileList.length;
          const percentage = ((completed / total) * 100).toFixed(2);
          console.log(
            `${completed}/${total}`,
            `(${percentage}%)`,
            `[${averageSpeed.toFixed(1)} tasks/s, ETA ${ETA.toFixed(0)}s]`
          );
          time = Date.now();
          mark = fileList.length;
        }
      }
      if (fileList.length === 0) {
        worker.kill();
      } else {
        worker.send(fileList.pop());
      }
    });

    worker.on("exit", (code) => {
      finished += 1;
      if (finished === concurrency) {
        console.log("all done");
        process.exit();
      }
    });
  }
} else {
  process.send("");
  process.on("message", (message) => {
    try {
      const result = child_process
        .execSync(
          [
            "ffprobe",
            "-show_format",
            "-show_streams",
            "-v quiet",
            "-print_format json=compact=1",
            `'${message.replace(/'/g, "'\\''")}'`,
          ].join(" ")
        )
        .toString();
      process.send(JSON.stringify({ filePath: message, result: result }));
    } catch (e) {
      console.log(e, message);
    }
  });
}