compass-logs.js

#!/usr/bin/env node
const { promises: fs } = require("fs");
const path = require("path");
const { program } = require("commander");
const chronoNode = require("chrono-node");
const { pick } = require("lodash");
const fecha = require("fecha");
const { gunzip: gunzipCb, constants: zlibConstants } = require("zlib");
const { promisify } = require("util");
const { glob: globCb } = require("glob");
const debug = require("debug")("compass-logs");

const glob = promisify(globCb);
const gunzip = promisify(gunzipCb);

async function readLogFile(file) {
  const buffer = await fs.readFile(file);
  const unzipped = await gunzip(buffer, {
    finishFlush: zlibConstants.Z_SYNC_FLUSH,
  });

  const content = unzipped
    .toString()
    .split("\n")
    .filter(Boolean)
    .map((l) => {
      try {
        return JSON.parse(l);
      } catch (e) {
        debug("Failed to parse line as json", { line: l });
      }
    })
    .filter(Boolean);

  const infoLine = content.find((l) => l && l.id === 1001000001);
  const version = infoLine && infoLine.attr && infoLine.attr.version;
  const channelMatch =
    version && /^\d+\.\d+\.\d+(-([a-z]+)\.\d+)?$/.exec(version);
  const channel = channelMatch && channelMatch[2];

  return {
    file: path.basename(file),
    version: version,
    channel: channel,
    content: content,
  };
}

async function run(options) {
  const logFiles = await glob(options.filePattern);
  const contents = await Promise.all(logFiles.map(readLogFile));
  const records = [];

  for (const fileData of contents) {
    if (!matchFilter(options.channel, fileData.channel)) {
      continue;
    }

    for (const line of fileData.content) {
      const { t: bsonTimestamp, ...fields } = line;
      const record = {
        t: new Date(bsonTimestamp.$date),
        v: fileData.version,
        ch: fileData.channel,
        f: fileData.file,
        ...fields,
      };

      if (matchFilters(record, options)) {
        records.push(record);
      }
    }
  }

  const sortedRecords = records.sort((a, b) => {
    return a.t > b.t ? 1 : -1;
  });

  for (const record of sortedRecords) {
    const formattedRecord = options.pick ? pick(record, options.pick) : record;

    if (formattedRecord.t && options.timeFormat) {
      formattedRecord.t = fecha.format(formattedRecord.t, options.timeFormat);
    }

    console.log(JSON.stringify(formattedRecord));
  }
}

function matchFilters(record, filters) {
  const since = filters.since
    ? new Date(filters.since)
    : new Date("0000-01-01T00:00:00.000Z");
  const until = filters.until ? new Date(filters.until) : new Date();

  return (
    record.t >= since &&
    record.t <= until &&
    matchFilter(filters.component, record.c) &&
    matchFilter(filters.context, record.ctx) &&
    matchFilter(filters.severity, record.s)
  );
}

function matchFilter(filter, val) {
  if (!filter) {
    return true;
  }

  if (Array.isArray(filter)) {
    return filter.length === 0 || filter.includes(val);
  }

  return val === filter;
}

function fatal(err) {
  console.error(err);
  process.exit(1);
}

program
  .allowExcessArguments(false)
  .addArgument("<pattern>")
  .option("-c, --channel <string...>")
  .option("--component <string...>")
  .option("--context <string...>")
  .option("--since <date>")
  .option("--until <date>")
  .option("--fields, --pick <attrs...>")
  .option("--time-format <string>", "Format timestamps")
  .option("-s, --severity <string>");

program.parse();

const defaultPattern = path.join(
  process.env.HOME,
  ".mongodb",
  "compass",
  "*_log.gz"
);

const options = {
  filePattern: program.args[0] || defaultPattern,
  ...program.opts(),
};

if (options.since) {
  const parsedSince = chronoNode.parseDate(options.since);
  if (!parsedSince) {
    fatal('Cannot parse --since, did you forget "ago"?');
  }
  options.since = parsedSince;
}

if (options.until) {
  const parsedUntil = chronoNode.parseDate(options.until);
  if (!parsedUntil) {
    fatal('Cannot parse --until, did you forget "ago"?');
  }
  options.until = parsedUntil;
}

run(options);

package.json

{
  "dependencies": {
    "chrono-node": "^2.3.8",
    "commander": "^9.2.0",
    "debug": "^4.3.4",
    "fecha": "^4.2.3",
    "glob": "^8.0.3",
    "lodash": "^4.17.21",
    "zlib": "^1.0.5"
  }
}