Profile CSV/JSONL value frequencies while streaming in Node.js: count top values per column, with nested JSONL paths and no dependencies.

node-profile-value-frequencies.js

#!/usr/bin/env node
/**
 * Profile CSV/JSONL value frequencies while streaming.
 *
 * Usage:
 *   node node-profile-value-frequencies.js input.csv --format=csv --columns=country,status
 *   cat input.jsonl | node node-profile-value-frequencies.js --format=jsonl --columns=user.country,status
 *
 * Options:
 *   --top=N          Number of values to show per column (default: 10)
 *   --delimiter=,    CSV delimiter (default: ,)
 *
 * Notes:
 * - CSV mode uses header row.
 * - JSONL mode supports nested paths like user.country.
 * - Memory grows with distinct values per selected column.
 * - No dependencies.
 */

const fs = require("fs");
const readline = require("readline");

function usage() {
  console.error(`Usage:
  node node-profile-value-frequencies.js input.csv --format=csv --columns=country,status
  cat input.jsonl | node node-profile-value-frequencies.js --format=jsonl --columns=user.country,status

Options:
  --format=csv|jsonl
  --columns=col1,col2
  --top=N
  --delimiter=,
`);
}

function parseArgs(argv) {
  const args = {
    file: null,
    format: "jsonl",
    columns: null,
    top: 10,
    delimiter: ",",
  };

  for (const arg of argv.slice(2)) {
    if (!args.file && !arg.startsWith("--")) args.file = arg;
    else if (arg.startsWith("--format=")) args.format = arg.slice("--format=".length);
    else if (arg.startsWith("--columns=")) args.columns = arg.slice("--columns=".length).split(",").map(s => s.trim()).filter(Boolean);
    else if (arg.startsWith("--top=")) args.top = Number(arg.slice("--top=".length));
    else if (arg.startsWith("--delimiter=")) args.delimiter = arg.slice("--delimiter=".length);
    else if (arg === "-h" || arg === "--help") args.help = true;
    else throw new Error(`Unknown argument: ${arg}`);
  }

  return args;
}

function parseCSVLine(line, delimiter = ",") {
  const out = [];
  let cur = "";
  let inQuotes = false;

  for (let i = 0; i < line.length; i++) {
    const ch = line[i];

    if (inQuotes) {
      if (ch === '"') {
        if (line[i + 1] === '"') {
          cur += '"';
          i++;
        } else {
          inQuotes = false;
        }
      } else {
        cur += ch;
      }
      continue;
    }

    if (ch === '"') inQuotes = true;
    else if (ch === delimiter) {
      out.push(cur);
      cur = "";
    } else {
      cur += ch;
    }
  }

  out.push(cur);
  return out;
}

function getByPath(obj, keyPath) {
  let cur = obj;

  for (const part of keyPath.split(".")) {
    if (cur == null || typeof cur !== "object" || !(part in cur)) {
      return undefined;
    }
    cur = cur[part];
  }

  return cur;
}

function normalizeValue(value) {
  if (value === undefined) return "__MISSING__";
  if (value === null) return "__NULL__";
  if (value === "") return "__EMPTY__";
  if (typeof value === "object") return JSON.stringify(value);
  return String(value);
}

function createCounters(columns) {
  return Object.fromEntries(columns.map(col => [col, new Map()]));
}

function increment(map, value) {
  map.set(value, (map.get(value) || 0) + 1);
}

function printReport(counters, totalRows, topN) {
  const report = {};

  for (const [column, counts] of Object.entries(counters)) {
    const values = [...counts.entries()]
      .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
      .slice(0, topN)
      .map(([value, count]) => ({
        value,
        count,
        pct: totalRows === 0 ? 0 : Number(((count / totalRows) * 100).toFixed(2)),
      }));

    report[column] = {
      total_rows: totalRows,
      distinct_values: counts.size,
      top_values: values,
    };
  }

  console.log(JSON.stringify(report, null, 2));
}

async function runCSV(input, columns, delimiter, topN) {
  if (!columns || columns.length === 0) {
    throw new Error("CSV mode requires --columns=col1,col2");
  }

  const rl = readline.createInterface({ input, crlfDelay: Infinity });

  let headers = null;
  let counters = createCounters(columns);
  let totalRows = 0;
  let lineNum = 0;

  for await (const line of rl) {
    lineNum++;

    if (!line.trim()) continue;

    const fields = parseCSVLine(line, delimiter);

    if (!headers) {
      headers = fields;
      continue;
    }

    totalRows++;

    for (const col of columns) {
      const idx = headers.indexOf(col);
      const value = idx === -1 ? undefined : fields[idx];
      increment(counters[col], normalizeValue(value));
    }
  }

  printReport(counters, totalRows, topN);
}

async function runJSONL(input, columns, topN) {
  if (!columns || columns.length === 0) {
    throw new Error("JSONL mode requires --columns=col1,col2");
  }

  const rl = readline.createInterface({ input, crlfDelay: Infinity });

  const counters = createCounters(columns);
  let totalRows = 0;
  let lineNum = 0;

  for await (const line of rl) {
    lineNum++;

    if (!line.trim()) continue;

    let record;
    try {
      record = JSON.parse(line);
    } catch (err) {
      throw new Error(`Invalid JSON on line ${lineNum}: ${err.message}`);
    }

    totalRows++;

    for (const col of columns) {
      increment(counters[col], normalizeValue(getByPath(record, col)));
    }
  }

  printReport(counters, totalRows, topN);
}

async function main() {
  const args = parseArgs(process.argv);

  if (args.help || !["csv", "jsonl"].includes(args.format)) {
    usage();
    process.exit(args.help ? 0 : 2);
  }

  if (!Number.isInteger(args.top) || args.top <= 0) {
    throw new Error("--top must be a positive integer");
  }

  const input = args.file
    ? fs.createReadStream(args.file, { encoding: "utf8" })
    : process.stdin;

  if (args.format === "csv") {
    await runCSV(input, args.columns, args.delimiter, args.top);
  } else {
    await runJSONL(input, args.columns, args.top);
  }
}

main().catch((err) => {
  console.error("ERROR:", err && err.stack ? err.stack : err);
  process.exit(1);
});