ICU filtering

table.mjs

import Table from 'tty-table';
import json2csv from 'json-2-csv';
import fs from 'fs';

const MC = "/home/zbraniecki/projects/mozilla-unified";

function readFirefoxLocales() {
  let source = fs.readFileSync(`${MC}/browser/locales/all-locales`, "utf8");
  return source.split("\n").map(loc => {
    if (loc === "") {
      return null;
    }
    return loc;
  }).filter(loc => loc !== null);
}

function readICUFiles(path) {
  let files = fs.readdirSync(`${MC}/${path}`);
  return files.map(input => {
    if (input === "") {
      return null;
    }
    let loc = input.replace(".txt", "").replace(/_/g, "-");
    return loc;
  }).filter(loc => loc !== null);
}

function readChromiumWhitelist(path) {
  let source = fs.readFileSync(path, "utf8").replace(/^\/\/.*\n/, "");
  let data = JSON.parse(source);
  return data.localeFilter.whitelist;
}

let firefoxLocales = readFirefoxLocales();
let icuLocales = readICUFiles("intl/icu/source/data/locales");
let icuLanguages = readICUFiles("intl/icu/source/data/lang");
let chromiumCommon = readChromiumWhitelist("./data/chromium-common.json");


//////

const j2c = json2csv.json2csvAsync;

let localesData = new Map();

let defaultData = {
  icuLocale: false,
  icuLanguage: false,
  firefoxLocale: false,
  chromiumCommon: false,
};

function appendData(localesData, defaultData, inputList, prop) {
  for (let locale of inputList) {
    if (localesData.has(locale)) {
      let data = localesData.get(locale);
      data[prop] = true;
    } else {
      localesData.set(locale, {
        ...defaultData,
        [prop]: true,
      });
    }
  }
}

appendData(localesData, defaultData, icuLocales, "icuLocale");
appendData(localesData, defaultData, icuLanguages, "icuLanguage");
appendData(localesData, defaultData, firefoxLocales, "firefoxLocale");
appendData(localesData, defaultData, chromiumCommon, "chromiumCommon");

const headers = [
  "Locale",
  "ICU Locale",
  "ICU Language",
  "Firefox",
  "Chromium Common",
];
let limit = null;

let allLocales = Array.from(localesData.keys());

/* FILTER */

allLocales = allLocales.filter((locale) => {
  let data = localesData.get(locale);
  return (data.firefoxLocale || data.chromiumCommon) && data.icuLocale;
});

/* FILTER END */

/* Add languages for locales */

for (let s of allLocales) {
  try {
    let loc = new Intl.Locale(s);
    if ((loc.script || loc.region) && !allLocales.includes(loc.language)) {
      allLocales.push(loc.language);
    }
  } catch {
    // If the locale is not a correct BCP47 locale, we won't bother adding
    // a language for it.
    continue;
  }
}

/* Add languages for locales END */

allLocales.sort();

function printCLI() {
  let defaultOptions = {
    align: "left",
  };

  const header = headers.map((h) => {
    return { value: h, ...defaultOptions };
  });

  let rows = [];

  for (let locale of allLocales) {
    let data = localesData.get(locale);
    rows.push([locale, ...Object.values(data)]);

    if (limit !== null) {
      if (limit > 0) {
        limit--;
      } else {
        break;
      }
    }
  }

  const options = {
    borderStyle: "none",
    compact: true,
  };

  const table = Table(header, rows, options);

  console.log(table.render());
}

function printCSV() {
  let output = [];

  for (let locale of allLocales) {
    let data = localesData.get(locale);
    let entry = {};
    for (let i in headers) {
      if (i == 0) {
        entry[headers[i]] = locale;
      } else {
        entry[headers[i]] = Object.values(data)[i-1];
      }
    }
    output.push(entry);

    if (limit !== null) {
      if (limit > 0) {
        limit--;
      } else {
        break;
      }
    }
  }

  let result = j2c(output)
    .then((csv) => { console.log(csv) })
    .catch((err) => { });
}

// console.log(JSON.stringify(allLocales, null, 2));
printCSV();