Parse Node Download Data

go.js

'use strict';
const assert = require('node:assert');
const fs = require('node:fs/promises');
const DATA_HOME_URL = 'https://storage.googleapis.com/access-logs-summaries-nodejs/index.html';
const DATA_FILE_PATH = 'data.json';
const DATA_CSV_PATH = 'data.csv';
const DATA_CSV_SEVEN_DAY_PATH = 'data-seven-day-avg.csv';
const LINES = ['14', '16', '18', '19', '20'];

async function main() {
  const data = await loadJsonData();
  const series = getDownloadsSeries(data);

  await writeCsvFile(series);
  await writeCsvSevenDayFile(series);
}

async function writeCsvSevenDayFile(series) {
  let output = '';

  for (const [version, counts] of Object.entries(series)) {
    const sevenDayAverages = [];
    let sum = 0;

    for (let i = 0; i < counts.length; ++i) {
      sum += counts[i];

      if ((i + 1) % 7 === 0) {
        const avg = sum === 0 ? 0 : sum / 7;

        sevenDayAverages.push(Math.round(avg));
        sum = 0;
      }
    }

    output += `${version},${sevenDayAverages.join(',')}\n`;
  }

  await fs.writeFile(DATA_CSV_SEVEN_DAY_PATH, output);
}

async function writeCsvFile(series) {
  let output = '';

  for (const [version, counts] of Object.entries(series)) {
    output += `${version},${counts.join(',')}\n`;
  }

  await fs.writeFile(DATA_CSV_PATH, output);
}

function getDownloadsSeries(data) {
  const values = { total: [] };

  for (let i = 0; i < LINES.length; i++) {
    values[LINES[i]] = [];
  }

  for (let i = 0; i < data.length; i++) {
    const d = data[i];
    const versions = d.version;
    const counts = new Map();

    values.total.push(d.total);

    for (let i = 0; i < LINES.length; i++) {
      counts.set(LINES[i], 0);
    }

    for (const [version, count] of Object.entries(versions)) {
      const major = version.match(/^v(\d+)\./)?.[1];
      const curCount = counts.get(major);

      if (curCount !== undefined) {
        counts.set(major, curCount + count);
      }
    }

    for (const [version, count] of counts) {
      values[version].push(count);
    }
  }

  return values;
}

async function loadJsonData() {
  try {
    const data = await fs.readFile(DATA_FILE_PATH, 'utf8');

    return JSON.parse(data);
  } catch (err) {
    assert.strictEqual(err.code, 'ENOENT');

    const jsonUrls = await getJsonUrls();
    const data = await getJsonData(jsonUrls);

    await fs.writeFile(DATA_FILE_PATH, JSON.stringify(data, null, 2));
    return data;
  }
}

async function getJsonData(urls) {
  const data = [];

  for (let i = 0; i < urls.length; i++) {
    const res = await fetch(urls[i]);
    const json = await res.json();

    data.push(json);
  }

  return data;
}

async function getJsonUrls() {
  const res = await fetch(DATA_HOME_URL);
  const html = await res.text();
  const jsonUrls = Array.from(html.matchAll(/\<a href=\"(?<url>.+)\">/g));

  return jsonUrls.map((match) => {
    const url = match.groups?.url;
    assert(url);
    return url;
  });
}

main();