Gets add-on Normandy slug and XPI endpoint from the Normandy API (filtering for studies that use 'isFirstRun' in their recipes) and add-on size from AWS, outputting the join to addonInfo.json

addonInfo.json

{"https://net-mozaws-prod-us-west-2-normandy.s3.amazonaws.com/extensions/tracking-protection-messaging-study-new-users%40shield.mozilla.org-1.0.8-signed.xpi":{"slug":"addon-tracking-protection-messaging-1433473-new-users","addonSize":"142530"},"https://net-mozaws-prod-us-west-2-normandy.s3.amazonaws.com/extensions/pioneer-enrollment-study%40mozilla.org-2.0.4-signed.xpi":{"slug":"Pioneer Enrollment","addonSize":"153385"},"https://net-mozaws-prod-us-west-2-normandy.s3.amazonaws.com/extensions/pug.experience.mrrobotshield.mozilla.org-1.0.4-signed.xpi":{"slug":"looking-glass-2","addonSize":"12956"},"https://net-mozaws-prod-us-west-2-normandy.s3.amazonaws.com/extensions/focused-cfr-shield-studymozilla.com-1.0.1-signed.xpi":{"slug":"focused-cfr-beta-1","addonSize":"104589"},"https://net-mozaws-prod-us-west-2-normandy.s3.amazonaws.com/extensions/unified-urlbar-shield-study-opt-out-new-users-3.0.6-signed.xpi":{"slug":"unfied-search-release-new-users-1387245","addonSize":"31515"}}

getAddonInfo.js

/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/

/*
* This is a Node.js script to get a list of all add-on studies deployed
* through Normandy by their slug name and file size. This requires joining
* information from the Normandy API and response header info from AWS, where
* each add-on's XPI is hosted.
* 
* How to run:
* cd firstrun-addon-info
* npm install
* node getAddonInfo.js
*
* Generated output is 'addonInfo.json'
*/

const fs = require("fs");
const fetch = require("node-fetch");
const NORMANDY_API_ENDPOINT = "https://normandy.cdn.mozilla.net/api/v3/recipe/";
const addonInfo = {};
let done = false;

async function handleResponse(body) {
  for (const recipe of body.results) {
    if (recipe.action.name === "opt-out-study") {
      // this is an add-on study
      const slug = recipe.arguments.name;
      const xpiUrl = recipe.arguments.addonUrl;
      const filterExpression = recipe.filter_expression;
      if (xpiUrl && filterExpression.includes("isFirstRun")) {
        addonInfo[xpiUrl] = { slug };
        try {
          const response = await fetch(xpiUrl, {method: "HEAD"});
          await handleHeaderResponse(response, xpiUrl);
        } catch (error) {
          console.error(error);
        }
      }
    }
  }
  // Get info for each page
  if (body.next) {
    try {
      const response = await fetch(body.next);
      await handleResponse(await response.json());
    } catch (error) {
      console.error(error);
    }
  } else {
    fs.writeFile("addonInfo.json", JSON.stringify(addonInfo), "utf8", (err) => {
      if (err) console.log(err);
      console.log("The file has been saved!");
    });
  }
}

// get add-on size here and add it to addonInfo object
function handleHeaderResponse(response, xpiUrl) {
  // Content length is measured in bytes
  const addonSize = response.headers.get("content-length");
  if (addonInfo.hasOwnProperty(xpiUrl)) {
    addonInfo[xpiUrl].addonSize = addonSize;
    console.log(addonInfo[xpiUrl]);
  }
}

(async function main() {
  try {
    const response = await fetch(NORMANDY_API_ENDPOINT);
    handleResponse(await response.json());
  } catch (error) {
    console.error(error);
  }
}());

package.json

{
  "name": "firstrun-addon-info",
  "version": "1.0.0",
  "description": "",
  "main": "index.js",
  "scripts": {
    "test": "echo \"Error: no test specified\" && exit 1"
  },
  "keywords": [],
  "author": "",
  "license": "ISC",
  "dependencies": {
    "node-fetch": "2.2.0"
  }
}

Specific questions:

• (Revision 2 is easier to see) The done variable is a workaround, because this code is contingent on the async HEAD request returning before writing to file.
  • I was seeing that for the very last HEAD request, I would write to file before it returned, so the last entry in my JSON file was missing the add-on size. I created done to track the state of whether all HEAD requests had been made and returned.
• (Revision 3 is easier to see this diff) Because of this dependency, I also could not directly filter for recipes that targeted new users via isFirstRun until AFTER I made the HEAD request and it returned. This means I am requesting for all of the add-ons, not just the ones that have isFirstRun in their recipes. (32-ish add-ons instead of 8)

• On line 30, usually it is easier to use a for-of loop instead, like this:

  for (const recipe of body.results) {
    ...
  }

• Ultimately, your problem is that request does not return a Promise so you can't await request(...). Well, you can, but it doesn't do anything. So you weren't actually waiting for the last results before writing data out.

• I think that the real problem is that request, although a better API than what is available in Node directly, still is pretty hard to work with. I did some more research, and found node-fetch, which would have made things a lot easier. To demonstrate, I wrote a version of this that uses node-fetch (and some other tricks).

• Content-length is measured in bytes