domainExtractor.js

const _ = require('lodash');
const fs = require('fs');

const inputFilename = process.argv[2] || 'input.csv';
const s = fs.readFileSync(inputFilename, 'utf8');

const urls = s.split('\n');

const urlPattern = /^https?:\/\/([^:/]+)(:\d+)?.*$/;
const secretsPattern = /\{\{\s*secrets\.([^\s}]+)}}/;
const ipPattern = /\d{1,3}(?:\.\d{1,3}){3}/;

function ovrr(match, domain) {
  return { match: new RegExp(match, 'gi'), domain };
}

const domainOverrides = [
  ovrr('splunk', 'splunk'),
  ovrr('service-?now', 'service-now'),
  ovrr('googleapis', 'google'),
  ovrr('azure-automation', 'azure'),
  ovrr('amazonaws', 'aws'),
  ovrr('awmdm', 'airwatch'),
];
const hostOverrides = [
  // ovrr('hooks.slack.com', 'hooks.slack.com')
];

const e = url => {
  const urlMatch = urlPattern.exec(url);
  // TODO: {{secrets.jamfBaseUrl}}/JSSResource/accounts/userid/{id}
  if (!urlMatch) {
    const secretsMatch = secretsPattern.exec(url);
    if (secretsMatch) {
      return {
        domain: secretsMatch[1],
      };
    }
    return {
      domain: url,
    };
  }
  let host = urlMatch[1];
  let port = urlMatch[2];
  const parts = host.split(/\./);
  let tld = parts[parts.length - 1];
  let domain = parts[parts.length - 2];

  const secretsMatch = secretsPattern.exec(host);
  if (secretsMatch) {
    tld = undefined;
    domain = secretsMatch[1];
  }

  // TODO: IPv4 + IPv6 e.g. '34.73.51.142'
  if (ipPattern.test(host)) {
    tld = undefined;
    domain = host;
  }

  for (let i = 0; i < hostOverrides.length; i++) {
    if (hostOverrides[i].match.test(host)) {
      domain = hostOverrides[i].domain;
    }
  }

  for (let i = 0; i < domainOverrides.length; i++) {
    if (domainOverrides[i].match.test(domain)) {
      domain = domainOverrides[i].domain;
    }
  }

  return {
    tld,
    domain,
    host,
    port,
  };
};
const mapped = urls.map(url => e(url));

console.table(mapped);

const countMap = {};
mapped.forEach(m => (countMap[m.domain] = (countMap[m.domain] || 0) + 1));
const counts = _.reverse(
  _.sortBy(
    Object.keys(countMap).map(domain => ({
      domain,
      count: countMap[domain],
    })),
    'count',
  ),
);

console.table(counts);

package.json

{
  "name": "ba",
  "version": "1.0.0",
  "description": "",
  "main": "main.js",
  "scripts": {
    "test": "echo \"Error: no test specified\" && exit 1"
  },
  "keywords": [],
  "author": "",
  "license": "ISC",
  "dependencies": {
    "lodash": "^4.17.11"
  }
}