Security.txt

analyze.ts

const moment = require("moment");
const sqlite3 = require("sqlite3").verbose();

type SecurityTxt = {
  // attributes
  acknowledgments: Array<string>; // optional
  canonical: Array<string>; // optional
  contact: Array<string>; // required
  encryption: Array<string>; // optional
  expires: string; // required, max once
  hiring: Array<string>; // optional
  policy: Array<string>; // optional
  "preferred-languages": string; // optional, max once
  // metadata
  isSigned: boolean;
  signature: string;
  comments: Array<string>;
  other: Array<{ key: string; value: string }>;
  errors: Array<string>;
};

type NamedResults = {
  domain: string;
  validation: SecurityTxt;
};

//

const checkURL = (value) => {
  if (value.length === 0 || !value.trim()) {
    return;
  }

  try {
    const url = new URL(value);

    if (url.protocol === "http:") {
      return "HTTP instead of HTTPS used";
    }

    return;
  } catch (err) {
    // manually review each error to ensure no false positives
    // console.log("err", err);

    return "Invalid URL";
  }
};

const validate = (content: string, signature?: string): SecurityTxt => {
  const result: SecurityTxt = {
    acknowledgments: [],
    canonical: [],
    contact: [],
    encryption: [],
    expires: "",
    hiring: [],
    policy: [],
    "preferred-languages": "",
    isSigned: signature ? true : false,
    signature: signature ? signature : undefined,
    comments: [],
    other: [],
    errors: [],
  };

  let hasContactField = false;
  let hasExpiresField = false;
  let hasLanguageField = false;

  // split string into multiple lines
  let lines = content.match(/[^\r\n]+/g) || [];

  lines.forEach((line) => {
    // console.log("line: ", line);

    // push comments to its own array
    if (line.startsWith("#")) {
      result["comments"].push(line);
      return;
    }

    // remove empty lines
    if (line.length === 0 || !line.trim()) {
      return;
    }

    const match = /^(?<key>.*): (?<value>.*)$/.exec(line);
    if (!match || !match.groups) {
      // should not happen
      return;
    }

    const key = match.groups.key.toLowerCase();
    const value = match.groups.value;

    let urlError = "";

    switch (key) {
      default:
        result["other"].push({ key, value });
        break;

      case "acknowledgments":
        urlError = checkURL(value);
        if (urlError) {
          result["errors"].push(`${urlError} in ACKNOWLEDGMENTS field`);
        }

        result[key].push(value);
        break;

      case "canonical":
        urlError = checkURL(value);
        if (urlError) {
          result["errors"].push(`${urlError} in CANONICAL field`);
        }

        result[key].push(value);
        break;

      case "contact":
        hasContactField = true;

        urlError = checkURL(value);
        if (urlError) {
          result["errors"].push(`${urlError} in CONTACT field`);
        }

        result[key].push(value);
        break;

      case "encryption":
        urlError = checkURL(value);
        if (urlError) {
          result["errors"].push(`${urlError} in ENCRYPTION field`);
        }

        result[key].push(value);
        break;

      case "expires":
        // only one is allowed
        if (hasExpiresField) {
          result["errors"].push("More than one EXPIRES field");
          return;
        }

        if (!moment(value, moment.ISO_8601).isValid()) {
          result["errors"].push("Invalid date format in EXPIRES field");
        }

        hasExpiresField = true;

        result[key] = value;
        break;

      case "hiring":
        urlError = checkURL(value);
        if (urlError) {
          result["errors"].push(`${urlError} in HIRING field`);
        }

        result[key].push(value);
        break;

      case "policy":
        urlError = checkURL(value);
        if (urlError) {
          result["errors"].push(`${urlError} in POLICY field`);
        }

        result[key].push(value);
        break;

      case "preferred-languages":
        // only one is allowed
        if (hasLanguageField) {
          result["errors"].push("More than one PREFERRED-LANGUAGES field");
          return;
        }

        hasLanguageField = true;

        result[key] = value;
        break;
    }
  });

  // check if at least one contact field is set
  if (!hasContactField) {
    result["errors"].push("No CONTACT field found");
  }

  // check if the expires field was set once, having more than one
  // is being detected by the switch block already
  if (!hasExpiresField) {
    result["errors"].push("No EXPIRES field found");
  } else {
    const parsedExpiresDate = Date.parse(result.expires);
    const now = new Date();
    const oneYearFuture = new Date(now);
    oneYearFuture.setFullYear(now.getFullYear() + 1);

    if (isNaN(parsedExpiresDate)) {
      // check if the date is valid
      result["errors"].push("Invalid date in EXPIRES field");
    } else if (parsedExpiresDate < +now) {
      // check if the date is in the future
      result["errors"].push("Expired date in EXPIRES field");
    } else if (parsedExpiresDate > +oneYearFuture) {
      // check if the date is not more than one year in the future
      // technically not an error, but rather a recommendation
      result["errors"].push(
        "Date is more than one year in the future in EXPIRES field"
      );
    }
  }

  return result;
};

const validFields = [
  "acknowledgments",
  "canonical",
  "contact",
  "encryption",
  "expires",
  "policy",
  "preferred-languages",
  "acknowledgements",
  "acknowledgement",
  "signature",
];

const aggregateResults = (results: NamedResults[]) => {
  const aggregatedErrors = [];
  let pgpAmount = 0;
  let noErrorsAmount = 0;
  let wrongAcknowledgmentsFieldAmount = 0;
  let signatureFieldAmount = 0;
  let commentsAmount = 0;

  results.forEach((result) => {
    // console.log(result);

    result.validation.errors.forEach((error) => {
      if (!aggregatedErrors[error]) {
        aggregatedErrors[error] = 0;
      }

      aggregatedErrors[error]++;
    });

    if (result.validation.isSigned) {
      pgpAmount++;
    }

    if (result.validation.errors.length === 0) {
      noErrorsAmount++;
    }

    result.validation.other.forEach((other) => {
      // old version, is now plural with no second e -> "Acknowledgments"
      if (other.key === "acknowledgement" || other.key === "acknowledgements") {
        wrongAcknowledgmentsFieldAmount++;
      }

      // old version, was removed afaik
      if (other.key === "signature") {
        signatureFieldAmount++;
      }

      // log non-specified (old, new, misspelled) fields
      /*
      if (validFields.includes(other.key)) {
        return;
      }

      console.log(`${other.key}: ${other.value}`);
      */
    });

    commentsAmount += result.validation.comments.length;

    /*
    if (
      result.validation["preferred-languages"].length === 0 ||
      !result.validation["preferred-languages"].trim()
    ) {
      return;
    }

    console.log(result.validation["preferred-languages"]);
    */
  });

  console.log("length", results.length);
  console.log("aggregatedErrors", aggregatedErrors);
  console.log("pgpAmount", pgpAmount);
  console.log("noErrorsAmount", noErrorsAmount);
  console.log(
    "wrongAcknowledgmentsFieldAmount",
    wrongAcknowledgmentsFieldAmount
  );
  console.log("signatureFieldAmount", signatureFieldAmount);
  console.log("commentsAmount", commentsAmount);
};

//
//
//

// regex to detect if the message is pgp signed or not
const regex = /-----BEGIN PGP SIGNED MESSAGE-----((.|\n)*)-----BEGIN PGP SIGNATURE-----((.|\n)*)-----END PGP SIGNATURE-----/gm;

// setup db
const db = new sqlite3.Database("security-txt.db");
db.serialize();

const results: NamedResults[] = [];

// get all entries with a valid security.txt file
db.each(
  "SELECT domain, contents FROM results WHERE hasSecurityFile IS 1;",
  (err, row) => {
    let result;
    const matches = regex.exec(row.contents);

    if (matches) {
      // is pgp signed
      const content = matches[1];
      const signature = matches[3];

      result = validate(content, signature);
    } else {
      // not signed
      result = validate(row.contents);
    }

    results.push({
      domain: row.domain,
      validation: result,
    });
  },
  // final callback
  () => {
    // results.forEach((result) => console.log(result));
    aggregateResults(results);
  }
);

// close db
db.close();

crawl.js

const axios = require("axios");
const sqlite3 = require("sqlite3").verbose();
const fs = require("fs");

const db = new sqlite3.Database("security-txt.db");

const request = async (url) => {
  let data;

  try {
    // TODO disable redirects
    // const response = await axios.get(url, { maxRedirects: 0 });
    const response = await axios.get(url);

    data = {
      status: response.status,
      contentType: response.headers["content-type"],
      data: response.data,
    };

    console.log(`[${response.status}] ${url}`);
  } catch (error) {
    data = {
      status:
        error.response && error.response.status ? error.response.status : null,
      contentType: null,
      data: null,
    };

    console.log(
      `[${
        error.response && error.response.status ? error.response.status : "ERR"
      }] ${url}`
    );
  }

  return data;
};

const save = async (data) => {
  await db.run(
    "INSERT INTO `results` (timestamp, domain, path, statusCode, headerContentType, contents) VALUES (?, ?, ?, ?, ?, ?)",
    data
  );
};

const crawl = async (domain) => {
  const timestamp = new Date().toISOString();

  // /.well-known/security.txt
  const wellKnownSecurity = `https://${domain}/.well-known/security.txt`;
  const wellKnownData = await request(wellKnownSecurity);

  await save([
    timestamp,
    domain,
    `/.well-known/security.txt`,
    wellKnownData.status,
    wellKnownData.contentType,
    wellKnownData.data,
  ]);

  /*
  // /security.txt
  const topLevelSecurity = `https://${domain}/security.txt`;
  const topLevelData = await request(topLevelSecurity);

  await save([
    timestamp,
    domain,
    `/security.txt`,
    topLevelData.status,
    topLevelData.contentType,
    topLevelData.data,
  ]);
  */
};

(async () => {
  // setup db
  db.serialize();
  db.run(
    "CREATE TABLE IF NOT EXISTS `results` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `timestamp` datetime, `domain` varchar(255), `path` varchar(255), `statusCode` varchar(255), `headerContentType` varchar(255), `contents` varchar(255));"
  );

  // load domain
  const domains = fs.readFileSync("domains.txt").toString().split("\n");
  // const domains = ["google.com"];

  // query all pages
  await Promise.all(domains.map(async (domain) => crawl(domain)));

  // close db
  db.close();
})();