GatsbyJS compatible IndexNow script for NodeJS

README.md

GatsbyJs compatible IndexNow script for NodeJs

Assumes that your deploy script will deploy a fresh sitemap.xml file, so it can get the remote file, parse it and post the results to Bing.

• Go to https://www.bing.com/indexnow and generate an API Key.
• Save the key file to https://www.yourdomain.com/*************.txt
• Add index-now.mjs at scripts/index-now.mjs & npm install -dev cross-fetch xml-js
• Update host, key and sitemapUrl in scripts/index-now.mjs
• Add the seo:index-now block to your package.json
• Add && npm run seo:index-now at the end of your deploy block

On deploy, the script will deploy everything to Bing.

index-now.mjs

// `scripts/index-now.mjs`
/* eslint-disable no-console */
import fetch from 'cross-fetch';
import convert from 'xml-js';

const host = 'www.yourdomain.com';
const key = '******';
const sitemapUrl = `https://${host}/sitemap/sitemap-0.xml`;
const keyLocation = `https://${host}/${key}.txt`;

const readRemoteSitemapXml = async (url) => {
  console.info('Reading remote sitemap', url);
  const res = await fetch(url);
  if (res.status >= 400) throw new Error('Bad response from server');
  return res.text();
};

/**
 * Parse a sitemap.xml file & get an array of URLs
 * @param {*} xml
 * @returns
 */
const getUrlListFromXmlText = (xml) => {
  const result = convert.xml2json(xml, { compact: true, spaces: 4 });
  const siteMapJson = JSON.parse(result);

  // get an array of just URLs
  // eslint-disable-next-line no-underscore-dangle
  return siteMapJson.urlset.url.map((item) => item.loc._text);
};

/**
 * POST https://www.bing.com/indexnow HTTP/1.1
 * Content-Type: application/json; charset=utf-8
 * {
 *   "host": "www.example.org",
 *   "key": "e6677129dd464fc4a5439591f2ab2eb7",
 *   "keyLocation": "https://www.example.org/e6677129dd464fc4a5439591f2ab2eb7.txt",
 *   "urlList": [
 *       "https://www.example.org/url1",
 *       "https://www.example.org/folder/url2",
 *       "https://www.example.org/url3"
 *       ]
 * }
 */
const submitToBing = (payload) => {
  const url = 'https://www.bing.com/indexnow';
  fetch(url, {
    method: 'post',
    headers: {
      Accept: 'application/json',
      'Content-Type': 'application/json',
    },
    body: JSON.stringify(payload),
  }).then((response) => {
    console.log(
      `Submitted to IndexNow: ${url} • HTTP Response Code: ${response.status}`
    );
  });
};

(async () => {
  try {
    const xmlText = await readRemoteSitemapXml(sitemapUrl);

    // get an array of just URLs from the sitemap text
    const urlList = getUrlListFromXmlText(xmlText);

    console.log('Number of URLs: ', urlList.length);

    // the payload in IndexNow format
    const payload = {
      host,
      key,
      keyLocation,
      urlList,
    };

    submitToBing(payload);
  } catch (err) {
    console.error(err);
  }
})();

package.json

{
  ...
  "scripts": {
    ...
    "deploy": "... && npm run seo:index-now",
    ...
    "seo:index-now": "node scripts/index-now.mjs"
  },
  ...
}