API Doc Scraper: Endpoint Maps

spotify-apidoc-scraper.js

/**
 * Quickly scrape the API endpoints from the documentation.
 * Open the browser console and drop the code there. Copy the JSON from the console. enjoy!
 *
 * @link https://developer.spotify.com/documentation/web-api/reference/
 *
 * @created      21.03.2021
 * @author       @codemasher
 * @copyright    2021 @codemasher
 * @license      MIT
 */

function isset(accessor){
	try{
		return typeof accessor() !== 'undefined';
	}
	catch(e){
		return false;
	}
}

document.location.hash = '';

let clearfix = document.createElement('div');
clearfix.className = 'clearfix';
let el = document.getElementById('objects-index');
el.parentNode.insertBefore(clearfix, el);

let nodes    = document.getElementsByClassName('post-content')[0].children;
let content  = [];
let endpoint = {};

for(let i = 0; i < nodes.length; i++){

	if(nodes[i].tagName === 'H2'){
		endpoint.desc = nodes[i].childNodes[0].nodeValue;
		endpoint.link = document.location + nodes[i].id;
	}
	else if(nodes[i].className === 'hidden-xs'){
		if(isset(() => nodes[i].children[0].children[0].children[0].children[0].childNodes[0])){
			let val = nodes[i].children[0].children[0].children[0].children[0].childNodes[0].nodeValue.split(' ');

			endpoint.method = val[0];
			endpoint.url    = val[1].trim();
		}
	}
	else if(nodes[i].tagName === 'TABLE'){
		let type   = nodes[i].children[0].children[0].children[0].childNodes[0].nodeValue;
		let tbody  = nodes[i].children[1].children;
		let params = [];

		// loop through the parameters
		for(let j = 0; j < tbody.length; j++){
			// only the parameter fields, others are object description
			if(tbody[j].children.length !== 3){
				continue;
			}

			let param = {};

			param.name = tbody[j].children[0].children[0].innerText;
			param.type = tbody[j].children[1].innerText.toLowerCase();

			if(isset(() => tbody[j].children[0].children[2])){
				param.desc = tbody[j].children[0].children[2].innerText;
			}

			param.required = tbody[j].children[2].innerText.trim() === 'Required';
			params.push(param);
		}

		if(type === 'Header'){
			endpoint.headers = params;
		}
		else if(type === 'Path Parameter'){
			endpoint.path = params;
		}
		else if(type === 'Query Parameter'){
			endpoint.query = params;
		}
		else if(type === 'JSON Body Parameter'){
			endpoint.body = params;
		}
	}
	else if(nodes[i].className === 'clearfix'){
		// reset
		content.push(endpoint);
		endpoint = {};
	}
}

content.shift(); // remove the first (empty) element
console.log(JSON.stringify(content));

twitch-apidoc-scraper.js

/**
 * Quickly scrape the API endpoints from the documentation.
 * Open the browser console and drop the code there. Copy the JSON from the console. enjoy!
 *
 * @link https://dev.twitch.tv/docs/api/reference
 *
 * @created      21.03.2021
 * @author       @codemasher
 * @copyright    2021 @codemasher
 * @license      MIT
 */

document.location.hash = '';

let nodes    = document.getElementsByClassName('main')[0].children;
let content  = [];

for(let i = 0; i < nodes.length; i++){

	if(nodes[i].children.length !== 2){
		continue;
	}

	let body     = nodes[i].children[0].children;
	let endpoint = {};

	for(let j = 0; j < body.length; j++){

		if(body[j].tagName === 'H2'){
			endpoint.desc = body[j].innerText;
			endpoint.link = document.location + body[j].id;
		}
		else if(body[j].tagName === 'H3' && body[j].innerText.match(/URL/i)){
			let val = body[j+1].innerText.split(' ');

			endpoint.method = val.length >= 2 ? val[0] : 'GET';
			endpoint.url    = val.length >= 2 ? val[1].split('?')[0].trim() : val[0];
		}
		else if(body[j].tagName === 'H3' && (
			body[j].innerText.match(/((Required|Optional) )?(Body|Query) (Parameters?|Values?)/i)
			|| body[j].innerText.match(/Response Fields?/i)
			|| body[j].innerText.match(/Return Values?/i)
		)){
			let val      = body[j+1];
			let header   = body[j].innerText.split(' ')
			let required = header[0] === 'Required';
			let type     = (header.length === 3 ? header[1] : header[0]).toLowerCase();

			if(type === 'return'){
				type = 'response';
			}

			if(val.tagName !== 'TABLE'){

				if(!body[j+2] || body[j+2].tagName !== 'TABLE'){
					continue;
				}

				val = body[j+2];
			}

			let tbody = val.children[1] || val.children[0];

			if(tbody.tagName !== 'TBODY'){
				continue;
			}

			if(!endpoint[type]){
				endpoint[type] = [];
			}

			for(let k = 0; k < tbody.children.length; k++){
				let params = {};
				let keys = tbody.children[k].children.length === 3
					? ['name', 'type', 'desc']
					: ['name', 'required', 'type', 'desc'];

				for(let l = 0; l< tbody.children[k].children.length; l++){
					params[keys[l]] = tbody.children[k].children[l].innerText;
				}

				if(type !== 'response'){
					params.required = !params.required ? required : params.required === 'yes';
				}

				if(['Name', 'Field', 'Parameter'].includes(params.name) && params.type === 'Type'){
					continue;
				}

				endpoint[type].push(params);
			}
		}
	}

	content.push(endpoint);
}

console.log(JSON.stringify(content));