Skip to content

Instantly share code, notes, and snippets.

@codemasher
Last active March 23, 2021 09:32
Show Gist options
  • Save codemasher/f8aa34139b584798bbe1285abe2ec2ab to your computer and use it in GitHub Desktop.
Save codemasher/f8aa34139b584798bbe1285abe2ec2ab to your computer and use it in GitHub Desktop.
API Doc Scraper: Endpoint Maps
/**
* Quickly scrape the API endpoints from the documentation.
* Open the browser console and drop the code there. Copy the JSON from the console. enjoy!
*
* @link https://developer.spotify.com/documentation/web-api/reference/
*
* @created 21.03.2021
* @author @codemasher
* @copyright 2021 @codemasher
* @license MIT
*/
function isset(accessor){
try{
return typeof accessor() !== 'undefined';
}
catch(e){
return false;
}
}
document.location.hash = '';
let clearfix = document.createElement('div');
clearfix.className = 'clearfix';
let el = document.getElementById('objects-index');
el.parentNode.insertBefore(clearfix, el);
let nodes = document.getElementsByClassName('post-content')[0].children;
let content = [];
let endpoint = {};
for(let i = 0; i < nodes.length; i++){
if(nodes[i].tagName === 'H2'){
endpoint.desc = nodes[i].childNodes[0].nodeValue;
endpoint.link = document.location + nodes[i].id;
}
else if(nodes[i].className === 'hidden-xs'){
if(isset(() => nodes[i].children[0].children[0].children[0].children[0].childNodes[0])){
let val = nodes[i].children[0].children[0].children[0].children[0].childNodes[0].nodeValue.split(' ');
endpoint.method = val[0];
endpoint.url = val[1].trim();
}
}
else if(nodes[i].tagName === 'TABLE'){
let type = nodes[i].children[0].children[0].children[0].childNodes[0].nodeValue;
let tbody = nodes[i].children[1].children;
let params = [];
// loop through the parameters
for(let j = 0; j < tbody.length; j++){
// only the parameter fields, others are object description
if(tbody[j].children.length !== 3){
continue;
}
let param = {};
param.name = tbody[j].children[0].children[0].innerText;
param.type = tbody[j].children[1].innerText.toLowerCase();
if(isset(() => tbody[j].children[0].children[2])){
param.desc = tbody[j].children[0].children[2].innerText;
}
param.required = tbody[j].children[2].innerText.trim() === 'Required';
params.push(param);
}
if(type === 'Header'){
endpoint.headers = params;
}
else if(type === 'Path Parameter'){
endpoint.path = params;
}
else if(type === 'Query Parameter'){
endpoint.query = params;
}
else if(type === 'JSON Body Parameter'){
endpoint.body = params;
}
}
else if(nodes[i].className === 'clearfix'){
// reset
content.push(endpoint);
endpoint = {};
}
}
content.shift(); // remove the first (empty) element
console.log(JSON.stringify(content));
/**
* Quickly scrape the API endpoints from the documentation.
* Open the browser console and drop the code there. Copy the JSON from the console. enjoy!
*
* @link https://dev.twitch.tv/docs/api/reference
*
* @created 21.03.2021
* @author @codemasher
* @copyright 2021 @codemasher
* @license MIT
*/
document.location.hash = '';
let nodes = document.getElementsByClassName('main')[0].children;
let content = [];
for(let i = 0; i < nodes.length; i++){
if(nodes[i].children.length !== 2){
continue;
}
let body = nodes[i].children[0].children;
let endpoint = {};
for(let j = 0; j < body.length; j++){
if(body[j].tagName === 'H2'){
endpoint.desc = body[j].innerText;
endpoint.link = document.location + body[j].id;
}
else if(body[j].tagName === 'H3' && body[j].innerText.match(/URL/i)){
let val = body[j+1].innerText.split(' ');
endpoint.method = val.length >= 2 ? val[0] : 'GET';
endpoint.url = val.length >= 2 ? val[1].split('?')[0].trim() : val[0];
}
else if(body[j].tagName === 'H3' && (
body[j].innerText.match(/((Required|Optional) )?(Body|Query) (Parameters?|Values?)/i)
|| body[j].innerText.match(/Response Fields?/i)
|| body[j].innerText.match(/Return Values?/i)
)){
let val = body[j+1];
let header = body[j].innerText.split(' ')
let required = header[0] === 'Required';
let type = (header.length === 3 ? header[1] : header[0]).toLowerCase();
if(type === 'return'){
type = 'response';
}
if(val.tagName !== 'TABLE'){
if(!body[j+2] || body[j+2].tagName !== 'TABLE'){
continue;
}
val = body[j+2];
}
let tbody = val.children[1] || val.children[0];
if(tbody.tagName !== 'TBODY'){
continue;
}
if(!endpoint[type]){
endpoint[type] = [];
}
for(let k = 0; k < tbody.children.length; k++){
let params = {};
let keys = tbody.children[k].children.length === 3
? ['name', 'type', 'desc']
: ['name', 'required', 'type', 'desc'];
for(let l = 0; l< tbody.children[k].children.length; l++){
params[keys[l]] = tbody.children[k].children[l].innerText;
}
if(type !== 'response'){
params.required = !params.required ? required : params.required === 'yes';
}
if(['Name', 'Field', 'Parameter'].includes(params.name) && params.type === 'Type'){
continue;
}
endpoint[type].push(params);
}
}
}
content.push(endpoint);
}
console.log(JSON.stringify(content));
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment