Last active
March 23, 2021 09:32
-
-
Save codemasher/f8aa34139b584798bbe1285abe2ec2ab to your computer and use it in GitHub Desktop.
API Doc Scraper: Endpoint Maps
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/** | |
* Quickly scrape the API endpoints from the documentation. | |
* Open the browser console and drop the code there. Copy the JSON from the console. enjoy! | |
* | |
* @link https://developer.spotify.com/documentation/web-api/reference/ | |
* | |
* @created 21.03.2021 | |
* @author @codemasher | |
* @copyright 2021 @codemasher | |
* @license MIT | |
*/ | |
function isset(accessor){ | |
try{ | |
return typeof accessor() !== 'undefined'; | |
} | |
catch(e){ | |
return false; | |
} | |
} | |
document.location.hash = ''; | |
let clearfix = document.createElement('div'); | |
clearfix.className = 'clearfix'; | |
let el = document.getElementById('objects-index'); | |
el.parentNode.insertBefore(clearfix, el); | |
let nodes = document.getElementsByClassName('post-content')[0].children; | |
let content = []; | |
let endpoint = {}; | |
for(let i = 0; i < nodes.length; i++){ | |
if(nodes[i].tagName === 'H2'){ | |
endpoint.desc = nodes[i].childNodes[0].nodeValue; | |
endpoint.link = document.location + nodes[i].id; | |
} | |
else if(nodes[i].className === 'hidden-xs'){ | |
if(isset(() => nodes[i].children[0].children[0].children[0].children[0].childNodes[0])){ | |
let val = nodes[i].children[0].children[0].children[0].children[0].childNodes[0].nodeValue.split(' '); | |
endpoint.method = val[0]; | |
endpoint.url = val[1].trim(); | |
} | |
} | |
else if(nodes[i].tagName === 'TABLE'){ | |
let type = nodes[i].children[0].children[0].children[0].childNodes[0].nodeValue; | |
let tbody = nodes[i].children[1].children; | |
let params = []; | |
// loop through the parameters | |
for(let j = 0; j < tbody.length; j++){ | |
// only the parameter fields, others are object description | |
if(tbody[j].children.length !== 3){ | |
continue; | |
} | |
let param = {}; | |
param.name = tbody[j].children[0].children[0].innerText; | |
param.type = tbody[j].children[1].innerText.toLowerCase(); | |
if(isset(() => tbody[j].children[0].children[2])){ | |
param.desc = tbody[j].children[0].children[2].innerText; | |
} | |
param.required = tbody[j].children[2].innerText.trim() === 'Required'; | |
params.push(param); | |
} | |
if(type === 'Header'){ | |
endpoint.headers = params; | |
} | |
else if(type === 'Path Parameter'){ | |
endpoint.path = params; | |
} | |
else if(type === 'Query Parameter'){ | |
endpoint.query = params; | |
} | |
else if(type === 'JSON Body Parameter'){ | |
endpoint.body = params; | |
} | |
} | |
else if(nodes[i].className === 'clearfix'){ | |
// reset | |
content.push(endpoint); | |
endpoint = {}; | |
} | |
} | |
content.shift(); // remove the first (empty) element | |
console.log(JSON.stringify(content)); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/** | |
* Quickly scrape the API endpoints from the documentation. | |
* Open the browser console and drop the code there. Copy the JSON from the console. enjoy! | |
* | |
* @link https://dev.twitch.tv/docs/api/reference | |
* | |
* @created 21.03.2021 | |
* @author @codemasher | |
* @copyright 2021 @codemasher | |
* @license MIT | |
*/ | |
document.location.hash = ''; | |
let nodes = document.getElementsByClassName('main')[0].children; | |
let content = []; | |
for(let i = 0; i < nodes.length; i++){ | |
if(nodes[i].children.length !== 2){ | |
continue; | |
} | |
let body = nodes[i].children[0].children; | |
let endpoint = {}; | |
for(let j = 0; j < body.length; j++){ | |
if(body[j].tagName === 'H2'){ | |
endpoint.desc = body[j].innerText; | |
endpoint.link = document.location + body[j].id; | |
} | |
else if(body[j].tagName === 'H3' && body[j].innerText.match(/URL/i)){ | |
let val = body[j+1].innerText.split(' '); | |
endpoint.method = val.length >= 2 ? val[0] : 'GET'; | |
endpoint.url = val.length >= 2 ? val[1].split('?')[0].trim() : val[0]; | |
} | |
else if(body[j].tagName === 'H3' && ( | |
body[j].innerText.match(/((Required|Optional) )?(Body|Query) (Parameters?|Values?)/i) | |
|| body[j].innerText.match(/Response Fields?/i) | |
|| body[j].innerText.match(/Return Values?/i) | |
)){ | |
let val = body[j+1]; | |
let header = body[j].innerText.split(' ') | |
let required = header[0] === 'Required'; | |
let type = (header.length === 3 ? header[1] : header[0]).toLowerCase(); | |
if(type === 'return'){ | |
type = 'response'; | |
} | |
if(val.tagName !== 'TABLE'){ | |
if(!body[j+2] || body[j+2].tagName !== 'TABLE'){ | |
continue; | |
} | |
val = body[j+2]; | |
} | |
let tbody = val.children[1] || val.children[0]; | |
if(tbody.tagName !== 'TBODY'){ | |
continue; | |
} | |
if(!endpoint[type]){ | |
endpoint[type] = []; | |
} | |
for(let k = 0; k < tbody.children.length; k++){ | |
let params = {}; | |
let keys = tbody.children[k].children.length === 3 | |
? ['name', 'type', 'desc'] | |
: ['name', 'required', 'type', 'desc']; | |
for(let l = 0; l< tbody.children[k].children.length; l++){ | |
params[keys[l]] = tbody.children[k].children[l].innerText; | |
} | |
if(type !== 'response'){ | |
params.required = !params.required ? required : params.required === 'yes'; | |
} | |
if(['Name', 'Field', 'Parameter'].includes(params.name) && params.type === 'Type'){ | |
continue; | |
} | |
endpoint[type].push(params); | |
} | |
} | |
} | |
content.push(endpoint); | |
} | |
console.log(JSON.stringify(content)); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment