Created
August 27, 2019 17:27
-
-
Save pkej/91f616092cbd9a644becedb1efb15ae1 to your computer and use it in GitHub Desktop.
How to resolve promises in time?
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import { link, exists } from "fs"; | |
/* https://github.com/request/request-promise */ | |
let rp = require('request-promise'); | |
let cheerio = require('cheerio'); | |
const curseBase = "https://www.curseforge.com"; | |
const projectPath = "projects"; | |
const modPath = "minecraft/mc-mods"; | |
const modPackPath = "minecraft/modpacks"; | |
const texturePackPath = "minecraft/texture-packs"; | |
const membersPath = "members" | |
/* | |
id: the name of a curseforge author | |
*/ | |
export async function getAuthor(id: string): Promise<string> { | |
let options = { | |
uri: `${curseBase}/${membersPath}/${id}`, | |
resolveWithFullResponse: true, | |
transform: function (body: string) { | |
return cheerio.load(body); | |
} | |
}; | |
let response: any = {}; | |
return rp(options) | |
.then(($: any) => { | |
let authorAvatar = $("div.user-avatar.pr-5 > div > a > img") | |
.attr("src").trim(); | |
let authorName = $("div.username.text-xl") | |
.text().trim(); | |
let authorLastActive = $("div.joined.text-gray-500 > abbr").attr("data-epoch").trim(); | |
let authorJoined = $("div.text-base > div.joined.text-gray-500 > span.tip:nth-child(1)") | |
.text().trim(); | |
let canonicalURL = options.uri; | |
let links = []; | |
let linkURL = []; | |
let linkName = []; | |
$("div.profile-nav > nav > ul > li > a").each(function() { | |
if($(this).attr("href").startsWith('/')) { | |
linkURL.push(curseBase + $(this).attr("href").trim()); | |
} else { | |
linkURL.push($(this).attr("href").trim()); | |
} | |
linkName.push($(this).text().trim()); | |
}); | |
for(let i = 0; i < linkURL.length; i++) { | |
let currentItem = { | |
linkURL: linkURL[i], | |
linkName: linkName[i] | |
} | |
links.push(currentItem); | |
} | |
let obj = { | |
authorURL: canonicalURL, | |
authorName: authorName, | |
authorAvatar: authorAvatar, | |
authorLastActive: authorLastActive, | |
authorJoined: authorJoined, | |
authorLinks: links | |
} | |
//console.log(obj); | |
return obj; | |
}) | |
.catch((err: any) => { | |
console.log("Status Code Author: " + err); | |
}); | |
} | |
/* | |
id: the numeric id for a curseforge project | |
*/ | |
export async function getProjectOriginal(id: number): Promise<string> { | |
let options = { | |
uri: `${curseBase}/${projectPath}/${id}`, | |
resolveWithFullResponse: true, | |
transform: function (body: string) { | |
return cheerio.load(body); | |
} | |
}; | |
let projectObject = { | |
canonicalURL: '', | |
projectURL: '', | |
title: '', | |
slug: '', | |
game: '', | |
projectType: '', | |
image: '', | |
curseID: '', | |
createdEpoch: '', | |
updatedEpoch: '', | |
created: '', | |
updated: '', | |
downloads: '', | |
license: '', | |
links: [], | |
authors: [], | |
dependencies: [], | |
files: [] | |
} | |
let response: any = {}; | |
return rp(options) | |
.then(($: any) => { | |
let s:string = $("meta[property='og:url']").attr("content").trim().replace(curseBase+'/', ''); | |
let game:string = s.substring(0, s.indexOf('/')); | |
s = $("meta[property='og:url']").attr("content").trim().replace(curseBase+'/'+game+'/', ''); | |
let projectType:string = s.substring(0, s.indexOf('/')); | |
let createdEpoch = $("aside div.w-full.flex:nth-child(2) abbr").attr("data-epoch").trim(); | |
let updatedEpoch = $("aside div.w-full.flex:nth-child(3) abbr").attr("data-epoch").trim(); | |
let canonicalURL = $("meta[property='og:url']").attr("content").trim(); | |
let authors = []; | |
let authorURL = []; | |
let files = []; | |
let links = []; | |
let linkURL = []; | |
let linkName = []; | |
let auhtorPromises = []; | |
$('div.flex.flex-col.min-h-full.min-h-screen > main > div.z-0 > div.mx-auto.container.pb-5 > section > aside > div.my-4 > div > div:nth-child(3) p:nth-child(1) a').each(function() { | |
if($(this).attr("href").startsWith('/')) { | |
authorURL.push(curseBase + $(this).attr("href").trim()); | |
} else { | |
authorURL.push($(this).attr("href").trim()); | |
} | |
}); | |
for(let i = 0; i < authorURL.length; i++) { | |
let currentItem:any = getAuthor(authorURL[i].replace(curseBase+'/'+membersPath+'/', '')); | |
auhtorPromises.push(currentItem); | |
} | |
Promise.all(auhtorPromises) | |
.then((results) => { | |
authors = results; | |
console.log(JSON.stringify(results,null,2)); // Result of all resolve as an array | |
}).catch(err => console.log(err)); // First rejected promise | |
$('nav.mx-auto > ul > li >a').each(function() { | |
if($(this).attr("href").startsWith('/')) { | |
linkURL.push(curseBase + $(this).attr("href").trim()); | |
} else { | |
linkURL.push($(this).attr("href").trim()); | |
} | |
linkName.push($(this).text().trim()); | |
}); | |
for(let i = 0; i < linkURL.length; i++) { | |
let currentItem = { | |
linkURL: linkURL[i], | |
linkName: linkName[i] | |
} | |
links.push(currentItem); | |
} | |
let obj = { | |
canonicalURL: canonicalURL, | |
projectURL: options.uri, | |
title: $("meta[property='og:title']").attr("content").trim(), | |
slug: $("meta[property='og:url']").attr("content").replace(curseBase+'/'+game+'/'+projectType+'/', ''), | |
game: game, | |
projectType: projectType, | |
image: $("meta[property='og:image']").attr("content").trim(), | |
curseID: id, | |
createdEpoch: createdEpoch, | |
updatedEpoch: updatedEpoch, | |
created: new Date(+createdEpoch * 1e3).toISOString(), | |
updated: new Date(+updatedEpoch * 1e3).toISOString(), | |
downloads: $("aside div.w-full.flex:nth-child(4) span:nth-child(2)").text().trim().replace(/,/g,''), | |
license: curseBase + $("aside div.w-full.flex:nth-child(5) a").attr("href").trim(), | |
links: links, | |
authors: authors | |
} | |
return obj; | |
}) | |
.catch((err: any) => { | |
console.log("Status Code Project: " + err.statusCode); | |
}); | |
} | |
/* | |
id: the numeric id for a curseforge project | |
*/ | |
export async function getProject(id: number): Promise<string> { | |
let options = { | |
uri: `${curseBase}/${projectPath}/${id}`, | |
resolveWithFullResponse: true, | |
transform: function (body: string) { | |
return cheerio.load(body); | |
} | |
}; | |
let projectObject = { | |
canonicalURL: '', | |
projectURL: '', | |
title: '', | |
slug: '', | |
game: '', | |
projectType: '', | |
image: '', | |
curseID: 0, | |
createdEpoch: '', | |
updatedEpoch: '', | |
created: '', | |
updated: '', | |
downloads: '', | |
license: '', | |
links: [], | |
authors: [], | |
dependencies: [], | |
files: [] | |
} | |
let response: any = {}; | |
return rp(options) | |
.then(($: any) => { | |
let s:string = $("meta[property='og:url']").attr("content").trim().replace(curseBase+'/', ''); | |
projectObject.game = s.substring(0, s.indexOf('/')); | |
s = $("meta[property='og:url']").attr("content").trim().replace(curseBase+'/'+projectObject.game+'/', ''); | |
projectObject.projectType = s.substring(0, s.indexOf('/')); | |
projectObject.createdEpoch = $("aside div.w-full.flex:nth-child(2) abbr").attr("data-epoch").trim(); | |
projectObject.updatedEpoch = $("aside div.w-full.flex:nth-child(3) abbr").attr("data-epoch").trim(); | |
projectObject.canonicalURL = $("meta[property='og:url']").attr("content").trim(); | |
projectObject.created = new Date(+projectObject.createdEpoch * 1e3).toISOString(); | |
projectObject.updated = new Date(+projectObject.updatedEpoch * 1e3).toISOString(); | |
projectObject.projectURL = options.uri; | |
projectObject.title = $("meta[property='og:title']").attr("content").trim(); | |
projectObject.slug = $("meta[property='og:url']").attr("content").replace(curseBase+'/'+projectObject.game+'/'+projectObject.projectType+'/', ''); | |
projectObject.image = $("meta[property='og:image']").attr("content").trim(); | |
projectObject.curseID = id; | |
projectObject.downloads = $("aside div.w-full.flex:nth-child(4) span:nth-child(2)").text().trim().replace(/,/g,''); | |
projectObject.license = curseBase + $("aside div.w-full.flex:nth-child(5) a").attr("href").trim(); | |
return $; | |
}).then(($: any) => { | |
let links = []; | |
let linkURL = []; | |
let linkName = []; | |
$('nav.mx-auto > ul > li >a').each(function() { | |
if($(this).attr("href").startsWith('/')) { | |
linkURL.push(curseBase + $(this).attr("href").trim()); | |
} else { | |
linkURL.push($(this).attr("href").trim()); | |
} | |
linkName.push($(this).text().trim()); | |
}); | |
for(let i = 0; i < linkURL.length; i++) { | |
let currentItem = { | |
linkURL: linkURL[i], | |
linkName: linkName[i] | |
} | |
links.push(currentItem); | |
} | |
projectObject.links = links; | |
return $; | |
}).then(($: any) => { | |
let authors = []; | |
let authorURL = []; | |
let authorPromises = []; | |
$('div.flex.flex-col.min-h-full.min-h-screen > main > div.z-0 > div.mx-auto.container.pb-5 > section > aside > div.my-4 > div > div:nth-child(3) p:nth-child(1) a').each(function() { | |
if($(this).attr("href").startsWith('/')) { | |
authorURL.push(curseBase + $(this).attr("href").trim()); | |
} else { | |
authorURL.push($(this).attr("href").trim()); | |
} | |
}); | |
for(let i = 0; i < authorURL.length; i++) { | |
let currentItem:any = getAuthor(authorURL[i].replace(curseBase+'/'+membersPath+'/', '')); | |
authorPromises.push(currentItem); | |
} | |
Promise.all(authorPromises) | |
.then((results) => { | |
projectObject.authors = [...results]; | |
console.log(JSON.stringify(results,null,2)); // Result of all resolve as an array | |
}).catch(err => console.log(err)); // First rejected promise | |
return $; | |
}).then(($: any) => { | |
return projectObject; | |
}) | |
.catch((err: any) => { | |
console.log("Status Code Project: " + err.statusCode); | |
}); | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment