Skip to content

Instantly share code, notes, and snippets.

@pkej
Created August 27, 2019 17:27
Show Gist options
  • Save pkej/91f616092cbd9a644becedb1efb15ae1 to your computer and use it in GitHub Desktop.
Save pkej/91f616092cbd9a644becedb1efb15ae1 to your computer and use it in GitHub Desktop.
How to resolve promises in time?
import { link, exists } from "fs";
/* https://github.com/request/request-promise */
let rp = require('request-promise');
let cheerio = require('cheerio');
const curseBase = "https://www.curseforge.com";
const projectPath = "projects";
const modPath = "minecraft/mc-mods";
const modPackPath = "minecraft/modpacks";
const texturePackPath = "minecraft/texture-packs";
const membersPath = "members"
/*
id: the name of a curseforge author
*/
export async function getAuthor(id: string): Promise<string> {
let options = {
uri: `${curseBase}/${membersPath}/${id}`,
resolveWithFullResponse: true,
transform: function (body: string) {
return cheerio.load(body);
}
};
let response: any = {};
return rp(options)
.then(($: any) => {
let authorAvatar = $("div.user-avatar.pr-5 > div > a > img")
.attr("src").trim();
let authorName = $("div.username.text-xl")
.text().trim();
let authorLastActive = $("div.joined.text-gray-500 > abbr").attr("data-epoch").trim();
let authorJoined = $("div.text-base > div.joined.text-gray-500 > span.tip:nth-child(1)")
.text().trim();
let canonicalURL = options.uri;
let links = [];
let linkURL = [];
let linkName = [];
$("div.profile-nav > nav > ul > li > a").each(function() {
if($(this).attr("href").startsWith('/')) {
linkURL.push(curseBase + $(this).attr("href").trim());
} else {
linkURL.push($(this).attr("href").trim());
}
linkName.push($(this).text().trim());
});
for(let i = 0; i < linkURL.length; i++) {
let currentItem = {
linkURL: linkURL[i],
linkName: linkName[i]
}
links.push(currentItem);
}
let obj = {
authorURL: canonicalURL,
authorName: authorName,
authorAvatar: authorAvatar,
authorLastActive: authorLastActive,
authorJoined: authorJoined,
authorLinks: links
}
//console.log(obj);
return obj;
})
.catch((err: any) => {
console.log("Status Code Author: " + err);
});
}
/*
id: the numeric id for a curseforge project
*/
export async function getProjectOriginal(id: number): Promise<string> {
let options = {
uri: `${curseBase}/${projectPath}/${id}`,
resolveWithFullResponse: true,
transform: function (body: string) {
return cheerio.load(body);
}
};
let projectObject = {
canonicalURL: '',
projectURL: '',
title: '',
slug: '',
game: '',
projectType: '',
image: '',
curseID: '',
createdEpoch: '',
updatedEpoch: '',
created: '',
updated: '',
downloads: '',
license: '',
links: [],
authors: [],
dependencies: [],
files: []
}
let response: any = {};
return rp(options)
.then(($: any) => {
let s:string = $("meta[property='og:url']").attr("content").trim().replace(curseBase+'/', '');
let game:string = s.substring(0, s.indexOf('/'));
s = $("meta[property='og:url']").attr("content").trim().replace(curseBase+'/'+game+'/', '');
let projectType:string = s.substring(0, s.indexOf('/'));
let createdEpoch = $("aside div.w-full.flex:nth-child(2) abbr").attr("data-epoch").trim();
let updatedEpoch = $("aside div.w-full.flex:nth-child(3) abbr").attr("data-epoch").trim();
let canonicalURL = $("meta[property='og:url']").attr("content").trim();
let authors = [];
let authorURL = [];
let files = [];
let links = [];
let linkURL = [];
let linkName = [];
let auhtorPromises = [];
$('div.flex.flex-col.min-h-full.min-h-screen > main > div.z-0 > div.mx-auto.container.pb-5 > section > aside > div.my-4 > div > div:nth-child(3) p:nth-child(1) a').each(function() {
if($(this).attr("href").startsWith('/')) {
authorURL.push(curseBase + $(this).attr("href").trim());
} else {
authorURL.push($(this).attr("href").trim());
}
});
for(let i = 0; i < authorURL.length; i++) {
let currentItem:any = getAuthor(authorURL[i].replace(curseBase+'/'+membersPath+'/', ''));
auhtorPromises.push(currentItem);
}
Promise.all(auhtorPromises)
.then((results) => {
authors = results;
console.log(JSON.stringify(results,null,2)); // Result of all resolve as an array
}).catch(err => console.log(err)); // First rejected promise
$('nav.mx-auto > ul > li >a').each(function() {
if($(this).attr("href").startsWith('/')) {
linkURL.push(curseBase + $(this).attr("href").trim());
} else {
linkURL.push($(this).attr("href").trim());
}
linkName.push($(this).text().trim());
});
for(let i = 0; i < linkURL.length; i++) {
let currentItem = {
linkURL: linkURL[i],
linkName: linkName[i]
}
links.push(currentItem);
}
let obj = {
canonicalURL: canonicalURL,
projectURL: options.uri,
title: $("meta[property='og:title']").attr("content").trim(),
slug: $("meta[property='og:url']").attr("content").replace(curseBase+'/'+game+'/'+projectType+'/', ''),
game: game,
projectType: projectType,
image: $("meta[property='og:image']").attr("content").trim(),
curseID: id,
createdEpoch: createdEpoch,
updatedEpoch: updatedEpoch,
created: new Date(+createdEpoch * 1e3).toISOString(),
updated: new Date(+updatedEpoch * 1e3).toISOString(),
downloads: $("aside div.w-full.flex:nth-child(4) span:nth-child(2)").text().trim().replace(/,/g,''),
license: curseBase + $("aside div.w-full.flex:nth-child(5) a").attr("href").trim(),
links: links,
authors: authors
}
return obj;
})
.catch((err: any) => {
console.log("Status Code Project: " + err.statusCode);
});
}
/*
id: the numeric id for a curseforge project
*/
export async function getProject(id: number): Promise<string> {
let options = {
uri: `${curseBase}/${projectPath}/${id}`,
resolveWithFullResponse: true,
transform: function (body: string) {
return cheerio.load(body);
}
};
let projectObject = {
canonicalURL: '',
projectURL: '',
title: '',
slug: '',
game: '',
projectType: '',
image: '',
curseID: 0,
createdEpoch: '',
updatedEpoch: '',
created: '',
updated: '',
downloads: '',
license: '',
links: [],
authors: [],
dependencies: [],
files: []
}
let response: any = {};
return rp(options)
.then(($: any) => {
let s:string = $("meta[property='og:url']").attr("content").trim().replace(curseBase+'/', '');
projectObject.game = s.substring(0, s.indexOf('/'));
s = $("meta[property='og:url']").attr("content").trim().replace(curseBase+'/'+projectObject.game+'/', '');
projectObject.projectType = s.substring(0, s.indexOf('/'));
projectObject.createdEpoch = $("aside div.w-full.flex:nth-child(2) abbr").attr("data-epoch").trim();
projectObject.updatedEpoch = $("aside div.w-full.flex:nth-child(3) abbr").attr("data-epoch").trim();
projectObject.canonicalURL = $("meta[property='og:url']").attr("content").trim();
projectObject.created = new Date(+projectObject.createdEpoch * 1e3).toISOString();
projectObject.updated = new Date(+projectObject.updatedEpoch * 1e3).toISOString();
projectObject.projectURL = options.uri;
projectObject.title = $("meta[property='og:title']").attr("content").trim();
projectObject.slug = $("meta[property='og:url']").attr("content").replace(curseBase+'/'+projectObject.game+'/'+projectObject.projectType+'/', '');
projectObject.image = $("meta[property='og:image']").attr("content").trim();
projectObject.curseID = id;
projectObject.downloads = $("aside div.w-full.flex:nth-child(4) span:nth-child(2)").text().trim().replace(/,/g,'');
projectObject.license = curseBase + $("aside div.w-full.flex:nth-child(5) a").attr("href").trim();
return $;
}).then(($: any) => {
let links = [];
let linkURL = [];
let linkName = [];
$('nav.mx-auto > ul > li >a').each(function() {
if($(this).attr("href").startsWith('/')) {
linkURL.push(curseBase + $(this).attr("href").trim());
} else {
linkURL.push($(this).attr("href").trim());
}
linkName.push($(this).text().trim());
});
for(let i = 0; i < linkURL.length; i++) {
let currentItem = {
linkURL: linkURL[i],
linkName: linkName[i]
}
links.push(currentItem);
}
projectObject.links = links;
return $;
}).then(($: any) => {
let authors = [];
let authorURL = [];
let authorPromises = [];
$('div.flex.flex-col.min-h-full.min-h-screen > main > div.z-0 > div.mx-auto.container.pb-5 > section > aside > div.my-4 > div > div:nth-child(3) p:nth-child(1) a').each(function() {
if($(this).attr("href").startsWith('/')) {
authorURL.push(curseBase + $(this).attr("href").trim());
} else {
authorURL.push($(this).attr("href").trim());
}
});
for(let i = 0; i < authorURL.length; i++) {
let currentItem:any = getAuthor(authorURL[i].replace(curseBase+'/'+membersPath+'/', ''));
authorPromises.push(currentItem);
}
Promise.all(authorPromises)
.then((results) => {
projectObject.authors = [...results];
console.log(JSON.stringify(results,null,2)); // Result of all resolve as an array
}).catch(err => console.log(err)); // First rejected promise
return $;
}).then(($: any) => {
return projectObject;
})
.catch((err: any) => {
console.log("Status Code Project: " + err.statusCode);
});
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment