js.js

const request = require('request');
const cheerio = require('cheerio');
const fs = require('fs');
const util = require('util');


run();

async function run(){
	try{
		console.log('start');
		
		run_GB(GBs[0]);
		
	}catch(err){
		console.log('error ', err);
	}

}


async function run_GB(GB){
	let url = 'https://forgeofempires.fandom.com/wiki/'+GB[0];

	let body = await downloadFile(url);
	
	await process(body, GB[1]);
	
	console.log('Saved');
}

async function process(body, GB_prefix){
	console.log('processing '+GB_prefix);
	
	$ = cheerio.load(body);
	
	let levels = await process_levels($);
	
	//...
}


function request_promise(url){
	return new Promise(function(resolve, reject) {
        try{
			request(url, function(err, resp, body){
				if(err){
					console.log('err= ', err);
					console.log('body= ', body.length);
					reject(err);
					return;
				}
				resolve(body);
			});
		}catch(e){
			reject(e);
		}
    });
}

async function downloadFile(url){
	console.log('downloading '+url);
	
	let body = await request_promise(url);
	
	console.log('done, size= '+body.length);
	
	return body;
}