FetchByUrls

fetchByURLs.js

const fetchByURLs_$ = (urls, parserTemplate) => {
    const url = 'mongodb://database:27017/veshapi';
    const mongo = require('mongoskin');
    const db = mongo.db(url, {native_parser: true});
    db.bind('uris');

    let promises = [];
    let index = 1;
    urls.map(url => {
        let promise = new Promise(function (resolve, reject) {
            index++;
            setTimeout(function () {
                scrapeIt(url, parserTemplate, (err, result) => {
                    if (err) {
                        reject(Error(err));
                    } else {
                        resolve(result);
                    }
                });
            }, 10000 * index);
        });

        promises.push(promise);
    });

    promises.map(promise => {
        promise.then(result => {
            let uris = result.urls.map(uri => {
                return {uri: uri.url}
            });
            db.uris.insertMany(
                uris
            );
        }).catch(e => {
            console.log(Error(e));
        });
    });

    Promise.all(promises).then(values => {
        console.log("Finished!");
        db.close();
    }).catch(reason => {
        console.log(reason);
    })

};