Bloomberg scraper

bloomberg.js

var config = require('./config'), request = require('request'), fs = require('fs'), Zip = require('adm-zip'), path = require('path');

var tmpDir = 'tmp';
fs.mkdir(tmpDir);

request.post(config.bloomberg.bsym_url, function (err, res, body) {
    if(err) throw err;
    var record = eval('(' + body.substring(body.indexOf('{data'), body.lastIndexOf(';')));
    record.data.forEach(function(data) {
        var url = data.uri, name = path.join(tmpDir, url.substring(url.lastIndexOf('/') + 1));
        console.log('Downloading ' + url + ' ...');
        var dest = fs.createWriteStream(name);
        request(url).pipe(dest, {close: false});
        dest.on('close', function() {
            console.log('Unzipping ' + name + ' ...');
            var zip = new Zip(name);
            fs.writeFile(name.replace('.zip', ''), zip.readAsText(zip.getEntries()[0]), function(err, res) {
                fs.unlink(name);
            });
        });
    });
});