Last active
June 11, 2016 11:12
-
-
Save checkaayush/4228b87431c2caed7aa1962a2366c449 to your computer and use it in GitHub Desktop.
Node.js script to load .txt files from a set of URLs asynchronously and combine results in order of URLs using default modules (and with non-default modules)
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// Task using default Node.js packages | |
var http = require('http'); | |
var urls = ['http://screeningtest.vdocipher.com/file1.txt', | |
'http://screeningtest.vdocipher.com/file2.txt', | |
'http://screeningtest.vdocipher.com/file3.txt', | |
'http://screeningtest.vdocipher.com/file4.txt', | |
'http://screeningtest.vdocipher.com/file5.txt', | |
'http://screeningtest.vdocipher.com/file6.txt', | |
'http://screeningtest.vdocipher.com/file7.txt', | |
'http://screeningtest.vdocipher.com/file8.txt', | |
'http://screeningtest.vdocipher.com/file9.txt', | |
'http://screeningtest.vdocipher.com/file10.txt']; | |
var responses = []; | |
var completedRequests = 0; | |
/** | |
* Given a 'url', it returns a JSON object of the form | |
* {"url": <someUrl>, body: <textFileContentAtUrl>} | |
*/ | |
function httpGet(url, callback) { | |
var returnVal = {url: url}; | |
http.get(url, function(response) { | |
var output = ''; | |
response.on('data', function(chunk){ | |
output += chunk; | |
}); | |
response.on('end', function(){ | |
returnVal['body'] = output.trim(); | |
callback(returnVal); | |
}); | |
}); | |
} | |
function httpHelper(urls, callback) { | |
for (var index = 0; index < urls.length; index++) { | |
httpGet(urls[index], function(body) { | |
completedRequests++; | |
responses.push(body); | |
if (completedRequests == urls.length) { | |
callback(responses); | |
} | |
}); | |
} | |
} | |
httpHelper(urls, function(responses) { | |
var outputArray = []; | |
for (var index in responses) { | |
var urlIndex = urls.indexOf(responses[index].url); | |
outputArray.splice(urlIndex, 0, responses[index].body); | |
} | |
var outputString = outputArray.join(); | |
console.log(outputString); | |
}); | |
// Additional Method using non-default Node.js packages: 'async' & 'request' | |
var async = require('async'); | |
var request = require('request'); | |
function httpGet(url, callback) { | |
const options = { | |
url : url, | |
json : true | |
}; | |
request(options, | |
function(err, res, body) { | |
callback(err, body); | |
} | |
); | |
} | |
async.map(urls, httpGet, function (err, res){ | |
if (err) return console.log(err); | |
console.log(res.join()); | |
}); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment