Skip to content

Instantly share code, notes, and snippets.

@StevenMcD
Created October 21, 2012 20:14
Show Gist options
  • Save StevenMcD/3928359 to your computer and use it in GitHub Desktop.
Save StevenMcD/3928359 to your computer and use it in GitHub Desktop.
NodeJS - Parallel Tasking - reading multiple files at once and counting word totals
var fs = require('fs'),
completedTasks = 0,
tasks = [],
wordCounts = {},
filesDir = './text';
function checkIfComplete(){
completedTasks++;
if(completedTasks == tasks.length){
for(var index in wordCounts){
console.log(index + ': ' + wordCounts[index]);
}
}
};
function countWordsInText(text) {
var words = text
.toString()
.toLowerCase()
.split(/\W+/)
.sort();
for(var index in words) {
var word = words[index];
if (word) {
wordCounts[word] = (wordCounts[word]) ? wordCounts[word] + 1 : 1;
}
}
};
fs.readdir(filesDir, function(err, files){
if(err) throw err;
for(var index in files){
var task = (function(file){
return function(){
fs.readFile(file, function(err, text){
if(err) throw err;
countWordsInText(text);
checkIfComplete();
});
};
})(filesDir + '/' + files[index]);
tasks.push(task);
};
for(var task in tasks){
tasks[task]();
}
});
@solnurkarim
Copy link

solnurkarim commented Jul 10, 2018

Why did you use self-invoking function and then call it again within another for loop?

Precicely, why this:

fs.readdir(filesDir, function(err, files){
	if(err) throw err;

	for(var index in files){
		var task = (function(file){
			return function(){
				fs.readFile(file, function(err, text){
					if(err) throw err;

					countWordsInText(text);
					checkIfComplete();
				});
			};
		})(filesDir + '/' + files[index]);

		tasks.push(task);
	};

	for(var task in tasks){
		tasks[task]();
	}
});

and not this:

fs.readdir(filesDir, function(err, files) {
	if (err) throw err;
	for (var index in files) {
		tasksCount++; // tasks counter added
		fs.readFile(filesDir + '/' + files[index], function(err, text) {
			if (err) throw err;

			countWordsInText(text);
			checkIfComplete();
		});
	}
});

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment