line-by-line read large text file

demo.js

var lineReader = require('./line-by-line.js');

var fetchBlock = lineReader( filepath );
var i = 0;  //从最开始计数，第几行

var fetchLine = function(callback){
    var line = lines.shift();

    if(!line){
        fetchBlock(function(lns, start){
            console.log('input> fetch block');
            if(!lns){
                console.log('input> file read out!');
                callback(null);
            }else{
                lines = lns;
                i = start;
                process.nextTick(function(){
                    fetchLine(callback);
                });
            }
        });
    }else{
        callback(line, i);
        i++;
    }
};

var next = function(){
    fetchLine(function(line, i){
        if(!line){
            done();
            return;
        }
        //line 是当前行
        //i 是行序号
        do_something();  //...
        setTimeout(next);  //防止调用栈溢出
    });
};

//一个有用的工具函数，调用linux的sed命令取得文件行数
//callback(n)  n是行数
var totalLines = function(filepath, callback){
    //读取输入文件总行数
    var exec = require('child_process').exec;
    var cmd = 'sed "/^\s*$/d" '+ filepath +' | wc -l'
    var wc = exec(cmd, function(err, stdout){
        if(err){
            throw err;
        }else{
            var c = parseInt(stdout);
            console.log('total work ', c)
            callback(c);
        }
    });
}