Incremental Map-Reduce of MongoDB, http://meetfp.com/en/blog/mongo-mapreduce-cron

Using cron to perform incremental Map-Reduce in MongoDB

Using cron to perform incremental Map-Reduce in MongoDB

1. init_tags.js

//The Map function, simply count tags
var mapTags = function() {
    if (!this.tags) {
        return;
    }

    for (index in this.tags) {
        emit(this.tags[index], 1);
    }
};

//The Reduce function, accumulate the count
var reduceTags = function(previous, current) {
    var count = 0;

    for (index in current) {
        count += current[index];
    }

    return count;
}

var conn = new Mongo();
var db = conn.getDB("your_db");

//save the functions
db.system.js.save(
               { _id: "mapTags",
                 value : mapTags
                   }
                 )

db.system.js.save(
               { _id: "reduceTags",
                 value : reduceTags
                   }
                 )
//execute the Map-Reduce, 
//the results will be stored in the calc_tags collection
db.yeedd.mapReduce( mapTags,
                    reduceTags,
                    {
                      out: "calc_tags"
                    }
                  )

2. inc_tags.js

//convert ISO data to ObjectId so that it can be compared with ObjectIds in db
var ISODateToObjectId = function(date){
    // Convert date to hex seconds
    var hexSeconds = Math.floor(date.getTime()/1000).toString(16);

    // Create an ObjectId with the hex timestamp
    var id = ObjectId(hexSeconds + "0000000000000000");

    return id
}

var conn = new Mongo();
var db = conn.getDB("your_db");

var start = new Date();
start.setDate(start.getDate()  - 1)

// The Map and the Reduce functions have been stored into system.js
// and they can be used in the mapReduce function now 
db.yeedd.mapReduce( mapTags,
                    reduceTags,
                    {
                         query: { _id: { $gt: ISODateToObjectId(start) } },
                         out: { reduce: "calc_tags" }
                       }
                     );
}

3. init_tags.sh

# execute javascript file with mongo shell. 
# please use your own file path and database path instead
/usr/bin/mongo "localhost:27017/db"  /path/to/init_tags.js

4. inc_tags.sh

# execute javascript file with mongo shell. 
# please use your own file path and database path instead
/usr/bin/mongo "localhost:27017/db"  /path/to/inc_tags.js

5. cron_job

01 0 * * * /path/to/inc_tags_job.sh