Skip to content

Instantly share code, notes, and snippets.

@scriby
Last active December 11, 2015 18:08
Show Gist options
  • Save scriby/4639034 to your computer and use it in GitHub Desktop.
Save scriby/4639034 to your computer and use it in GitHub Desktop.
Experiment to increase serialization speed of mongodb nodejs driver
// Write the data out to the socket
Connection.prototype.write = function(command, callback) {
try {
// If we have a list off commands to be executed on the same socket
if(Array.isArray(command)) {
for(var i = 0; i < command.length; i++) {
var binaryCommand = command[i].toBinary()
if(!this.socketOptions['disableDriverBSONSizeCheck'] && binaryCommand.length > this.maxBsonSize)
return callback(new Error("Document exceeds maximal allowed bson size of " + this.maxBsonSize + " bytes"));
if(this.logger != null && this.logger.doDebug)
this.logger.debug("writing command to mongodb", {binary: binaryCommand, json: command[i]});
if(Array.isArray(binaryCommand)) {
for(var b = 0; b < binaryCommand.length; b++){
this.writeSteam.write(binaryCommand[b]);
}
} else {
this.writeSteam.write(binaryCommand);
}
}
} else {
var binaryCommand = command.toBinary()
if(!this.socketOptions['disableDriverBSONSizeCheck'] && binaryCommand.length > this.maxBsonSize)
return callback(new Error("Document exceeds maximal allowed bson size of " + this.maxBsonSize + " bytes"));
if(this.logger != null && this.logger.doDebug)
this.logger.debug("writing command to mongodb", {binary: binaryCommand, json: command});
if(Array.isArray(binaryCommand)) {
for(var b = 0; b < binaryCommand.length; b++){
this.writeSteam.write(binaryCommand[b]);
}
} else {
this.writeSteam.write(binaryCommand);
}
}
} catch (err) {
console.log(err);
if(typeof callback === 'function') callback(err);
}
}
var FakeBuffer = function(){
//Pass an instance of this to serializeWithBufferAndIndex instead of a buffer
//Allocates buffers as necessary, and adds to a buffer array
//Requires changing the way the buffer is written to in the serializeWithBufferAndIndex method
};
InsertCommand.prototype.toBinary = function() {
// Calculate total length of the document
var totalLengthOfCommand;
// Let's build the single pass buffer command
var _command = new Buffer(4 + Buffer.byteLength(this.collectionName) + 1 + (4 * 4));
var idx = 4;//Skip first 4 bytes, which is total message length
// Write the header information to the buffer
// Write the request ID
_command[idx++] = this.requestId & 0xff;
_command[idx++] = (this.requestId >> 8) & 0xff;
_command[idx++] = (this.requestId >> 16) & 0xff;
_command[idx++] = (this.requestId >> 24) & 0xff;
// Write zero
_command[idx++] = 0;
_command[idx++] = 0;
_command[idx++] = 0;
_command[idx++] = 0;
// Write the op_code for the command
_command[idx++] = InsertCommand.OP_INSERT & 0xff;
_command[idx++] = (InsertCommand.OP_INSERT >> 8) & 0xff;
_command[idx++] = (InsertCommand.OP_INSERT >> 16) & 0xff;
_command[idx++] = (InsertCommand.OP_INSERT >> 24) & 0xff;
// Write flags if any
_command[idx++] = this.flags & 0xff;
_command[idx++] = (this.flags >> 8) & 0xff;
_command[idx++] = (this.flags >> 16) & 0xff;
_command[idx++] = (this.flags >> 24) & 0xff;
// Write the collection name to the command
idx += _command.write(this.collectionName, idx, 'utf8');
_command[idx++] = 0;
var docBuffers = [ _command ];
// Write all the bson documents to the buffer at the index offset
for(var i = 0; i < this.documents.length; i++) {
// Document binary length
var documentLength = 0;
var object = this.documents[i];
// Serialize the selector
// If we are passing a raw buffer, do minimal validation
var buf;
if(Buffer.isBuffer(object)) {
documentLength = object.length;
// Copy the data into the current buffer
buf = object;
} else {
// Serialize the document straight to the buffer
buf = new Buffer(4096); //new FakeBuffer()
documentLength = this.db.bson.serializeWithBufferAndIndex(object, this.checkKeys, buf, 0, this.serializeFunctions) + 1;//Add null bit
}
// Write the length to the document
buf[0] = documentLength & 0xff;
buf[1] = (documentLength >> 8) & 0xff;
buf[2] = (documentLength >> 16) & 0xff;
buf[3] = (documentLength >> 24) & 0xff;
buf[documentLength - 1] = 0;
docBuffers.push(buf.slice(0, documentLength));
//Add underlying buffers from the FakeBuffer to docBuffers here
idx += documentLength;
}
totalLengthOfCommand = idx;
_command[0] = totalLengthOfCommand & 0xff;
_command[1] = (totalLengthOfCommand >> 8) & 0xff;
_command[2] = (totalLengthOfCommand >> 16) & 0xff;
_command[3] = (totalLengthOfCommand >> 24) & 0xff;
return docBuffers;
};
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment