Last active
December 11, 2015 18:08
-
-
Save scriby/4639034 to your computer and use it in GitHub Desktop.
Experiment to increase serialization speed of mongodb nodejs driver
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// Write the data out to the socket | |
Connection.prototype.write = function(command, callback) { | |
try { | |
// If we have a list off commands to be executed on the same socket | |
if(Array.isArray(command)) { | |
for(var i = 0; i < command.length; i++) { | |
var binaryCommand = command[i].toBinary() | |
if(!this.socketOptions['disableDriverBSONSizeCheck'] && binaryCommand.length > this.maxBsonSize) | |
return callback(new Error("Document exceeds maximal allowed bson size of " + this.maxBsonSize + " bytes")); | |
if(this.logger != null && this.logger.doDebug) | |
this.logger.debug("writing command to mongodb", {binary: binaryCommand, json: command[i]}); | |
if(Array.isArray(binaryCommand)) { | |
for(var b = 0; b < binaryCommand.length; b++){ | |
this.writeSteam.write(binaryCommand[b]); | |
} | |
} else { | |
this.writeSteam.write(binaryCommand); | |
} | |
} | |
} else { | |
var binaryCommand = command.toBinary() | |
if(!this.socketOptions['disableDriverBSONSizeCheck'] && binaryCommand.length > this.maxBsonSize) | |
return callback(new Error("Document exceeds maximal allowed bson size of " + this.maxBsonSize + " bytes")); | |
if(this.logger != null && this.logger.doDebug) | |
this.logger.debug("writing command to mongodb", {binary: binaryCommand, json: command}); | |
if(Array.isArray(binaryCommand)) { | |
for(var b = 0; b < binaryCommand.length; b++){ | |
this.writeSteam.write(binaryCommand[b]); | |
} | |
} else { | |
this.writeSteam.write(binaryCommand); | |
} | |
} | |
} catch (err) { | |
console.log(err); | |
if(typeof callback === 'function') callback(err); | |
} | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
var FakeBuffer = function(){ | |
//Pass an instance of this to serializeWithBufferAndIndex instead of a buffer | |
//Allocates buffers as necessary, and adds to a buffer array | |
//Requires changing the way the buffer is written to in the serializeWithBufferAndIndex method | |
}; | |
InsertCommand.prototype.toBinary = function() { | |
// Calculate total length of the document | |
var totalLengthOfCommand; | |
// Let's build the single pass buffer command | |
var _command = new Buffer(4 + Buffer.byteLength(this.collectionName) + 1 + (4 * 4)); | |
var idx = 4;//Skip first 4 bytes, which is total message length | |
// Write the header information to the buffer | |
// Write the request ID | |
_command[idx++] = this.requestId & 0xff; | |
_command[idx++] = (this.requestId >> 8) & 0xff; | |
_command[idx++] = (this.requestId >> 16) & 0xff; | |
_command[idx++] = (this.requestId >> 24) & 0xff; | |
// Write zero | |
_command[idx++] = 0; | |
_command[idx++] = 0; | |
_command[idx++] = 0; | |
_command[idx++] = 0; | |
// Write the op_code for the command | |
_command[idx++] = InsertCommand.OP_INSERT & 0xff; | |
_command[idx++] = (InsertCommand.OP_INSERT >> 8) & 0xff; | |
_command[idx++] = (InsertCommand.OP_INSERT >> 16) & 0xff; | |
_command[idx++] = (InsertCommand.OP_INSERT >> 24) & 0xff; | |
// Write flags if any | |
_command[idx++] = this.flags & 0xff; | |
_command[idx++] = (this.flags >> 8) & 0xff; | |
_command[idx++] = (this.flags >> 16) & 0xff; | |
_command[idx++] = (this.flags >> 24) & 0xff; | |
// Write the collection name to the command | |
idx += _command.write(this.collectionName, idx, 'utf8'); | |
_command[idx++] = 0; | |
var docBuffers = [ _command ]; | |
// Write all the bson documents to the buffer at the index offset | |
for(var i = 0; i < this.documents.length; i++) { | |
// Document binary length | |
var documentLength = 0; | |
var object = this.documents[i]; | |
// Serialize the selector | |
// If we are passing a raw buffer, do minimal validation | |
var buf; | |
if(Buffer.isBuffer(object)) { | |
documentLength = object.length; | |
// Copy the data into the current buffer | |
buf = object; | |
} else { | |
// Serialize the document straight to the buffer | |
buf = new Buffer(4096); //new FakeBuffer() | |
documentLength = this.db.bson.serializeWithBufferAndIndex(object, this.checkKeys, buf, 0, this.serializeFunctions) + 1;//Add null bit | |
} | |
// Write the length to the document | |
buf[0] = documentLength & 0xff; | |
buf[1] = (documentLength >> 8) & 0xff; | |
buf[2] = (documentLength >> 16) & 0xff; | |
buf[3] = (documentLength >> 24) & 0xff; | |
buf[documentLength - 1] = 0; | |
docBuffers.push(buf.slice(0, documentLength)); | |
//Add underlying buffers from the FakeBuffer to docBuffers here | |
idx += documentLength; | |
} | |
totalLengthOfCommand = idx; | |
_command[0] = totalLengthOfCommand & 0xff; | |
_command[1] = (totalLengthOfCommand >> 8) & 0xff; | |
_command[2] = (totalLengthOfCommand >> 16) & 0xff; | |
_command[3] = (totalLengthOfCommand >> 24) & 0xff; | |
return docBuffers; | |
}; |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment