Last active
December 20, 2015 21:09
-
-
Save andersonmat/6195490 to your computer and use it in GitHub Desktop.
Streaming Riak bucket deletion in JavaScript. Needs riak-js. Now clustered because node runs out of RAM when you have too many keys.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/** | |
This script will delete an entire bucket's contents. Use with caution. | |
*/ | |
// allow us to spawn children to evade memory issues | |
var cluster = require('cluster'); | |
// connect to riak | |
var riakConnection = require('riak-js').getClient({ | |
host : '10.2.48.20', | |
port : 8098 | |
}), bucketName = 'fusionData', keyIncrement = 10000; | |
// run key retrieval on the master | |
if(cluster.isMaster) { | |
// key listing, removal counting for pretty printing in the console | |
var keysToRemove = [], removedCount = 0; | |
// retrieve the database keys (streaming, this helps with large buckets) | |
console.log('Retrieving Keys To Delete'); | |
riakConnection.keys(bucketName, { | |
keys : 'stream' | |
}).on('keys', function(keyList) { | |
// append values to remove | |
keysToRemove = keysToRemove.concat(keyList); | |
}).on('end', function() { | |
console.log('Finished Retrieving Keys, Length:', keysToRemove.length); | |
// remove individual keys and delete them | |
var nextArray; | |
while((nextArray = keysToRemove.splice(0, keyIncrement)) && nextArray.length) { | |
// spawn new child to do dirty work | |
cluster.fork().send(nextArray); | |
} | |
}).start(); | |
} else { | |
process.on('message', function(keyData) { | |
keyData.forEach(function(aKey, index) { | |
console.log('[', process.pid,'] Has', keyData.length - index, ' Keys Left'); | |
riakConnection.remove(bucketName, aKey); | |
}); | |
}); | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment