Forked from littleskunk/Storj_Farmer_Delete_Lost_Shards.js
Last active
November 27, 2017 18:21
-
-
Save AlexeyALeonov/9be2cabec6b40907674d476c70f302fa to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
$start=0; | |
do { | |
$res = node ~\Storj_Farmer_Delete_Lost_Shards.js $start; | |
$res; | |
$start = ($res | sls "Reading bucket (\d*)" | select -Last 1).Matches.Groups[1].Value; | |
$start; | |
} until ($res.Contains("Reading bucket 255")) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// The shard reaper will only delete expired contract inclusive shards. | |
// He will not detect shards with a missing contract | |
// This script will delete the lost shards | |
// How to run it | |
// Requires nodejs LTS, git, python2 | |
// Open command line and move to the folder where this script is stored | |
// Execute "npm install storj-lib" | |
// Execute "node Storj_Farmer_Delete_Lost_Shards.js" | |
// Optional argument - starting number of the bucket | |
var start = Number(process.argv[2]); | |
if (!start) {start = 0;} | |
var async = require('async'); | |
var storj = require('storj-lib'); | |
var stream = require('readable-stream'); | |
var persistence = new storj.EmbeddedStorageAdapter('insert your storage location here'); | |
var manager = new storj.StorageManager(persistence); | |
var StorageAdapter = require('storj-lib/lib/storage/adapter'); | |
var StorageItem = require('storj-lib/lib/storage/item'); | |
var rstream = manager._storage.createReadStream(); | |
var list = []; | |
var buckets = []; | |
for(var i = start; i <= 255; i++){ | |
buckets.push(i); | |
} | |
console.log('Reading contracts'); | |
rstream.on('data', function(item) { | |
rstream.pause(); | |
list.push(item.fskey || item.hash); | |
rstream.resume(); | |
}); | |
rstream.on('end', function() { | |
console.log('Reading contracts finished'); | |
async.forEachLimit(buckets, 1, function (i, callback){ | |
console.log('Reading bucket %s', i); | |
persistence._fs.list(i, (err, keys) => { | |
if (keys) { | |
keys.forEach((result) => { | |
if (list.indexOf(result.baseKey) == -1) { | |
persistence._fs.unlink(result.baseKey, function(/* err */) { | |
console.log('Deleting lost shard: %s', result.baseKey); | |
}); | |
} | |
}); | |
} | |
persistence._fs._getSbucketAtIndex(i).close(); | |
callback(); | |
}); | |
}, function(err) { | |
console.log('flushing shards, some buckets will be inaccessible'); | |
manager._storage.flush(function(err) { | |
/* istanbul ignore if */ | |
if (err) { | |
console.log('problem while flushing shards, %s', err.message); | |
} | |
console.log('flushing shards finished'); | |
process.exit(); | |
}); | |
}); | |
}); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// How to run it | |
// Requires nodejs LTS, git, python2 | |
// Open command line and move to the folder where this script is stored | |
// Execute "npm install storj-lib" | |
// Execute "node Storj_Farmer_Manuall_Reaper.js" | |
var storj = require('storj-lib'); | |
// insert your storage location here. | |
// on Windows you need double backslash. C:\\storjshare\\whatever | |
var persistence = new storj.EmbeddedStorageAdapter('insert your storage location here'); | |
var manager = new storj.StorageManager(persistence); | |
var StorageAdapter = require('storj-lib/lib/storage/adapter'); | |
var StorageItem = require('storj-lib/lib/storage/item'); | |
var rstream = manager._storage.createReadStream(); | |
var timestamp = Date.now(); | |
rstream.on('data', function(item) { | |
rstream.pause(); | |
var total = Object.keys(item.contracts).length; | |
var endedOrIncomplete = 0; | |
for (var nodeID in item.contracts) { | |
var ended = item.contracts[nodeID].get('store_end') < timestamp; | |
var incomplete = !item.contracts[nodeID].isComplete(); | |
if (ended || incomplete ) { | |
endedOrIncomplete++; | |
} | |
} | |
if (total === endedOrIncomplete) { | |
console.log('destroying shard/contract for %s', item.hash); | |
manager._storage.del(item.hash, function(/* err */) { | |
rstream.resume(); | |
}); | |
} else { | |
rstream.resume(); | |
} | |
}); | |
rstream.on('end', function() { | |
console.log('flushing shards, some buckets will be inaccessible'); | |
manager._storage.flush(function(err) { | |
/* istanbul ignore if */ | |
if (err) { | |
console.log('problem while flushing shards, %s', err.message); | |
} | |
console.log('flushing shards finished'); | |
process.exit(); | |
}); | |
}); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment