Storj_Farmer_Delete_Lost_Shards.js

// The shard reaper will only delete expired contract inclusive shards.
// He will not detect shards with a missing contract
// This script will delete the lost shards

// How to run it
// Requires nodejs LTS, git, python2
// Open command line and move to the folder where this script is stored
// Execute "npm install storj-lib"
// Execute "node Storj_Farmer_Delete_Lost_Shards.js"

var async = require('async');
var storj = require('storj-lib');
var stream = require('readable-stream');

var persistence = new storj.EmbeddedStorageAdapter('insert your storage location here');
var manager = new storj.StorageManager(persistence);

var StorageAdapter = require('storj-lib/lib/storage/adapter');
var StorageItem = require('storj-lib/lib/storage/item');

var rstream = manager._storage.createReadStream();
var list = [];
var buckets = [];

for(var i = 0; i <= 255; i++){
        buckets.push(i);
}


console.log('Reading contracts');

rstream.on('data', function(item) {
        rstream.pause();
        list.push(item.fskey || item.hash);
        rstream.resume();
});

rstream.on('end', function() {
        console.log('Reading contracts finished');
        async.forEachLimit(buckets, 1, function (i, callback){
                console.log('Reading bucket %s', i);
                persistence._fs.list(i, (err, keys) => {
                        if (keys) {
                                keys.forEach((result) => {
                                        if (list.indexOf(result.baseKey) == -1) {
                                                persistence._fs.unlink(result.baseKey, function(/* err */) {
                                                        console.log('Deleting lost shard: %s', result.baseKey);
                                                });
                                        }
                                });
                        }
                        persistence._fs._getSbucketAtIndex(i).close();
                        callback();
                });
        }, function(err) {
                console.log('flushing shards, some buckets will be inaccessible');
                manager._storage.flush(function(err) {
                        /* istanbul ignore if */
                        if (err) {
                                console.log('problem while flushing shards, %s', err.message);
                        }

                        console.log('flushing shards finished');
                        process.exit();
                });
        });
});

Storj_Farmer_Manuall_Reaper.js

// How to run it
// Requires nodejs LTS, git, python2
// Open command line and move to the folder where this script is stored
// Execute "npm install storj-lib"
// Execute "node Storj_Farmer_Manuall_Reaper.js"

var storj = require('storj-lib');

// insert your storage location here.
// on Windows you need double backslash. C:\\storjshare\\whatever
var persistence = new storj.EmbeddedStorageAdapter('insert your storage location here');
var manager = new storj.StorageManager(persistence);

var StorageAdapter = require('storj-lib/lib/storage/adapter');
var StorageItem = require('storj-lib/lib/storage/item');

var rstream = manager._storage.createReadStream();

var timestamp = Date.now();

rstream.on('data', function(item) {
  rstream.pause();

  var total = Object.keys(item.contracts).length;
  var endedOrIncomplete = 0;

  for (var nodeID in item.contracts) {
    var ended = item.contracts[nodeID].get('store_end') < timestamp;
    var incomplete = !item.contracts[nodeID].isComplete();

    if (ended || incomplete ) {
      endedOrIncomplete++;
    }
  }

  if (total === endedOrIncomplete) {
    console.log('destroying shard/contract for %s', item.hash);
    manager._storage.del(item.hash, function(/* err */) {
      rstream.resume();
    });
  } else {
    rstream.resume();
  }
});

rstream.on('end', function() {
  console.log('flushing shards, some buckets will be inaccessible');
  manager._storage.flush(function(err) {
    /* istanbul ignore if */
    if (err) {
      console.log('problem while flushing shards, %s', err.message);
    }

    console.log('flushing shards finished');
    process.exit();
  });
});