indexReduction.js

//

var fs = require('fs');

var matches = {};
var input = fs.readFileSync('/Users/braydonbatungbacal/Desktop/big.txt').toString();
var validIndexes = [];
var length = 2; // the start length

for (var i = 0; i < input.length; i++) {
    validIndexes.push(i);
}

while (validIndexes.length > 0) {
    var newValdIndexes = [];
    var currentLengthMatches = {}; // prevents memory build up in matches as it's freed per length increment.
    var delayedIndexes = {};

    for (var i = 0; i < validIndexes.length; i++) {
        var match = input.substr(validIndexes[i], length);

        if (match.length !== length) {
            continue; // edge case for substr that can't satisfy length near end of input.
        }

        if (currentLengthMatches[match]) {
            if (delayedIndexes[match] !== undefined) {
                newValdIndexes.push(delayedIndexes[match]);
                delete delayedIndexes[match];
            }

            newValdIndexes.push(validIndexes[i]);
            matches[match] = (matches[match]) ? matches[match] + 1 : 2; // first match is else, starting here is 2nd.
        } else {
            delayedIndexes[match] = validIndexes[i];
            currentLengthMatches[match] = true; // not pushing first index
        }
    }

    validIndexes = newValdIndexes;

    length *= 2;

    console.log(`LENGTH IS ${length - 1}`);
    console.log(`REMAINING INDEXES ${validIndexes.length}`);
    console.log('--------');
}

fs.writeFileSync('/Users/braydonbatungbacal/Desktop/output.json', JSON.stringify(matches));

console.log(`Final match on length ${length - 1}`);
console.log(matches);
console.timeEnd('algo');