lodash: automated tester for documentation examples

README

This is intended to be run in Node.  Provided a path to the lodash source file as a command-line argument.

There were a few edge case handlers that I removed; they only helped in a few places while severely reducing code clarity.  There's also some weirdness with how it interprets literals that cause them to not match (see _.escapeRegExp), but it's not worth the time to fix.

Let me know.

check_examples.js

#!/usr/bin/env node

/*
 *  much of this was taken from:
 *  https://bitbucket.org/ariya/missing-doc/src/master/missing-doc.js
 */

'use strict';

var _ = require('lodash'),
    fs = require('fs'),
    esprima = require('esprima'),
    estraverse = require('estraverse'),
    doctrine = require('doctrine'),
    vm = require('vm'),
    indentString = require('indent-string'),
    jsdom = require('jsdom').jsdom; // requires 3.x version to work in node



/////////////////////////////
////  SETUP & CONSTANTS  ////
/////////////////////////////
var reResultComment = /^\/\/ => (.*?)(?: \(iteration order is not guaranteed\))?$/;
var reObjectLiteral = /^\s*{.*}\s*$/;
// base context for running sandbox VMs; most of this exists to prevent silly errors
var globalContext = {
    _: _,
    document: jsdom(),
    Uint8Array: Uint8Array,
    require: require,
    console: {
        log: _.identity // this is cheating
    },
    asyncSave: _.noop,
    mage: {
        castSpell: _.noop
    }
};
_.times(20, function () {
    var el = globalContext.document.createElement('div');
    globalContext.document.body.appendChild(el);
});
_.times(103, _.uniqueId);



////////////////////////
////  MAIN ROUTINE  ////
////////////////////////
// globals because I'm lazy
var total = 0,
    successes = 0;

// lodash source file as command-line argument
buildAndWalkTree(process.argv[2]);

console.log('\n' + successes + ' out of ' + total + ' examples succeeeded.')



///////////////////
////  WORKERS  ////
///////////////////
function buildAndWalkTree (filename) {
    var content, tree;
    try {
        content = fs.readFileSync(filename, 'utf-8');
        tree = esprima.parse(content, { attachComment: true, loc: true });
        // walk the tree
        estraverse.traverse(tree, { enter: workNode });
    } catch (e) {
        console.error(e.toString());
        process.exit(1);
    }
}

function workNode (node) {
    if (node.type === 'Identifier') {
        // just a duplicate from something else
        return;
    }
    if (node.leadingComments && node.id && node.id.leadingComments) {
        // this should never happen, but it's worth getting a warning if it does
        console.error(node.id.name + ' has leadingComments in two places in the AST');
        process.exit(1);
    }

    var leadingComments = node.leadingComments || (node.id && node.id.leadingComments);
    var name = node.name || (node.id && node.id.name);

    _.forEach(leadingComments, _.partial(workComment, name));
}

function workComment (name, comment) {
    // unwrap: pulls out the leading `*`s; recoverable: allows for badly-formed JSDoc
    var data = doctrine.parse(comment.value, { unwrap: true, recoverable: true });
    // some functions have @name annotations that should take precedence
    name = _.result(_.find(data.tags, { title: 'name' }), 'name') || name;
    // pull out all of the examples
    var examples = _.where(data.tags, { title: 'example' });

    // shouldn't be more than one examples 
    _.forEach(examples, function (example) {
        // do the work
        var errors = testExample(example, name);

        if (errors.length) {
            console.log(name + '\n' + indentString(errors.join('\n'), ' ', 4));
        } else {
            ++successes;
        }
        ++total;
    });
}

// executes the example code, comparing with expected output, returning array of errors found
function testExample (example) {
    var errors = [];
    var lines = example.description.split('\n');

    var sandbox = vm.createContext(globalContext);
    // to prevent issue with ASI and other stupidity, we build up a chunk of code to execute
    var lineBuffer = [];

    for (var i = 0; i < lines.length; ++i) {
        var line = lines[i];
        var match = line.match(reResultComment);
        if (match) { // output comment line
            var expectation = match[1], // what the comment says we should get at this point
                code = lineBuffer.join('\n'),
                result,
                expectedResult;
            lineBuffer = []; // clear it for the next run

            // eval code and check for error
            try {
                var result = evalCode(code, sandbox);
            } catch (e) {
                errors.push('ERROR EXECUTING CODE: ' + e.toString() + '\n' +
                    indentString(code.trimLeft(), ' ', 4));
                continue;
            }

            // eval expectation and check for error
            try {
                var expectedResult = evalCode(expectation);
            } catch (e) {
                errors.push('ERROR PARSING EXPECTED RESULT:\n' +
                    indentString(expectation, ' ', 4));
                continue;
            }

            if (! _.isEqual(result, expectedResult)) {
                errors.push('EXPECTED: ' + expectation + '\n  RESULT: ' + result);
             }
        } else { // regular line of code
            // we can't eval each line independently due to mutli-line expressions
            lineBuffer.push(line);
            // some lines may not execute if there are no expectation comments that follow
        }
    }
    return errors;
}

// may throw, which is sort of the point
function evalCode (code, sandbox) {
    // create a bare context if necessary
    sandbox = sandbox || vm.createContext();
    // fix object literals being mis-interpreted as code blocks
    if (code.match(reObjectLiteral)) {
        code = '(' + code + ')';
    }
    return vm.runInContext(code, sandbox);
}

package.json

{
  "dependencies": {
    "lodash": "~3.5.0",
    "estraverse": "~3.1.0",
    "esprima": "~2.1.0",
    "doctrine": "~0.6.4",
    "indent-string": "~1.2.1",
    "jsdom": "~3.1.2"
  }
}