JavaScript testing framework

test.js

/**
 * This is a proof of concept for a JS testing API.
 * 
 * The eventual goal is to unify and replace the testing API across all of Firefox's
 * testing frameworks (xpcshell, mochitest, etc).
 * 
 * How it Works
 * ===========
 * Tests are declared in JavaScript files which are loaded by a test harness dependent
 * mechanism. Traditionally, the set of files is declared in a manifest somewhere and
 * the test harness iterates through the test files and loads and executes them.
 * Tests are declared in files by calling the globally-available "test" function.
 * The "test" function receives a single argument, an object containing test metadata.
 * The most important piece of data on this object are the individual test functions
 * themselves.
 */

// Here is a simple test definition. This is what I want people to focus on. See how
// easy it is to declare tests!
"use strict";

test({
  testSimpleMath: function () {
    this.assertEqual(2 + 2, 4, "2 + 2 = 4");
  },
  
  testTrue: function () {
    this.assertTrue(true, "true is true.")
  },
  
  testAsync: function () {
    this.waitForFinish = true;
  
    function onResult() {
      this.finished();
    }
  
    doSomethingAsyncThatCallsACallback(onResult);
  },
});

// This is another group of tests. This one has a tearDown function which will be called
// after every test!
test({
  setUp: function () {
    this.docs = [];
  }
  
  tearDown: function () {
    for (let doc of this.docs) {
      doc.close();
    }
  }
  
  testOpenWindow: function () {
    // I'm not sure if this is valid DOM APIs or what! Shouldn't matter. You get the idea.
    let doc = window.open("about:blank");
    this.docs.push(doc);
    
    self.assertNotNull(doc);
  },
});

// We can also use Task.jsm and promises to make writing async tests easy!
test({
  testTask: function () {
    let uri = "http://www.mozilla.org";
    
    // fetchURI() issues an HTTP request and returns a promise that will be resolved
    // once the HTTP response has come back (many milliseconds and ticks later).
    let response = yield fetchURI(uri);
    
    self.assertEqual(response.status, 200, "200 status code.");
    
    // Do it again. Generator magic means no crazy indenting or callback spaghetti.
    // This is the benefit of promises!
    let response2 = yield fetchURI(uri);
    
    // We just fall off the end of the generator and we are done. No need to call a
    // function or anything.
  },
})
 
/**
 * Our base class for all tests.
 * 
 * The object passed into test() will eventually be chained up to this (that's
 * where the magic "this.assertEqual" etc functions come from.)
 */
let BaseTest = {
    assertEqual: function (actual, expected, reason) {
      if (action != expected) {
        throw new AssertionFailure(...);
      } else {
        // Record success, etc.
      }
    },
    
    // These are guaranteed to be called at test startup and shutdown, respectively.
    // They can be used as init and cleanup functions. There are only called once per
    // test object.
    startup: function () { },
    shutdown: function () { },
};

/*
 * Here is what our test harness would look like. It started off as pseudocode but
 * I made it look like actual working JS. Don't be fooled, though! The code doesn't
 * handle asynchronous tests very well. This is intentional because I was striving
 * for code readability. The important aspect I want people looking at is the overall
 * design, specifically from the individual test perspective. Most people will not
 * bother to look at the test harness itself. Individual test files are where the
 * attention needs to be.
 */

for (let testFile of testFiles) {
  let testDefinitions = [];
  // Create a new execution context for our test file.
  let sandbox = new Sandbox(testFile);
  // Have test() simply accumulate the arguments passed to it.
  sandbox.test = testObjects.push.bind(testObjects);
    
  // Parse the test file.
  try {
    sandbox.eval();
  } catch (ex) {
    // ...
    continue;
  }
    
  // Now loop over all the declared test objects:
  for (let testDefinition of testDefinitions) {
    if (typeof test != "Object") {
      // ...
      continue;
    }
        
    // We take the object passed in and dynamically create a new type based on it.
    // We require all test objects to eventually chain up to a known base test prototype
    // which declares common testing primitives such as assertEqual(). There are a
    // number of ways to do this in JS. We can bikeshed over exact in the actual
    // implementation.
        
    let testProto = testDefinition; // Aliased for readability.
    if (!("__proto__" in testProto)) {
      testProto.__proto__ = BaseTest;
    }
    
    // Prevent changes. This is so we can instantiate the test multiple times with
    // less fear of it mucking with internal test state.
    Object.freeze(testProto);
    
    // Now instantiate a new test instance based on the prototype.
    let test = Object.create(testProto);
    
    // If we wanted to influence test behavior from the presence of special properties
    // in the test object, here is where we could do it.
    
    try {
      test.startup.call(test);
        
      // Iterate over the properties of the test. If it begins with "test" and is a function,
      // it's an individual test function! We call it.
      for (let prop in test) {
        if (!prop.startsWith("test")) {
          continue;
        }
              
        let fn = test[prop];
        if (typeof fn != "function") {
          throw new Error("Tests cannot have properties beginning with 'test' that aren't functions!");
        }
              
        // Tests can define set up and tear down functions that get called on every test function
        // run. This facilitates code reuse and elegant code for cleanup tasks.
        try {
          test.setUp();
          let result = fn.call(test);
                
          // If the result is a generator, we assume this is not a simple synchronous
          // function but instead a Task.jsm-based test with promises.
          if (result && typeof result.send == "function") {
            // Do generator magic here.
          }
                
          // If the test marked itself as an async test by setting a property on itself,
          // we wait around for it to be called before proceeding.
          if (test.waitForFinish) {
            // ...
          }
        } catch (AssertionFailure) {
          // Report error.
            
        } finally {
          // Be sure to always run tear down tasks!
          try {
            test.tearDown();
          } catch (ex) { ... }
        }
      }
    }
  } finally {
    test.shutdown();
  }
}

/**
 * Misc Notes
 * 
 * Astute readers will note we effectively perform two passes through the test file: 1 pass
 * to collect all the tests and another to actually run them. While the code isn't all there,
 * a robust test runner would count all tests on the first pass before execution. That way, if
 * the test fails, we'll know exactly which tests failed to run. Contrast this with the existing
 * test suites where we mainly have file-level visibility into test results. I want to go deeper.
 * 
 * Nothing here is set in stone. I'm just throwing something out there.
 * 
 * There are nearly infinite variations on this implementation. But, I will strongly argue that we want
 * test functions to inherit test/assertion functions through prototype inheritance instead of from
 * globals, which is how we do it now. I believe this will foster better testing because we'll more
 * easily be able to add new functions into inheritance chains, have setup and teardown functions,
 * etc.
 * 
 * Something completely missing from this code is representation of test results. I would love
 * for this to be abstracted in such a way such that the results of tests are captured properly
 * instead of merely by printing an OK message. The end goal is that tools should be able to
 * switch output depending on the context. e.g. build bot machines should output machine readable
 * files. Tools like mach can pretty-print. etc.
 */