Mutation testing: Hacking python ASTs for fun and profit

mutate.py

import ast, _ast, copy

class Mutator(ast.NodeTransformer):
  MUTATIONS = [
  # (FromOp, ToOp)
    (_ast.Or, _ast.And),
    (_ast.And, _ast.Or)
  ]

  def mutate(self, root, mutation):
    self._changed = False
    self._mutation = mutation

    mutant = self.visit(root)
    return (mutant, self._changed)

  def visit_BoolOp(self, node):
    (FromOp, ToOp) = self._mutation
    if(type(node.op) == FromOp):
      self._changed = True
      node.op = ToOp()

    return node

# returns all mutants for given ast
def mutate(ast):
  mutants = []
  mutator = Mutator()

  for mutation in Mutator.MUTATIONS:
    (mutant, mutated) = mutator.mutate(copy.deepcopy(ast), mutation)
    if mutated:
      mutants.append(mutant)

  return mutants

foobar.py

class FooBar:

  @classmethod
  def baz(cls, x):
    if (x % 3 == 0) or (x % 5 == 0):
      return "foo"
    elif x % 2 == 0:
      return "bar"
    else:
      return "baz"

foobar_test.py

import unittest
import ast, astunparse, inspect
from foobar import FooBar
from mutate import mutate

class FooBarTest(unittest.TestCase):

  # missing case
  def test_foo(self):
    # self.assertEquals(FooBar.baz(9), 'foo')
    pass

  def test_bar(self):
    self.assertEquals(FooBar.baz(2), 'bar')

  def test_baz(self):
    self.assertEquals(FooBar.baz(7), 'baz')


def run_unittest_without_output():
  suite = unittest.TestSuite()
  for test in ['test_foo', 'test_bar', 'test_baz']:
    suite.addTest(FooBarTest(test))
  result = unittest.TestResult()
  suite.run(result)
  return result


def unittest_with_mutant(mutant):
  code = compile(mutant, '<string>', 'exec')
  # don't do that at home
  exec code in globals()
  return run_unittest_without_output()


def unittest_with_mutants(mutants):
  print "Running mutation tests..."
  passedWithMutant = False
  for mutant in mutants:
    result = unittest_with_mutant(mutant)
    if result.wasSuccessful():
      # mutant passed => failed
      passedWithMutant = True
      source = astunparse.unparse(mutant)
      print "FAILED - Everything passed with the following mutant: %s" % source

  if not passedWithMutant:
    print "OK - Tested with %i mutant(s), everything covered" % len(mutants)


if __name__ == '__main__':
  # Test without mutations
  unittest.main(exit=False)

  # Test with mutations
  source_ast = ast.parse(inspect.getsource(FooBar))
  mutants = mutate(source_ast)
  unittest_with_mutants(mutants)

stdout

...
----------------------------------------------------------------------
Ran 3 tests in 0.001s

OK

Running mutation tests...
FAILED - Everything passed with the following mutant:

class FooBar:

    @classmethod
    def baz(cls, x):
        if (((x % 3) == 0) and ((x % 5) == 0)):
            return 'foo'
        elif ((x % 2) == 0):
            return 'bar'
        else:
            return 'baz'