gistfile1.py

from collections import Counter, defaultdict

class ConfusionMatrix(object):
    def __init__(self, trueLabels, clusterLabels):
        self._counts = defaultdict(Counter)
        self.allLabels = set(trueLabels + clusterLabels)
        for trueLabel, clusterLabel in zip(trueLabels, clusterLabels):
            self._counts[trueLabel][clusterLabel] += 1
        
    def __getitem__(self, item):
        return self._counts[item[0]][item[1]]
        
    def __setitem__(self, item, value):
        self._counts[item[0]][item[1]] = value

Initialization is

>>> x = ConfusionMatrix([0, 0, 1, 1], [1, 1, 1, 2])

With access like:

>>> x[0, 1]
2

>>> x[4, 3] += 8
>>> x[4, 3]
8

and all labels are available:

>>> x.allLabels
{0, 1, 2}