.gitignore

*.pyc
*.log
*~
.#*

numpy_safe_inplace.py

#!/usr/bin/env python
"""
numpy_safe_inplace.py SCRIPT.py [ARGS...]

Run a Python script with a monkeypatched Numpy that prints error
messages if trying to do unsafe calls to in-place binary ufuncs.
Messages are also logged to `numpy_safe_inplace.log`.

Note that the may_share_memory check by default doesn't report cases
where input and output arrays are exactly the same (typical
elementwise in-place operation).  Use --may-share-no-simple-depends to
report also those (noisy!).

"""

import sys
import traceback
import argparse
import numpy as np


_PREV_UFUNCS = None
_PREV_OPS = None
_PREV_BUFSIZE = None


_REPORT_MAY_SHARE = None

_np_equal = np.equal
_np_isnan = np.isnan
_np_logical_and = np.logical_and
_np_logical_not = np.logical_not

try:
    np.may_share_memory(1, 1, max_work=3)
    MAY_SHARE_HAS_MAX_WORK = True
except TypeError:
    MAY_SHARE_HAS_MAX_WORK = False


_MAY_SHARE_SIMPLE_DEPENDS = True


def _np_all(x):
    return _np_logical_and.reduce(x, axis=None)


def _np_may_share_memory(a, b):
    if isinstance(a, np.ndarray) and isinstance(b, np.ndarray) and _MAY_SHARE_SIMPLE_DEPENDS:
        a_if = a.__array_interface__
        b_if = b.__array_interface__

        # simple dependency analysis
        if (a_if['data'] == b_if['data'] and
            a_if['strides'] == b_if['strides'] and
            a_if['shape'] == b_if['shape'] and
            a_if['descr'] == b_if['descr']):
            return False

    if MAY_SHARE_HAS_MAX_WORK:
        return np.may_share_memory(a, b, max_work=_REPORT_MAY_SHARE)
    else:
        return np.may_share_memory(a, b)


stderr = sys.stderr
log_file = None


class UndefinedOperation(ValueError):
    pass


class MayShareMemory(ValueError):
    pass


class safe_ufunc(object):
    def __init__(self, ufunc):
        if ufunc.nin != 2 or ufunc.nout != 1 or type(ufunc) is not np.ufunc:
            raise ValueError("%r is not a binary ufunc" % (ufunc,))
        self._ufunc = ufunc
        self.nargs = 3
        self.nin = 2
        self.nout = 1
        self.ntypes = ufunc.ntypes
        self.identity = ufunc.identity
        self.signature = ufunc.signature
        self.types = ufunc.types

    def _cmp(self, r, r2):
        if isinstance(r, np.ma.MaskedArray) and issubclass(r.dtype.type, np.number):
            if isinstance(r.dtype.type, np.inexact):
                r = r.filled(np.nan)
            else:
                r = r.filled(-123)

        if isinstance(r2, np.ma.MaskedArray) and issubclass(r2.dtype.type, np.number):
            if isinstance(r.dtype.type, np.inexact):
                r2 = r2.filled(np.nan)
            else:
                r2 = r2.filled(-123)

        r = np.asarray(r)
        r2 = np.asarray(r2)

        if r.ndim == 0 and r2.ndim == 0:
            if issubclass(r.dtype.type, np.inexact) and issubclass(r2.dtype.type, np.inexact):
                if not ((_np_isnan(r) and _np_isnan(r2)) or _np_equal(r, r2)):
                    raise UndefinedOperation()
            else:
                if not _np_equal(r, r2):
                    raise UndefinedOperation()
        elif r.ndim > 0 and r2.ndim == r.ndim:
            if issubclass(r.dtype.type, np.inexact) and issubclass(r2.dtype.type, np.inexact):
                m = _np_logical_not(_np_isnan(r))
                if not _np_all(_np_equal(r[m], r2[m])) or not _np_all(_np_equal(_np_isnan(r), _np_isnan(r2))):
                    raise UndefinedOperation()
            else:
                if not _np_all(_np_equal(r, r2)):
                    raise UndefinedOperation()

    # Note: these methods are not allowed to raise exceptions, since
    # they are called from Numpy internals without error checking...

    def _copy(self, obj):
        if hasattr(obj, 'copy'):
            return obj.copy()
        else:
            return np.array(obj)

    def _wrap(self, func, a, kw, iout=None):
        a2 = a
        kw2 = dict(kw)
        outarg = None

        if 'out' in kw2:
            outarg = kw2['out']
            kw2['out'] = self._copy(outarg)
        elif iout is not None and iout < len(a):
            outarg = a[iout]
            a2 = a[:iout] + (self._copy(outarg),) + a[iout+1:]

        if outarg is not None:
            r2 = func(*a2, **kw2)

        r = func(*a, **kw)
        try:
            if outarg is not None:
                self._cmp(r, r2)
                if _REPORT_MAY_SHARE is not None:
                    if any(_np_may_share_memory(x, outarg) for x in a):
                        raise MayShareMemory()
        except BaseException as exc:
            self._report(a, outarg, exc, nstack=-3)

        return r

    def _report(self, a, outarg, exc, nstack):
        if isinstance(exc, (UndefinedOperation, MayShareMemory)):
            msg = "*" * 79
            msg += "\n%s in %r !\n" % (exc.__class__.__name__,
                                       self._ufunc.__name__)
            a = a + (outarg,)
            msg += "args = [\n"
            for j, arr in enumerate(a):
                if j < len(a) - 1:
                    if arr is outarg:
                        outarg = None
                elif outarg is None:
                    continue

                if hasattr(arr, '__array_interface__'):
                    msg += "    %r %r\n" % (type(arr), arr.__array_interface__)
                else:
                    msg += "    %r\n" % (type(arr),)
            msg += "]\n"
            msg += "".join(traceback.format_stack()[2:nstack])
            msg += "\n"
            stderr.write(msg)
            log_file.write(msg)
            stderr.flush()
            log_file.flush()
        else:
            msg = "*" * 79
            msg += "\ninternal failure in safe_ufunc\n"
            msg += traceback.format_exc()
            msg += "\n"
            stderr.write(msg)
            log_file.write(msg)
            stderr.flush()
            log_file.flush()

    def __call__(self, *a, **kw):
        return self._wrap(self._ufunc.__call__, a, kw, iout=2)

    def accumulate(self, *a, **kw):
        return self._wrap(self._ufunc.accumulate, a, kw, iout=3)

    def at(self, a, indices, b=None):
        if b is None:
            r = self._ufunc.at(a, indices)
        else:
            r = self._ufunc.at(a, indices, b)
            r2 = self._ufunc.at(a, indices, np.array(b, copy=True))
            try:
                self._cmp(r, r2)
                if _REPORT_MAY_SHARE is not None and _np_may_share_memory(a, b):
                    raise MayShareMemory()
            except BaseException as exc:
                self._report((a, b), None, exc, nstack=-2)
        return r

    def outer(self, a, b):
        return self._ufunc.outer(a, b)

    def reduce(self, *a, **kw):
        return self._wrap(self._ufunc.reduce, a, kw, iout=3)

    def reduceat(self, *a, **kw):
        return self._wrap(self._ufunc.reduceat, a, kw, iout=4)

    def __hash__(self):
        return hash(self._ufunc)


def install():
    global _PREV_UFUNCS, _PREV_OPS, _PREV_BUFSIZE

    if _PREV_UFUNCS is not None or _PREV_OPS is not None or _PREV_BUFSIZE is not None:
        uninstall()

    ufuncs = [(x, getattr(np, x)) for x in dir(np)
              if isinstance(getattr(np, x), np.ufunc)]
    ops = np.set_numeric_ops()
    _PREV_OPS = dict(ops)

    for name, func in list(ops.items()):
        if func.nin == 2 and func.nout == 1:
            ops[name] = safe_ufunc(func)
    np.set_numeric_ops(**ops)

    _PREV_UFUNCS = {}
    for name, func in ufuncs:
        if func.nin == 2 and func.nout == 1:
            _PREV_UFUNCS[name] = getattr(np, name)
            setattr(np, name, safe_ufunc(func))

    _PREV_BUFSIZE = np.getbufsize()
    np.setbufsize(16)


def uninstall():
    global _PREV_OPS, _PREV_UFUNCS, _PREV_BUFSIZE

    if _PREV_OPS is not None:
        np.set_numeric_ops(**_PREV_OPS)
        _PREV_OPS = None

    if _PREV_UFUNCS is not None:
        for name, func in _PREV_UFUNCS.items():
            setattr(np, name, func)
        _PREV_UFUNCS = None

    if _PREV_BUFSIZE is not None:
        np.setbufsize(_PREV_BUFSIZE)
        _PREV_BUFSIZE = None


def main(argv=None):
    global log_file, _REPORT_MAY_SHARE, _MAY_SHARE_SIMPLE_DEPENDS

    p = argparse.ArgumentParser(usage=__doc__.strip())
    p.add_argument('--may-share', action="store", type=int, default=None,
                   help="report ufuncs on arrays sharing memory, with given max_work")
    p.add_argument('--may-share-no-simple-depends', action="store_true",
                   help="report may-share even if operation is safe according to simple dependency analysis")
    p.add_argument('script', help="script to execute", metavar='SCRIPT.py')
    p.add_argument('args', nargs=argparse.REMAINDER, metavar='ARGS')
    args = p.parse_args(argv)

    if args.may_share is not None:
        if not MAY_SHARE_HAS_MAX_WORK:
            print("*"*79)
            print("WARNING: may_share_memory does not have max_work= argument")
            print("*"*79)
        _REPORT_MAY_SHARE = int(args.may_share)

    _MAY_SHARE_SIMPLE_DEPENDS = not args.may_share_no_simple_depends

    install()

    with open('numpy_safe_inplace.log', 'w') as log:
        log_file = log
        sys.argv = [args.script] + args.args
        sys.modules['numpy_safe_inplace'] = sys.modules[__name__]
        execfile(args.script, {}, {})


if __name__ == "__main__":
    main()

run_astropy_test.py

import astropy
import numpy as np

# astropy.units doesn't import with replaced ufuncs...
import numpy_safe_inplace
numpy_safe_inplace.uninstall()
import astropy.units
import astropy.units.quantity_helper
numpy_safe_inplace.install()

for k, v in list(astropy.units.quantity_helper.UFUNC_HELPERS.items()):
    if isinstance(k, np.ufunc) and k.__name__ in dir(np):
        uf = getattr(np, k.__name__)
        astropy.units.quantity_helper.UFUNC_HELPERS[uf] = v

for k in list(astropy.units.quantity_helper.UNSUPPORTED_UFUNCS):
    if isinstance(k, np.ufunc) and k.__name__ in dir(np):
        uf = getattr(np, k.__name__)
        astropy.units.quantity_helper.UNSUPPORTED_UFUNCS.add(uf)

# many units-related tests fail due to the replaced ufuncs
astropy.test()

run_numpy_test.py

import numpy
numpy.test(verbose=2)

run_pandas_test.py

import nose
nose.main(argv=['xxx', 'pandas', '--verbose'])

run_scipy_test.py

import scipy
scipy.test(verbose=2)

run_simple_test.py

import numpy as np

x = np.zeros([5,5])
x += x.T

x = np.random.rand(5,5)
x += x.T

x[:,0] += x[:,1]

np.add.at(x, [0], x[:1,:1])

run_sklearn_test.py

import nose
nose.main(argv=['xxx', 'sklearn', '--exe', '--verbose'])