basic function from information theory

infotheory.py

import numpy as np
import random
import unittest
from math import isnan


def softmax(x):
    """
    Compute the softmax function for each row of the input x.
    It is crucial that this function is optimized for speed because
    it will be used frequently in later code.
    You might find numpy functions np.exp, np.sum, np.reshape,
    np.max, and numpy broadcasting useful for this task. (numpy
    broadcasting documentation:
    http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
    You should also make sure that your code works for one
    dimensional inputs (treat the vector as a row), you might find
    it helpful for your later problems.
    You must implement the optimization in problem 1(a) of the
    written assignment!
    """

    # ## YOUR CODE HERE

    # when x is an one dimensional input
    # we transform it in an array having
    # x as its only member
    if type(x[0]) != np.ndarray:
        x = x.reshape((1, len(x)))
    all_constants = - np.amax(x, axis=1)
    x = x+all_constants[:, np.newaxis]
    x = np.exp(x)
    all_sums = np.sum(x, 1)
    all_sums = np.power(all_sums, -1)
    y = x*all_sums[:, np.newaxis]
    # ## END YOUR CODE
    return y

def safe_x_log_1_over_x(x):
    if x == 0:
        return 0
    else:
        return x*np.log2(1/x)

def safe_log(x):
    if x == 0:
        return 0
    else:
        return np.log2(x)
    
def safe_array_log(a):
    return np.array([safe_log(x) for x in a])
    
def safe_array_x_log_1_over_x(a):
    return np.array([safe_x_log_1_over_x(x) for x in a])

def H(p):
    return np.sum(safe_array_x_log_1_over_x(p))

def Dkl(p,q):
    return np.sum(p*safe_array_log(p/q))

def CE(p,q):
    return H(p) + Dkl(p,q)