cross-entropy.py

import numpy as np

def crossEntropy(Y, P):
    Y = np.float_(Y)
    P = np.float_(P)
    CE = -np.sum(Y*np.log(P) + (1-Y)*np.log(1-P))
    return CE


#cross entropy tells us when two vectors are similar or different. 
#To compute the cross entropy simply calculate the negative of the natural logarithm of the product of probabilities.
#That is, given probabilities p1, p2, p3 then the cross entropy is -ln(p1*p2*p3) = -ln(p1)-ln(p2)-ln(p3)
#example: write a function that takes two lists and return the XEntropy

softmax.py

import numpy as np

def softmax(L):

    exp_f = np.exp(L)
    sum_exp = sum(exp_f)
    result = []
    for i in exp_f:
        result.append(i*1.0/sum_exp)
    return result
#softmax function replaces the sigmoid function when dealing with 3 or more classes