First softmax function in python

soft_max.py

#SOFTMAX FUNCTION

import matplotlib.pyplot as plt
import numpy as np
import scipy as sp
import os
from sklearn.linear_model import LogisticRegression
from six.moves import cPickle as pickle
from sklearn.datasets import load_iris
%matplotlib inline

#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>data treatment(begining)

iris = load_iris()


def normalize(Xdata):
    l=[]
    for i in range(len(Xdata.T)):
        l.append(np.array([(j - (np.sum(Xdata.T[i]))/len(Xdata.T[i]))/\
        	(max(Xdata.T[i]) - min(Xdata.T[i])) for j in Xdata.T[i]]))
    return np.array(l).T    
    
    
def randomize(dataset, labels):
  permutation = np.random.permutation(labels.shape[0])
  shuffled_dataset = dataset[permutation]
  shuffled_labels = labels[permutation]
  print(permutation.shape, len(permutation))
  return shuffled_dataset, shuffled_labels


def randomize_in_place(list1,list2, init):
    np.random.seed(seed=init)
    np.random.shuffle(list1)
    np.random.seed(seed=init)
    np.random.shuffle(list2)
        
qX = normalize(iris['data'])
allX = np.array(np.insert(qX, 0, 1, axis=1))
ally = iris['target']
randomize_in_place(allX, ally,0)

X = allX[:131]
y = ally[:131]
tX = allX[-20:]
ty = ally[-20:]

#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>data treatment(end)

def ind(var_y,var_k):
    if var_y == var_k:
        return 1
    else:
        return 0

par1 = np.random.random_sample((5,))
par2 = np.random.random_sample((5,))
par3 = np.random.random_sample((5,))

tes_theta1 =np.array([par1,par2,par3])
tes_theta2 =np.array([par1,par2,par3])
tes_theta3 =np.array([par1,par2,par3])

theta_z = np.array([par1*0]*3)

def hyp(theta,x):
    total = np.sum([np.exp(theta[j].dot(x)) for j in range(len(theta))])
    vec_aux = np.array([np.exp(theta[j].dot(x)) for j in range(len(theta))])
    return total*vec_aux


def P_cost_aux(var_theta,var_x,vk):
    return np.exp(var_theta[vk].dot(var_x))/np.sum([np.exp(var_theta[j].dot(var_x))\
     for j in range(len(var_theta))])

def lis_K_cost_aux(var_theta,var_x,var_y):
    return np.sum([ind(var_y,k)*np.log(P_cost_aux(var_theta,var_x,k))\
     for k in range(len(var_theta))])

def cost(theta,vX,vy):
    return - np.sum([lis_K_cost_aux(theta,vX[i],vy[i])\
     for i in range(len(vy))]) 



def del_cost_del_j(j,theta,vX,vy):
    return - np.sum([vX[i]*(ind(vy[i],j) - P_cost_aux(theta,vX[i],j))\
     for i in range(len(vy))])
    
def gra_fixed(alpha,times,theta,vX,vy):
    init = theta
    graf = []
    for i in range(times):
        for j in range(len(theta)):
            init[j] = init[j] - alpha*del_cost_del_j(j,theta,vX,vy)
        theta = init
        graf.append(cost(theta,vX,vy))
    
    plt.plot(range(times),graf)
    plt.show()
    return theta

def gra_mut1(alpha,times,theta,X,y):
    init = theta
    graf = []
    for i in range(times):
        alpha = alpha*0.66
        for j in range(len(theta)):
            init[j] = init[j] - alpha*del_cost_del_j(j,theta,X,y)
        theta = init
        graf.append(cost(theta,X,y))
    
    plt.plot(range(times),graf)
    plt.show()
    return theta


def gra_mut2(alpha,times,theta,X,y):
    init = theta
    graf = []
    for i in range(times):
        alpha = alpha*0.5
        for j in range(len(theta)):
            init[j] = init[j] - alpha*del_cost_del_j(j,theta,X,y)
        theta = init
        graf.append(cost(theta,X,y))
    
    plt.plot(range(times),graf)
    plt.show()
    return theta


print(cost(tes_theta1,X,y))
gra_fixed(0.01,300,tes_theta1,X,y)
print(cost(tes_theta1,X,y))

print("space")
print(cost(tes_theta2,X,y))
gra_mut1(0.5,300,tes_theta2,X,y)
print(cost(tes_theta2,X,y))

print("space")
print(cost(tes_theta3,X,y))
gra_mut2(0.3,300,tes_theta3,X,y)
print(cost(tes_theta3,X,y))