A simple MNIST classifer AND autoencoder in one

tensorflow_autoencoder.py

"""A simple MNIST classifer AND autoencoder in one
"""

# Import data
import input_data
mnist = input_data.read_data_sets("/data/mnist/", one_hot=True)

import tensorflow as tf
sess = tf.InteractiveSession()

# Create the model
x = tf.placeholder("float", [None, 784])
# l_rate = tf.placeholder("float", [1])# 0.01
l_rate = tf.Variable(0.001)
W1 = tf.Variable(tf.zeros([784,400]))
b1 = tf.Variable(tf.zeros([400]))
W2 = tf.Variable(tf.ones([400,20]))
b2 = tf.Variable(tf.zeros([20]))
# b2t = tf.Variable(tf.zeros([20]))
h= tf.nn.tanh( tf.matmul(x,W1)+b1) #
h=tf.nn.dropout(h) # THAT!
_y = tf.matmul(h,W2) + b2 # 10 Numbers + 10 'styles'
print("_y ",tf.rank(_y))
y = tf.nn.softmax(tf.slice(_y,[0,0],[-1,10])) #  softmax of all batches(-1) only on the numbers(10)
e1= tf.matmul(_y,tf.transpose(W2)) #+b2
e1=tf.nn.tanh(e1)
x_=x_reconstructed= tf.nn.sigmoid(tf.matmul(e1,tf.transpose(W1)))

# Define loss and optimizer
y_ = tf.placeholder("float", [None,10])
mnist_entropy = -tf.reduce_sum(y_*tf.log(y))
# encod_entropy = -tf.reduce_sum(x_*tf.log(x))
encod_entropy = tf.sqrt(tf.reduce_mean(tf.square(x - x_)))
cross_entropy = encod_entropy * mnist_entropy
# cross_entropy = -tf.reduce_sum(y_*tf.log(y))
train_step = tf.train.AdamOptimizer(learning_rate=l_rate).minimize(cross_entropy)
pretrainer = tf.train.AdamOptimizer(learning_rate=l_rate).minimize(mnist_entropy)
# encod_step = tf.train.AdamOptimizer(learning_rate=l_rate).minimize(encod_entropy)
accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(y,1), tf.argmax(y_,1)), "float"))


import sys

def eval(feed):
    print("it %d"%i, end=' ')
    print("cross_entropy ",cross_entropy.eval(feed), end=' ')
    # print("encod_entropy ",encod_entropy.eval(feed), end=' ')
    print("overal accuracy ",accuracy.eval({x: mnist.test.images, y_: mnist.test.labels}))#, end='\r'WWWWAAA)


# Train
tf.initialize_all_variables().run()
for i in range(1000):#pretrain
  batch_xs, batch_ys = mnist.train.next_batch(100)
  feed = {x: batch_xs, y_: batch_ys}
  pretrainer.run(feed)
  if(i%100==0):eval(feed)

for i in range(100000):
  batch_xs, batch_ys = mnist.train.next_batch(100)
  feed = {x: batch_xs, y_: batch_ys}
  # if((i+1)%9000==0):sess.run(tf.assign(l_rate,l_rate*0.3))
  train_step.run(feed)
  if(i%100==0):eval(feed)
  # encod_step.run(feed) # alternating!