sigmoid.py

"""Optimizer to find the closest sigmoidal function to the identity."""

import matplotlib.pyplot as plt
import numpy as np
import tensorflow as tf


LEARNING_RATE = 1e-3  # make this small to make optimization converge.
BATCH_SIZE = 10000  # make this big to make optimization more accurate.
STEPS = 1000  # increase this if the learning rate is too small.

# This affects the end result a lot.
STDDEV = .5

sampled_points = tf.random_normal([BATCH_SIZE], mean=.5, stddev=STDDEV)

b = tf.Variable(1., name='b')
a = tf.exp(b / 2)  # analytic solution to make 0.5 a fixed point.

sigmoided = tf.reciprocal(1 + a * tf.exp(-b * sampled_points))
loss = tf.nn.l2_loss(sigmoided - sampled_points)

train_op = tf.train.GradientDescentOptimizer(LEARNING_RATE).minimize(loss)

with tf.Session() as sess:
    sess.run(tf.initializers.global_variables())

    step = 0
    for _ in range(1000):
        sess.run(train_op)

    b = b.eval()
    a = np.exp(b / 2)


x = np.arange(-1., 2., .005)
y1 = 1 / (1 + a * np.exp(-b * x))
y2 = x

fig, ax = plt.subplots()
ax.plot(x, y1)
ax.plot(x, y2)
title = "Optimized sigmoid vs. identity (b={:.4f})".format(b)
ax.set(xlabel="x", ylabel="y", title=title)
ax.grid()
plt.show()