rmsprop.py

import numpy as np

epsilon = 1e-8
g2 = 0

for g in [1, 10, 100]:
    for t in range(10):
        g2 = 0.95 * g2 + 0.05 * g ** 2
        dx = -1e-4 / np.sqrt(g2 + epsilon) * g
        print(t, dx)

# Result
# 0 -0.000447213550779
# 1 -0.000320256291187
# 2 -0.000264790351464
# 3 -0.000232185635364
# 4 -0.000210249703143
# 5 -0.00019429085759
# 6 -0.000182070331555
# 7 -0.000172367792128
# 8 -0.000164454422172
# 9 -0.000157864836382
# 0 -0.000431082543416
# 1 -0.000314469452702
# 2 -0.000261653428025
# 3 -0.000230166780322
# 4 -0.000208821535075
# 5 -0.000193218133417
# 6 -0.000181230547534
# 7 -0.000171690169511
# 8 -0.000163894889111
# 9 -0.000157394352457
# 0 -0.000430991153107
# 1 -0.00031443574369
# 2 -0.000261634980533
# 3 -0.000230154850839
# 4 -0.000208813071567
# 5 -0.000193211764046
# 6 -0.000181225554389
# 7 -0.00017168613638
# 8 -0.000163891556168
# 9 -0.000157391548159