adagrad.py

import numpy as np

epsilon = 1e-8
g2 = 0

for g in [1, 10, 100]:
    for t in range(10):
        g2 += g ** 2
        dx = -5e-4 / np.sqrt(g2 + epsilon) * g
        print(t, dx)

# Result
# 0 -0.0004999999975
# 1 -0.000353553389709
# 2 -0.000288675134114
# 3 -0.000249999999688
# 4 -0.000223606797526
# 5 -0.000204124145062
# 6 -0.00018898223637
# 7 -0.000176776695186
# 8 -0.000166666666574
# 9 -0.000158113882929
# 0 -0.000476731294601
# 1 -0.000345032779663
# 2 -0.000283980917119
# 3 -0.000246932399159
# 4 -0.000221403721383
# 5 -0.000202444082543
# 6 -0.000187646656259
# 7 -0.00017568209223
# 8 -0.000165748386032
# 9 -0.000157329193881
# 0 -0.000476514746215
# 1 -0.000344950658337
# 2 -0.000283935124828
# 3 -0.000246902290963
# 4 -0.000221382018329
# 5 -0.000202427490807
# 6 -0.000187633443103
# 7 -0.000175671248662
# 8 -0.000165739279729
# 9 -0.000157321405886