adadelta.py

import numpy as np

epsilon = 1e-8
dx2 = 0
g2 = 0

for g in [1, 10, 100]:
    for t in range(10):
        g2 = 0.95 * g2 + 0.05 * g ** 2
        dx = -np.sqrt(dx2 + epsilon) / np.sqrt(g2 + epsilon) * g
        dx2 = 0.95 * dx2 + 0.05 * dx ** 2
        print(t, dx)

# Result
# 0 -0.000447213550779
# 1 -0.000452910767786
# 2 -0.000456764555727
# 3 -0.000459705739871
# 4 -0.00046210346643
# 5 -0.000464141207124
# 6 -0.000465923319952
# 7 -0.000467514783232
# 8 -0.000468958777606
# 9 -0.000470285445393
# 0 -0.00133474040553
# 1 -0.00133659137432
# 2 -0.0013378713046
# 3 -0.00133886088335
# 4 -0.00133967488093
# 5 -0.0013403713847
# 6 -0.00134098384571
# 7 -0.00134153328328
# 8 -0.00134203376517
# 9 -0.00134249516794
# 0 -0.00381070820524
# 1 -0.00381135678169
# 2 -0.00381180576112
# 3 -0.00381215316105
# 4 -0.00381243909698
# 5 -0.00381268388507
# 6 -0.00381289923058
# 7 -0.00381309249154
# 8 -0.00381326859415
# 9 -0.00381343099795