Skip to content

Instantly share code, notes, and snippets.

@riveSunder
Created September 19, 2020 17:39
Show Gist options
  • Save riveSunder/17b7f2410133f2f7e1cc0ebead0dd2cd to your computer and use it in GitHub Desktop.
Save riveSunder/17b7f2410133f2f7e1cc0ebead0dd2cd to your computer and use it in GitHub Desktop.
Adam Update
"""
Just an example of computing the adam update for a list of parameter gradients.
"""
def adam_update(l_grad, l_m=None, l_v=None):
# l_n = list of running exponential average of first moment of gradient
# l_v = list of running exponential average of second moment of gradient
# l_grad = list of gradients of current batch
β1 = 0.9
β2 = 0.999
ϵ = 1e-7
l_update = []
l_mt1 = []
l_vt1 = []
if l_m is None:
l_m = [elem * 0.0 for elem in l_grad]
if l_v is None:
l_v = [elem * 0.0 for elem in l_grad]
for my_grad, m, v in zip(l_grad, l_m, l_v):
m = β1 * m + (1-β1) * my_grad
v = β2 * v + (1-β2) * my_grad**2
l_update.append((m / (np.sqrt(v) + ϵ)))
l_mt1.append(m)
l_vt1.append(v)
return l_update, l_mt1, l_vt1
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment