remykarem
/ gradient_descent.tex
Last active
December 31, 2022 06:19
https://towardsdatascience.com/10-gradient-descent-optimisation-algorithms-86989510b5e9
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| \documentclass[8pt]{article} | |
| \usepackage[usenames]{color} %used for font color | |
| \usepackage{amssymb} %maths | |
| \usepackage{amsmath} %maths | |
| %\usepackage[utf8]{inputenc} %useful to type directly diacritic characters | |
| \begin{document} | |
| \begin{align*} | |
| \text{Vanilla SGD} \\ | |
| w_{t+1} &= w_t - \alpha \frac{\partial L}{\partial w_t} \\ | |
| \text{Momentum} \\ |
remykarem
/ learning_rates.tex
Last active
December 31, 2022 06:19
https://towardsdatascience.com/10-gradient-descent-optimisation-algorithms-86989510b5e9
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| \documentclass[10pt]{article} | |
| \usepackage[usenames]{color} %used for font color | |
| \usepackage{amssymb} %maths | |
| \usepackage{amsmath} %maths | |
| \usepackage[utf8]{inputenc} %useful to type directly diacritic characters | |
| \begin{document} | |
| \begin{align*}%w_{t+1} = w_t - \alpha_t \frac{\partial L}{\partial w_t} | |
| %Exponential | |
| %\alpha_t &= \alpha_0 \cdot k^t \\ | |
| %\alpha_t &= \alpha_0 \cdot k^{\lfloor \frac{t}{S} \rfloor} \\ |