https://towardsdatascience.com/10-gradient-descent-optimisation-algorithms-86989510b5e9

learning_rates.tex

\documentclass[10pt]{article}
\usepackage[usenames]{color} %used for font color
\usepackage{amssymb} %maths
\usepackage{amsmath} %maths
\usepackage[utf8]{inputenc} %useful to type directly diacritic characters
\begin{document}
\begin{align*}%w_{t+1} = w_t - \alpha_t \frac{\partial L}{\partial w_t}
%Exponential
%\alpha_t &= \alpha_0 \cdot k^t \\
%\alpha_t &= \alpha_0 \cdot k^{\lfloor \frac{t}{S} \rfloor} \\
%Exponential
%\alpha_t &= \alpha_0 \cdot e^{-kt} \\
%\alpha_t &= \alpha_0 \cdot e^{-k{\lfloor \frac{t}{S} \rfloor}} \\
%%Inverse time
%\alpha_t &= \alpha_0 \cdot \frac{1}{1+kt} \\
%\alpha_t &= \alpha_0 \cdot \frac{1}{1+k \Big\lfloor \frac{t}{S} \Big\rfloor} \\
%Polynomial decay
%\alpha_t &= \alpha_0 \cdot (1 - \frac{t}{T})^k \\
%\alpha_t &= \alpha_0 \cdot (1 - \frac{\big\lfloor \frac{t}{S} \big\rfloor}{T})^k \\
%%Cosine annealing
%\alpha_t &= \alpha_{\text{min}} + (\alpha_{\text{max}} - \alpha_{\text{min}}) \cdot \frac{1}{2} \Big(1 + \text{cos} \frac{t^*}{S} \pi \Big)  \\
%SGDR
%\alpha_t &= \alpha_{\text{min}} + \frac{1}{2} (\alpha_{\text{max}} - \alpha_{\text{min}}) \Big[1 + \text{cos}\Big(\frac{t^*}{S} \pi \Big) \Big]  \\
%Cyclic triangular
%\alpha_t &= \alpha_{min} + (\alpha_{max} - \alpha_{min}) \cdot max(0,1-x) \\
%x &= \Big|\frac{t}{S} - 2C + 1 \Big| \\
%C &= \Big\lfloor 1 + \frac{t}{2S} \Big\rfloor
%Cyclic triangular2
%\alpha_t &= \alpha_{min} + (\alpha_{max} - \alpha_{min}) \cdot max(0,1-x) \cdot \frac{1}{2^{C-1}} \\
%x &= \Big|\frac{t}{S} - 2C + 1 \Big| \\
%C &= \Big\lfloor 1 + \frac{t}{2S} \Big\rfloor
%Cyclic exp
%\alpha_t &= \alpha_{min} + (\alpha_{max} - \alpha_{min}) \cdot max(0,1-x) \cdot \gamma^t \\
%x &= \Big|\frac{t}{S} - 2C + 1 \Big| \\
%C &= \Big\lfloor 1 + \frac{t}{2S} \Big\rfloor
% Please add the following required packages to your document preamble:
% \usepackage{multirow}
% \usepackage[table,xcdraw]{xcolor}
% If you use beamer only pass "xcolor=table" option, i.e. \documentclass[xcolor=table]{beamer}
% Please add the following required packages to your document preamble:
% \usepackage[table,xcdraw]{xcolor}
% If you use beamer only pass "xcolor=table" option, i.e. \documentclass[xcolor=table]{beamer}
% Please add the following required packages to your document preamble:
% \usepackage[table,xcdraw]{xcolor}
% If you use beamer only pass "xcolor=table" option, i.e. \documentclass[xcolor=table]{beamer}
% Please add the following required packages to your document preamble:
% \usepackage[table,xcdraw]{xcolor}
% If you use beamer only pass "xcolor=table" option, i.e. \documentclass[xcolor=table]{beamer}
\begin{tabular}{|l|c|c|c|}
\hline
\rowcolor[HTML]{EFEFEF} 
\multicolumn{1}{|c|}{\cellcolor[HTML]{EFEFEF}\textbf{Scheduler}} & \textbf{Decay} & \textbf{Cyclic} & \textbf{With restart} \\ \hline
Exponential Decay                                                & \checkmark     &                 &                       \\ \hline
Natural Exponential Decay                                        & \checkmark     &                 &                       \\ \hline
Inverse Time Decay                                               & \checkmark     &                 &                       \\ \hline
Polynomial Decay                                                 & \checkmark     &                 &                       \\ \hline
Cosine Decay                                                     & \checkmark     &                 &                       \\ \hline
Cyclic Learning Rate                                             &                & \checkmark      &                       \\ \hline
SGDR                                                             &                &                 & \checkmark            \\ \hline
\end{tabular}
\end{align*}
\end{document}