char_level_lang_model_backprop.py

def clip_gradients(gradients, max_value):
    """
    Implements gradient clipping element-wise on gradients to be
    between the interval [-max_value, max_value].
    """

    for grad in gradients.keys():
        np.clip(gradients[grad], -max_value, max_value, out=gradients[grad])
    return gradients

def rnn_backward(y, parameters, cache):
    """
    Implements Backpropagation on one name.
    """
  
    # Retrieve xs, hs, and probs
    xs, hs, probs = cache
    
    # Initialize all gradients to zero
    dh_next = np.zeros_like(hs[0])
    parameters_names = ["Whh", "Wxh", "b", "Why", "c"]
    grads = {}
    for param_name in parameters_names:
        grads["d" + param_name] = np.zeros_like(parameters[param_name])
    
    # Iterate over all time steps in reverse order starting from Tx
    for t in reversed(range(len(xs))):
        dy = np.copy(probs[t])
        dy[y[t]] -= 1
        grads["dWhy"] += np.dot(dy, hs[t].T)
        grads["dc"] += dy
        dh = np.dot(parameters["Why"].T, dy) + dh_next
        dhraw = (1 - hs[t] ** 2) * dh
        grads["dWhh"] += np.dot(dhraw, hs[t - 1].T)
        grads["dWxh"] += np.dot(dhraw, xs[t].T)
        grads["db"] += dhraw
        dh_next = np.dot(parameters["Whh"].T, dhraw)
        # Clip the gradients using [-5, 5] as the interval
        grads = clip_gradients(grads, 5)
    
    # Get the last hidden state
    h_prev = hs[len(xs) - 1]
    return grads, h_prev