Cycle hyper parameter

dynamic_params.py

def cycle_hyper_param(low, high):
    '''Dynamically oscillate a hyper-parameter between two values.
    
    Uses the loss momentum to adjust the rate of change. The idea is
    that the value should move through regions where the loss is flat
    faster, and linger in values where the loss improves.
    '''
    inc = 0.0001
    trend = 0.
    prev = 0.
    curr = 0.
    def get_next_value(new):
        nonlocal inc, trend, prev, curr
        trend = 0.5 * trend + 0.5 * (prev-new)
        curr += inc * (1. / trend)
        if curr >= high or curr < low:
            inc *= -1
        curr = max(low, min(high, curr))
        prev = new
        return curr
    return get_next_value