forecast error-based timeseries anomaly detection example

main.juttle

// timeseries anomaly detection based on a forecast confidence interval
// A EWMA smoothed version of the timeseries is computed, and its time-varying
// variance provides an expected range for the subsequent point in the series.
// When the series falls outside this envelope, an event is generated.
// 
// This is ad-hoc and difficult to tune, but demonstrates the basic
// idea of forecast-error-based anomaly detection in a single timeseries.
// 
sub cpu(from, to) { 
  demo cdn metrics 'cpu' -from from -to to -every :m:
    -nhosts 4 -dos .5 -dos_dur :5m: -ripple .3 -cpu_alpha 0.8
  | filter host ~ 'sjc*'
}

reducer ewma(fld, alpha) {
    var yhat;
    var initial = true;

    function update() {
        if (initial) {
            yhat = *fld;
            initial = false;
        } else {
            yhat = alpha * *fld + (1 - alpha) * yhat;
        }
    }
    function result() {
        return yhat;
    }
    function reset() {
        initial = true;
    }
}
reducer ewma_sd(fld, beta) {
    var sigma2, last;
    var initial = true;

    function update() {
        if (initial) {
            sigma2 = 0;
            last = *fld;
            initial = false;
        } else {
            sigma2 = beta * (*fld - last) * (*fld - last) + (1 - beta) * sigma2;
            last = *fld;
        }
    }
    function result() {
        return Math.sqrt(sigma2 / 2);
    }
    function reset() {
        initial = true;
    }
}
const t=3;
cpu -from :2014-01-01T00:00:00: -to :2014-01-01T01:20:00:
| put yhat = ewma(value,0.5) | put sd=ewma_sd(yhat, 0.1), high=yhat+t*sd, low=yhat-t*sd
| ( keep time, value | @timechart -id 'cpu'
  ;
  put name='oob', value= (value > high || value < low) && sd > .04
  | delta value | filter value=true
  | keep time,name,value,sd
  | @events -on 'cpu' -nameField value -messageField sd
  )