Generate a timeseries for splunk

README.md

Extended Example (generate apache logs for a 24hr battle of 100k events):

steps=$(ruby normal.rb -s 2 -m 9 -n 25000 -- 0 24)
yes '12.154.191.10 - - [%d/%b/%Y:%H:%M:%S %z] "POST /battle/lasers?color=red HTTP/1.1" 200 -' |
ruby timeseries.rb -s '2011/11/11' -t '2011/11/12' $steps > thebattle.tmp
steps=$(ruby normal.rb -s 2 -m 15 -n 25000 -- 0 24)
yes '204.12.10.120 - - [%d/%b/%Y:%H:%M:%S %z] "POST /battle/lasers?color=blue HTTP/1.1" 200 -' |
ruby timeseries.rb -s '2011/11/11' -t '2011/11/12' $steps >> thebattle.tmp
steps=$(ruby normal.rb -s 3 -m 12 -n 50000 -- 0 24)
cat > events.tmp <<DOC
12.154.191.10 - - [%d/%b/%Y:%H:%M:%S %z] "GET /battle/knowing HTTP/1.1" 500 36
204.12.10.120 - - [%d/%b/%Y:%H:%M:%S %z] "GET /battle/knowing HTTP/1.1" 200 48
DOC
n=$(( 50000 / 2 ))
while [ $((n--)) -gt 0 ]; do cat events.tmp; done |
ruby timeseries.rb -s 2011/11/11 -t 2011/11/12 $steps >> thebattle.tmp
sort -k 3 thebattle.tmp > access_common.txt
rm events.tmp thebattle.tmp

Then in splunk:

index=thebattle | rex field=uri "/battle/(?<thing>[^?]+)(\?color=(?<color>\w+))?" | eval color=if(isnull(color), "", color) | eval strategy=color.thing | chart count by strategy
index=thebattle | rex field=uri "/battle/(?<thing>[^?]+)(\?color=(?<color>\w+))?" | eval color=if(isnull(color), "", color) | eval strategy=color.thing | chart count over date_hour by strategy
index=thebattle | rex field=uri "/battle/(?<thing>[^?]+)(\?color=(?<color>\w+))?" | eval color=if(isnull(color), "", color) | eval strategy=color.thing | stats avg(bytes) by strategy
index=thebattle | rex field=uri "/battle/(?<thing>[^?]+)(\?color=(?<color>\w+))?" | eval color=if(isnull(color), "", color) | eval strategy=color.thing | search strategy="knowing" | stats sum(bytes) by clientip

Try setting up summary indexing:

search: index=thebattle | sistats avg(bytes) by strategy
start: @h
stop: +1h@h
run: hourly
fields:
report: avg_bytes_by_strategy

index=summary report=avg_bytes_by_strategy | stats avg(bytes) by strategy

To backfill (notice the -1 second to ensure a backfill for the first summary period):

cd /opt/splunk/bin
sudo ./splunk cmd python fill_summary_index.py -app search -name "Summary Avg Bytes by Strategy" -et $((1320969600 - 1)) -lt 1321056000 -dedup true -owner OWNER -auth USER:PASSWORD

normal.rb

#!/usr/bin/env ruby
begin
require 'optparse'

mean = 0
stddev = 1
num_events = 100

OptionParser.new do |opts|
  opts.banner = %{
Usage: #{$0} [options] N M

Generate an integer series representing a normal distribution at integers
between N and M inclusive. Change the mean, standard deviation and maximum
number of events in the series using the options.

  $ normal.rb -- -3 3

Options:
}.lstrip

  opts.on("-h", "--help", "print this help") do
    puts opts
    puts
    exit
  end

  opts.on("-m", "--mean [VALUE]", "mean (0)") do |value|
    mean = value.to_f
  end

  opts.on("-n", "--num-events [VALUE]", "n events in the series (100)") do |value|
    num_events = value.to_i
  end

  opts.on("-s", "--standard-dev [VALUE]", "standard deviation (1)") do |value|
    stddev = value.to_f
  end
end.parse!

n, m = ARGV

if stddev == 0
  raise "standard deviation cannot be zero"
end

# Calculate the normal distribution with specified mean,stddev.  Normalize to
# percents and calculate the number of events per step in the series.
#

include Math
def calc(x, mean=0, stddev=1)
  1 / (stddev * sqrt(2*PI)) * exp(-1 * ((x-mean) ** 2)/(2 * (stddev ** 2)))
end

fractions = (n.to_i..m.to_i).to_a.map {|x| calc(x, mean, stddev) }
total     = fractions.inject(0) {|sum, frac| sum + frac }
percents  = fractions.map {|frac| frac/total }

steps = percents.map {|percent| (num_events * percent).floor }
total_events = steps.inject(0) {|sum, n| sum + n }

#
# Evenly distribute any remaining events across the series.
#

remainder = num_events - total_events
if remainder > 0
  # Make an array of indicies to distribute events evenly from the peak. Odd
  # remainders distribute symmetrically (the odd value goes to the peak) while
  # even remainders put one extra event on the leading edge of the series.
  a, b = Array.new(steps.length) {|i| i }.partition {|i| i >= steps.index(steps.max) }
  distribution = a.zip(b.reverse).flatten

  # Note the remainder cannot be larger than the distribution array because
  # the previous calculation only rounds down once per step.
  distribution.inject(remainder) do |n, i|
    if i
      steps[i] += 1 if n > 0
      n - 1
    else
      n
    end
  end
end

#
# Print out the result
#

steps.each do |n|
  puts n
end

rescue
  raise if $DEBUG
  puts "#{$!.message} (see '#{$0} --help')"
  exit 1
end

timeseries.rb

#!/usr/bin/env ruby
begin
require 'optparse'
require 'time'

start  = nil
stop   = nil
period = 60
format = nil

OptionParser.new do |opts|
    opts.banner = %{
Usage: #{$0} [options] N_EVENTS_IN_STEP...

Generates a time series of n events evenly spaced in periods of 1 minute per
step. The start time and period may be adjusted using the configs. If a stop
time is specified then the period will be calculated by evenly spacing the
steps (not the events) between the start and stop times. The format should be
a strftime. If no format is specified then the format for each line will be
read from stdin.

Example:

  # two steps of 3 and 6 events each, starting 11/11/11
  $ ruby timeseries.rb -f "%d/%m/%Y %H:%M:%S" -s '11/11/11' 3 6 

  # same but reading format from stdin
  $ yes "%d/%m/%Y %H:%M:%S" | ruby timeseries.rb -s '11/11/11' 3 6

Extended Example:

  # generate apache logs for a 24hr battle of 100k events
  $ steps=$(ruby normal.rb -s 1.5 -n 25000 -- -6 6)
  $ yes '12.154.191.10 - - [%d/%b/%Y:%H:%M:%S %z] "POST /battle/lasers?color=red HTTP/1.1" 200 -' |
  > ruby timeseries.rb -s '2011/11/11 00:00' -t '2011/11/11 12:00' $steps > thebattle.tmp
  $ yes '12.154.191.10 - - [%d/%b/%Y:%H:%M:%S %z] "POST /battle/lasers?color=blue HTTP/1.1" 200 -' |
  > ruby timeseries.rb -s '2011/11/11 12:00' -t '2011/11/12 00:00' $steps >> thebattle.tmp
  $ steps=$(ruby normal.rb -s 3 -n 50000 -- -12 12)
  $ cat > events.tmp <<DOC
  > 208.80.152.201 - - [%d/%b/%Y:%H:%M:%S %z] "GET /battle/knowing HTTP/1.1" 200 36
  > 208.80.152.201 - - [%d/%b/%Y:%H:%M:%S %z] "GET /battle/knowing HTTP/1.1" 200 48
  > DOC
  $ n=$(( 50000 / 2 ))
  $ while [ $((n--)) -gt 0 ]; do cat events.tmp; done |
  > ruby timeseries.rb -s 2011/11/11 -t 2011/11/12 $steps >> thebattle.tmp
  $ sort -k 3 thebattle.tmp > access_common.txt
  $ rm events.tmp thebattle.tmp

Options:
}.lstrip

  opts.on("-h", "--help", "print this help") do
    puts opts
    puts
    exit
  end

  opts.on("-s", "--start-time [TIME]", "start time (now)") do |value|
    start = Time.parse(value)
  end

  opts.on("-t", "--stop-time [TIME]", "evenly space steps to a stop time (nil)") do |value|
    stop = Time.parse(value)
  end

  opts.on("-p", "--period [VALUE]", "n seconds per step, if no stop time (60)") do |value|
    period = value.to_i
  end

  opts.on("-f", "--format [STR]", "timestamp strftime format") do |value|
    format = value
  end
end.parse!

num_events = ARGV

def get_format
  $stdin.gets || begin
    raise "no more formats available on stdin"
  end
end

now = start || Time.now
per = stop  ? (stop - start)/num_events.length : period

num_events.each do |num|
  num  = num.to_i
  if num > 0
    step = per.to_f/num
    num.times do
      puts now.strftime(format || get_format)
      now += step
    end
  else
    now += per
  end
end

rescue
  raise if $DEBUG
  puts "#{$!.message} (see '#{$0} --help')"
  exit 1
end