FluentD configuration

td-agent.conf

#Varnish HLS Stats
<source>
  @type tail
  format /(?<path>[^ ]*) (?<code>[^ ]*) (?<userid>[^ ]*)/
  path /var/log/hls-access.log
  pos_file /var/log/td-agent/hls-access.log.pos
  tag td.varnish.userid
</source>

<filter td.varnish.userid>
  @type grep
  regexp1 path index\.m3u8$
  exclude1 path \.ts$
  exclude2 code ^4\d\d$
</filter>

<match td.varnish.userid>
  # You can install using this command "td-agent-gem install fluent-plugin-statsd-output"
  type statsd 
  # I use telegraf for this (https://www.influxdata.com/open-source/#telegraf)
  host yout.statsd.server.here
  port 8125 # optional
  namespace streams # optional but recommended to more easy filtering later
  buffer_type memory
  # this tuning be important to avoid "queue size exceeds limit" warning message
  buffer_chunk_limit 256m
  buffer_queue_limit 8192
  num_threads 4
  
  <metric>
    # Explanation about SETs Type 
    # https://github.com/etsy/statsd/blob/master/docs/metric_types.md#sets
    statsd_type set
    statsd_val ${record['userid']}
    statsd_key ${record['path']}
  </metric>
</match>