garlandkr

Apache config snippets :

Alias /kibana3 /var/www/kibana3

# Elasticsearch API is behind /es/
RewriteEngine On
RewriteCond %{REQUEST_METHOD} ^(TRACE|TRACK|OPTIONS|DELETE)
RewriteRule .* - [F]
RewriteCond %{LA-U:REMOTE_USER} !^$
RewriteRule /es/(.*)logstash-([0-9]+.[0-9]+.[0-9]+)(.*) /es/$1%{LA-U:REMOTE_USER}-$2$3 [N] # Redirect all logstash-xx-xx-xx accesses to user aliases

garlandkr

# NCSA log format, to be used by HTTP log analyzers
VARNISHNCSA_ENABLED=1

# JSON formatting for Daemon Options
DAEMON_OPTS[0]=-a
DAEMON_OPTS[1]=-w
DAEMON_OPTS[2]=/mnt/log/varnish/varnishncsa.log
DAEMON_OPTS[3]=-D
DAEMON_OPTS[4]=-F
DAEMON_OPTS[5]="{ \"remote_addr\": \"%h\",\"remote_user\": \"%u\",\"x_forwarded_for\": \"%{X-Forwarded-For}i\",\"hit_miss\": \"%{Varnish:hitmiss}x\",\"body_bytes_sent\": \"%b\",\"request_time\": \"%{Varnish:time_firstbyte}x\",\"status\": \"%s\",\"request\": \"%r\",\"host\": \"%{host}i\",\"request_method\": \"%m\",\"time_first_byte\": \"%{Varnish:time_firstbyte}x\",\"handling\": \"%{Varnish:handling}x\",\"http_referrer\": \"%{Referrer}i\",\"http_user_agent\": \"%{User-agent}i\"}"

garlandkr

I've got this script:

NAME=varnishncsa
DAEMON=/usr/bin/$NAME
DAEMON_OPTS="-a -f -w ${LOGFILE} -D -P ${PIDFILE}"

# Include defaults if available
if [ -f /etc/default/$NAME ] ; then
        . /etc/default/$NAME
fi

garlandkr

├── dist
│   ├── varnish
│   │   ├── features
│   │   │   ├── hiera.yaml
│   │   │   ├── manifests
│   │   │   │   └── site.pp
│   │   │   ├── modules
│   │   │   │   ├── varnish_config.feature
│   │   │   │   ├── varnish_init.feature
│   │   │   │   ├── varnish_package.feature

garlandkr

daemon_opts=(-a -w "$logfile" -D -F "{ \"remote_addr\": \"%h\",\"remote_user\": \"%u\",\"x_forwarded_for\": \"i\",\"hit_miss\": \"x\",\"body_bytes_sent\": \"%b\",\"request_time\": \"x\",\"status\": \"%s\",\"request\": \"%r\",\"host\": \"i\",\"request_method\": \"%m\",\"time_first_byte\": \"x\",\"handling\": \"x\",\"http_referrer\": \"i\",\"http_user_agent\": \"i\"}\"")

garlandkr

#!/bin/bash

### BEGIN INIT INFO
# Provides:          varnishncsa
# Required-Start:    $local_fs $remote_fs $network
# Required-Stop:     $local_fs $remote_fs $network
# Default-Start:     2 3 4 5
# Default-Stop:      0 1 6
# Short-Description: Start HTTP accelerator log daemon
# Description:       This script provides logging for varnish

garlandkr

daemon_opts=(-a -w "$logfile" -D -F "{ \"remote_addr\": \"%h\",
    \"remote_user\": \"%u\",
    \"x_forwarded_for\": \"%{X-Forwarded-For}i\",
    \"hit_miss\": \"%{Varnish:hitmiss}x\",
    \"body_bytes_sent\": \"%b\",
    \"request_time\": \"%{Varnish:time_firstbyte}x\",
    \"status\": \"%s\",
    \"request\": \"%r\",\"host\": \"%{host}i\",
    \"request_method\": \"%m\",
    \"time_first_byte\": \"%{Varnish:time_firstbyte}x\",

garlandkr

varnishncsa -F '{
    "@source" => "unknown",
    "@type" => "varnish",
    "@fields" => {
        "remote_addr"=>"%h",
        "remote_user" => "%u",
        "x_forwarded_for" => "%{X-Forwarded-For}i",
        "hit_miss"=>" %{Varnish:hitmiss}x",
        "body_bytes_sent"=>"%b",
        "request_time"=>"%{Varnish:time_firstbyte}x",

garlandkr

pattern => "%{IPORHOST:remoteip} - - \[%{HTTPDATE:timestamp}\] \"%{WORD:verb} %{NOTSPACE:request} HTTP/%{NUMBER:httpversion}\" (?:%{NUMBER:status}|\(null\)) (?:%{NUMBER:bytes}|-) \"(?:%{NOTSPACE:referrer}|-)\" %{QUOTEDSTRING:agent}"

garlandkr

#!/usr/bin/python
import re
import sh
import urllib
 
from lxml import objectify, etree
from distutils.version import StrictVersion
 
url = "https://logstash.objects.dreamhost.com/"  # xml listing of releases
logstash_dir = "/opt/logstash"